- tor-commits - lists.torproject.org

[ooni-probe/master] psiphon: fix bug that psiphon hangs ooniprobe -ng
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 85ae20219c13a19e26896d867ea121fdf24ccc0e Author: Simone Basso <bassosimone(a)gmail.com> Date: Sun Nov 15 16:58:37 2015 +0100 psiphon: fix bug that psiphon hangs ooniprobe -ng Marked my fix with XXX because I'm not sure whether this is the proper approach, or whether it would be better to work on a patch to teach ooniprobe's core to deal with that case. For now, and given that a psiphon MVP must be delivered as soon as possible, this fix will do. --- ooni/nettests/third_party/psiphon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 637a505..543e856 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -95,12 +95,18 @@ connect(False) log.debug('PsiphonTest.test_psiphon') if not os.path.exists(self.psiphonpath): - log.debug('psiphon path does not exists, is it installed?') + log.err('psiphon path does not exists, is it installed?') self.report['success'] = False self.report['psiphon_installed'] = False log.debug("Adding %s to report" % self.report) - #FIXME: this completes the test but ooni doesn't stop running, why? - return defer.succeed(None) + # XXX: the original code written by juga0 readed + # > return defer.succeed(None) + # but this caused `ooniprobe -ng` to hang forever, so I + # rewrote the code to return a deferred and simulate calling + # its callback method, to trigger an event. + # -sbs + reactor.callLater(0.0, self.bootstrapped.callback, None) + return self.bootstrapped self.report['psiphon_installed'] = True log.debug("Adding %s to report" % self.report)

1 0

[ooni-probe/master] change the way HTTP request errors are handled
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 6da1e0e2dd98aaeafe31b6f2241d92223738551d Author: juga0 <juga> Date: Thu Sep 24 23:03:30 2015 +0000 change the way HTTP request errors are handled * eliminate addFailureToReport because the errors are handled by HTTPTest errback and addToReport * add ConnectionDone exception to handleAllErrors, so that an HTTP request to a domain that is not reachable is reported --- ooni/errors.py | 8 ++++++-- ooni/nettests/third_party/psiphon.py | 16 ++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 1368f45..cb18fe9 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -6,7 +6,7 @@ from twisted.web.error import Error from twisted.internet.error import ConnectionRefusedError, TCPTimedOutError from twisted.internet.error import DNSLookupError, ConnectError, ConnectionLost from twisted.internet.error import TimeoutError as GenericTimeoutError -from twisted.internet.error import ProcessDone +from twisted.internet.error import ProcessDone, ConnectionDone from twisted.python import usage @@ -50,7 +50,8 @@ def handleAllFailures(failure): CommandNotSupported, ConnectError, ConnectionLost, - CancelledError) + CancelledError, + ConnectionDone) return failureToString(failure) @@ -148,6 +149,9 @@ def failureToString(failure): elif isinstance(failure.value, ProcessDone): string = 'process_done' + elif isinstance(failure.value, ConnectionDone): + string = 'connection_done' + else: # log.err("Unknown failure type: %s" % type(failure.value)) string = 'unknown_failure %s' % str(failure.value) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index f8cb82a..8ac287d 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -60,7 +60,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): else: # FIXME: use http://google.com? # self.url = 'https://wtfismyip.com/text' - self.url = 'https://check.torproject.orggg' + self.url = 'https://check.torproject.org' if self.localOptions['psiphonpath']: self.psiphonpath = self.localOptions['psiphonpath'] @@ -112,17 +112,6 @@ connect(False) path=self.psiphonpath, usePTY=1) - - def addFailureToReport(failure): - log.debug("PsiphonTest.test_psiphon.addFailureToReport") - log.debug(repr(failure )) - self.report['failure'] = handleAllFailures(failure) - self.report['success'] = False - log.debug("Adding %s to report" % self.report) - # FIXME: these keys are not being wroten in the report - # probably because report is being defined in NetTestCase as - # a class attribute that is initialized again in NetTescase._setUp - def callDoRequest(_): return self.doRequest(self.url) self.bootstrapped.addCallback(callDoRequest) @@ -132,8 +121,7 @@ connect(False) self.processDirector.transport.signalProcess('INT') os.remove(self.command[0]) return finished - - self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(cleanup) return self.bootstrapped

1 0

[ooni-probe/master] Add IRC travis build notifications
by art＠torproject.org 29 Apr '16

29 Apr '16

commit bc62c2d9b28e763bf2aa7abba924782e545a8ae3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 20:14:03 2015 +0100 Add IRC travis build notifications IRC build notifications will be send to channel #ooni (OFTC) triggered on build success change and always on build failure --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index db639dd..f265676 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,3 +34,9 @@ script: - sudo $(which coverage) run $(which trial) ooni after_success: - coveralls +notifications: + irc: + channels: + - "irc.oftc.net#ooni" + on_success: change + on_failure: always

1 0

[ooni-probe/master] Proper fix for issue reported by @anadahz in #401
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 11511563659b4d3a18aa9e2c1a189d3cf1139155 Author: Simone Basso <bassosimone(a)gmail.com> Date: Tue Nov 17 15:44:03 2015 +0100 Proper fix for issue reported by @anadahz in #401 --- ooni/utils/trueheaders.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index ac73970..149b440 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -20,6 +20,9 @@ SOCKS5ClientFactory.noisy = False from ooni.utils import log +import twisted +from twisted.python.versions import Version + class TrueHeaders(http_headers.Headers): def __init__(self, rawHeaders=None): @@ -160,7 +163,19 @@ class TrueHeadersAgent(client.Agent): self._pool = HTTPConnectionPool(reactor, False) +_twisted_15_0 = Version('twisted', 15, 0, 0) + + class TrueHeadersSOCKS5Agent(SOCKS5Agent): def __init__(self, *args, **kw): super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) - self._pool = HTTPConnectionPool(reactor, False) + pool = HTTPConnectionPool(reactor, False) + # + # With Twisted > 15.0 txsocksx wraps the twisted agent using a + # wrapper class, hence we must set the _pool attribute in the + # inner class rather than into its external wrapper. + # + if twisted.version >= _twisted_15_0: + self._wrappedAgent._pool = pool + else: + self._pool = pool

1 0

[ooni-probe/master] initial psiphon test. Failure cases doesn't fail propertly
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 01bf963445daea590510044ea40269d9762498fe Author: juga0 <juga> Date: Sun Sep 20 21:09:06 2015 +0000 initial psiphon test. Failure cases doesn't fail propertly --- ooni/nettests/third_party/psiphon.py | 141 +++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py new file mode 100644 index 0000000..f8cb82a --- /dev/null +++ b/ooni/nettests/third_party/psiphon.py @@ -0,0 +1,141 @@ +import tempfile +import stat +import os + +from twisted.internet import defer, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.web.client import readBody +from twisted.python import usage + +from txsocksx.http import SOCKS5Agent + +from ooni.errors import handleAllFailures, TaskTimedOut +from ooni.utils import log +from ooni.templates import process, httpt +from ooni.templates.process import ProcessTest + + +class UsageOptions(usage.Options): + log.debug("UsageOptions") + optParameters = [ + ['url', 'u', None, 'Specify a single URL to test.'], + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], + ['socksproxy', 's', None, 'Specify psiphon socks proxy ip:port.'],] + + +class PsiphonTest(httpt.HTTPTest, process.ProcessTest): + + """ + This class tests Psiphon python client + + test_psiphon: + Starts a Psiphon, check if it bootstraps successfully + (print a line in stdout). + Then, perform an HTTP request using the proxy + """ + + name = "Psiphon Test" + description = "Bootstraps Psiphon and \ + does a HTTP GET for the specified URL" + author = "juga" + version = "0.0.1" + timeout = 20 + usageOptions = UsageOptions + + def _setUp(self): + # it is necessary to do this in _setUp instead of setUp + # because it needs to happen before HTTPTest's _setUp. + # incidentally, setting this option in setUp results in HTTPTest + # *saying* it is using this proxy while not actually using it. + log.debug('PiphonTest._setUp: setting socksproxy') + self.localOptions['socksproxy'] = '127.0.0.1:1080' + super(PsiphonTest, self)._setUp() + + def setUp(self): + log.debug('PsiphonTest.setUp') + + self.bootstrapped = defer.Deferred() + if self.localOptions['url']: + self.url = self.localOptions['url'] + else: + # FIXME: use http://google.com? + # self.url = 'https://wtfismyip.com/text' + self.url = 'https://check.torproject.orggg' + + if self.localOptions['psiphonpath']: + self.psiphonpath = self.localOptions['psiphonpath'] + else: + # FIXME: search for pyclient path instead of assuming is in the + # home? + # psiphon is not installable and to run it manually, it has to be + # run from the psiphon directory, so it wouldn't make sense to + # nstall it in the PATH + from os import path, getenv + self.psiphonpath = path.join( + getenv('HOME'), + 'psiphon-circumvention-system/pyclient') + log.debug('psiphon path: %s' % self.psiphonpath) + + x = """#!/usr/bin/env python +from psi_client import connect +connect(False) +""" + f = tempfile.NamedTemporaryFile(delete=False) + f.write(x) + f.close() + os.chmod(f.name, os.stat(f.name).st_mode | stat.S_IEXEC) + self.command = [f.name] + log.debug('command: %s' % ''.join(self.command)) + + def handleRead(self, stdout, stderr): + if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: + if not self.bootstrapped.called: + log.debug("PsiphonTest.test_psiphon: calling bootstrapped.callback") + self.bootstrapped.callback(None) + + def test_psiphon(self): + log.debug('PsiphonTest.test_psiphon') + + if not os.path.exists(self.psiphonpath): + log.debug('psiphon path does not exists, is it installed?') + self.report['success'] = False + self.report['psiphon_installed'] = False + log.debug("Adding %s to report" % self.report) + #FIXME: this completes the test but ooni doesn't stop running, why? + return defer.succeed(None) + + self.report['psiphon_installed'] = True + log.debug("Adding %s to report" % self.report) + + finished = self.run(self.command, + env=dict(PYTHONPATH=self.psiphonpath), + path=self.psiphonpath, + usePTY=1) + + + def addFailureToReport(failure): + log.debug("PsiphonTest.test_psiphon.addFailureToReport") + log.debug(repr(failure )) + self.report['failure'] = handleAllFailures(failure) + self.report['success'] = False + log.debug("Adding %s to report" % self.report) + # FIXME: these keys are not being wroten in the report + # probably because report is being defined in NetTestCase as + # a class attribute that is initialized again in NetTescase._setUp + + def callDoRequest(_): + return self.doRequest(self.url) + self.bootstrapped.addCallback(callDoRequest) + + def cleanup(_): + log.debug('PsiphonTest:cleanup') + self.processDirector.transport.signalProcess('INT') + os.remove(self.command[0]) + return finished + + self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(cleanup) + return self.bootstrapped + + +

1 0

[ooni-probe/master] add psiphon_install.sh, code should be improved
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c3e0912271313d1d28c3d31bd182586829a1dd52 Author: juga0 <juga> Date: Sun Sep 20 21:23:47 2015 +0000 add psiphon_install.sh, code should be improved --- scripts/psiphon_install.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh new file mode 100644 index 0000000..e03e663 --- /dev/null +++ b/scripts/psiphon_install.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +PSIPHON_PATH=$HOME +PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient + +sudo apt-get install mercurial +cd $PSIPHON_PATH +hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system +# optional, compile their ssh +#cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ +#./configure +#make +#cp ssh ../../../pyclient/ +#cd $PSIPHON_PATH + +# create psi_client.dat +cat <<EOF > psi_generate_dat.py +#!/usr/bin/env python + +import wget +import os +import json + +# Delete 'server_list' if exists +if os.path.exists("server_list"): + # os.remove("server_list") + # os.rename("server_list", "server_list") + pass +else: + # Download 'server_list' + url ="https://psiphon3.com/server_list' + wget.download(url) + +# convert server_list to psi_client.dat +dat = {} +dat["propagation_channel_id"] = "FFFFFFFFFFFFFFFF" +dat["sponsor_id"] = "FFFFFFFFFFFFFFFF" +dat["servers"] = json.load(open('server_list'))['data'].split() +json.dump(dat, open('psi_client.dat', 'w')) +EOF + +chmod +x psi_generate_dat.py +./psi_generate_dat.py +cp psi_client.dat $PSIPHON_PYCLIENT_PATH + +# assuming to be inside a virtualenv +pip install jsonpickle pexpect + +# run psiphon +# cd $PSIPHON_PYCLIENT_PATH +# psi_client.py

1 0

[ooni-probe/master] Skip failing tests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 4c794e8303eda88f0e50da2ec34a4e6cbabc63b3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 19:42:29 2015 +0100 Skip failing tests test_http_header_field_manipulation uses an unreachable backend test_sniffing_activated due to non configured packat capture (?) --- ooni/tests/test_oonicli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 9e73061..834fb55 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -151,6 +151,7 @@ class TestRunDirector(ConfigTestCase): @defer.inlineCallbacks def test_http_header_field_manipulation(self): + self.skipTest("This test requires a property configured backend") def verify_function(entry): assert 'agent' in entry assert 'requests' in entry @@ -170,6 +171,7 @@ class TestRunDirector(ConfigTestCase): @defer.inlineCallbacks def test_sniffing_activated(self): + self.skipTest("Not properly set packet capture?") filename = os.path.abspath('test_report.pcap') self.filenames.append(filename) conf_file = os.path.abspath('fake_config.conf')

1 0

[ooni-probe/master] Minor improvements commented on https://github.com/TheTorProject/ooni-probe/pull/410
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9921f4827f58c2282b00903f3169a2ca243df9a5 Author: juga0 <juga> Date: Wed Nov 11 12:48:16 2015 +0000 Minor improvements commented on https://github.com/TheTorProject/ooni-probe/pull/410 * add comment about why using usePTY * add comment about why is needed to run psiphon from an script * fix identation --- ooni/nettests/third_party/psiphon.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 6a2e08e..9fec2fd 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -25,10 +25,10 @@ class UsageOptions(usage.Options): class PsiphonTest(httpt.HTTPTest, process.ProcessTest): - + """ This class tests Psiphon python client - + test_psiphon: Starts a Psiphon, check if it bootstraps successfully (print a line in stdout). @@ -51,7 +51,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): log.debug('PiphonTest._setUp: setting socksproxy') self.localOptions['socksproxy'] = '127.0.0.1:1080' super(PsiphonTest, self)._setUp() - + def setUp(self): log.debug('PsiphonTest.setUp') @@ -77,6 +77,8 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): 'psiphon-circumvention-system/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) + # psi_client.py can not be run directly because the paths in the + # code are relative, so it'll fail to execute from this test x = """ from psi_client import connect connect(False) @@ -107,6 +109,10 @@ connect(False) self.report['psiphon_installed'] = True log.debug("Adding %s to report" % self.report) + # Using pty to see output lines as soon as they get wrotten in the + # buffer, otherwise the test might not see lines until the buffer is + # full with some block size and therefore the test would + # terminate with error finished = self.run(self.command, env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, @@ -121,7 +127,7 @@ connect(False) self.processDirector.transport.signalProcess('INT') os.remove(self.command[1]) return finished - + self.bootstrapped.addBoth(cleanup) return self.bootstrapped

1 0

[ooni-probe/master] Downgrading Twisted version is not needed
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9402fe9511b377507d22358eed75cac97eb5f7a0 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 21:50:08 2015 +0100 Downgrading Twisted version is not needed Thanks to @bassosimone --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 7c624dc..bf52114 100644 --- a/README.rst +++ b/README.rst @@ -60,12 +60,6 @@ On unix systems:: sudo pip install ooniprobe -**BUG** Note: Twisted version needs to be >=12.2.0 and <=14.0.0 -unistall/downgrage Twisted:: - - sudo pip uninstall Twisted - sudo pip install 'Twisted>=12.2.0,<=14.0.0' - To install it from the current master run:: sudo pip install https://github.com/TheTorProject/ooni-probe/archive/master.zip

1 0

[ooni-probe/master] Add all supported twisted version, revise build dependencies
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ff417896e2bdedd13ac6e598c471a9ce4df9d0a2 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 20:09:51 2015 +0100 Add all supported twisted version, revise build dependencies Include all Twisted versions >= 12.2 and latest as travis ENV tests Add required build dependencies packages Remove duplicate pip package pydnet (pydumpnet) --- .travis.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99bb48f..db639dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,22 @@ language: python env: - TWISTED=Twisted==12.2 - TWISTED=Twisted==12.3 + - TWISTED=Twisted==13.0 + - TWISTED=Twisted==13.1 - TWISTED=Twisted==13.2 - TWISTED=Twisted==14.0 + - TWISTED=Twisted==14.0.1 + - TWISTED=Twisted==14.0.2 + - TWISTED=Twisted==15.0.0 + - TWISTED=Twisted==15.1.0 + - TWISTED=Twisted==15.2.0 + - TWISTED=Twisted==15.2.1 + - TWISTED=Twisted==15.3.0 + - TWISTED=Twisted==15.4.0 + - TWISTED=Twisted before_install: - - sudo apt-get install tor libpcap-dev libgeoip-dev libdumbnet-dev + - sudo apt-get install tor libpcap-dev libgeoip-dev libdumbnet-dev libffi-dev + libdumbnet-dev libssl-dev - sudo /etc/init.d/tor start python: - "2.7" @@ -13,7 +25,6 @@ python: # the first is for testing pip and the second for setuptools install: - pip install $TWISTED pyOpenSSL coveralls - - pip install https://github.com/hellais/pydnet/archive/master.zip - pip install pyrex-real - pip install -r requirements.txt - pip install -r requirements-dev.txt

1 0