April 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Add IRC travis build notifications
by art＠torproject.org 29 Apr '16

29 Apr '16

commit bc62c2d9b28e763bf2aa7abba924782e545a8ae3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 20:14:03 2015 +0100 Add IRC travis build notifications IRC build notifications will be send to channel #ooni (OFTC) triggered on build success change and always on build failure --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index db639dd..f265676 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,3 +34,9 @@ script: - sudo $(which coverage) run $(which trial) ooni after_success: - coveralls +notifications: + irc: + channels: + - "irc.oftc.net#ooni" + on_success: change + on_failure: always

1 0

[ooni-probe/master] Proper fix for issue reported by @anadahz in #401
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 11511563659b4d3a18aa9e2c1a189d3cf1139155 Author: Simone Basso <bassosimone(a)gmail.com> Date: Tue Nov 17 15:44:03 2015 +0100 Proper fix for issue reported by @anadahz in #401 --- ooni/utils/trueheaders.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index ac73970..149b440 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -20,6 +20,9 @@ SOCKS5ClientFactory.noisy = False from ooni.utils import log +import twisted +from twisted.python.versions import Version + class TrueHeaders(http_headers.Headers): def __init__(self, rawHeaders=None): @@ -160,7 +163,19 @@ class TrueHeadersAgent(client.Agent): self._pool = HTTPConnectionPool(reactor, False) +_twisted_15_0 = Version('twisted', 15, 0, 0) + + class TrueHeadersSOCKS5Agent(SOCKS5Agent): def __init__(self, *args, **kw): super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) - self._pool = HTTPConnectionPool(reactor, False) + pool = HTTPConnectionPool(reactor, False) + # + # With Twisted > 15.0 txsocksx wraps the twisted agent using a + # wrapper class, hence we must set the _pool attribute in the + # inner class rather than into its external wrapper. + # + if twisted.version >= _twisted_15_0: + self._wrappedAgent._pool = pool + else: + self._pool = pool

1 0

[ooni-probe/master] initial psiphon test. Failure cases doesn't fail propertly
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 01bf963445daea590510044ea40269d9762498fe Author: juga0 <juga> Date: Sun Sep 20 21:09:06 2015 +0000 initial psiphon test. Failure cases doesn't fail propertly --- ooni/nettests/third_party/psiphon.py | 141 +++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py new file mode 100644 index 0000000..f8cb82a --- /dev/null +++ b/ooni/nettests/third_party/psiphon.py @@ -0,0 +1,141 @@ +import tempfile +import stat +import os + +from twisted.internet import defer, reactor +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.web.client import readBody +from twisted.python import usage + +from txsocksx.http import SOCKS5Agent + +from ooni.errors import handleAllFailures, TaskTimedOut +from ooni.utils import log +from ooni.templates import process, httpt +from ooni.templates.process import ProcessTest + + +class UsageOptions(usage.Options): + log.debug("UsageOptions") + optParameters = [ + ['url', 'u', None, 'Specify a single URL to test.'], + ['psiphonpath', 'p', None, 'Specify psiphon python client path.'], + ['socksproxy', 's', None, 'Specify psiphon socks proxy ip:port.'],] + + +class PsiphonTest(httpt.HTTPTest, process.ProcessTest): + + """ + This class tests Psiphon python client + + test_psiphon: + Starts a Psiphon, check if it bootstraps successfully + (print a line in stdout). + Then, perform an HTTP request using the proxy + """ + + name = "Psiphon Test" + description = "Bootstraps Psiphon and \ + does a HTTP GET for the specified URL" + author = "juga" + version = "0.0.1" + timeout = 20 + usageOptions = UsageOptions + + def _setUp(self): + # it is necessary to do this in _setUp instead of setUp + # because it needs to happen before HTTPTest's _setUp. + # incidentally, setting this option in setUp results in HTTPTest + # *saying* it is using this proxy while not actually using it. + log.debug('PiphonTest._setUp: setting socksproxy') + self.localOptions['socksproxy'] = '127.0.0.1:1080' + super(PsiphonTest, self)._setUp() + + def setUp(self): + log.debug('PsiphonTest.setUp') + + self.bootstrapped = defer.Deferred() + if self.localOptions['url']: + self.url = self.localOptions['url'] + else: + # FIXME: use http://google.com? + # self.url = 'https://wtfismyip.com/text' + self.url = 'https://check.torproject.orggg' + + if self.localOptions['psiphonpath']: + self.psiphonpath = self.localOptions['psiphonpath'] + else: + # FIXME: search for pyclient path instead of assuming is in the + # home? + # psiphon is not installable and to run it manually, it has to be + # run from the psiphon directory, so it wouldn't make sense to + # nstall it in the PATH + from os import path, getenv + self.psiphonpath = path.join( + getenv('HOME'), + 'psiphon-circumvention-system/pyclient') + log.debug('psiphon path: %s' % self.psiphonpath) + + x = """#!/usr/bin/env python +from psi_client import connect +connect(False) +""" + f = tempfile.NamedTemporaryFile(delete=False) + f.write(x) + f.close() + os.chmod(f.name, os.stat(f.name).st_mode | stat.S_IEXEC) + self.command = [f.name] + log.debug('command: %s' % ''.join(self.command)) + + def handleRead(self, stdout, stderr): + if 'Press Ctrl-C to terminate.' in self.processDirector.stdout: + if not self.bootstrapped.called: + log.debug("PsiphonTest.test_psiphon: calling bootstrapped.callback") + self.bootstrapped.callback(None) + + def test_psiphon(self): + log.debug('PsiphonTest.test_psiphon') + + if not os.path.exists(self.psiphonpath): + log.debug('psiphon path does not exists, is it installed?') + self.report['success'] = False + self.report['psiphon_installed'] = False + log.debug("Adding %s to report" % self.report) + #FIXME: this completes the test but ooni doesn't stop running, why? + return defer.succeed(None) + + self.report['psiphon_installed'] = True + log.debug("Adding %s to report" % self.report) + + finished = self.run(self.command, + env=dict(PYTHONPATH=self.psiphonpath), + path=self.psiphonpath, + usePTY=1) + + + def addFailureToReport(failure): + log.debug("PsiphonTest.test_psiphon.addFailureToReport") + log.debug(repr(failure )) + self.report['failure'] = handleAllFailures(failure) + self.report['success'] = False + log.debug("Adding %s to report" % self.report) + # FIXME: these keys are not being wroten in the report + # probably because report is being defined in NetTestCase as + # a class attribute that is initialized again in NetTescase._setUp + + def callDoRequest(_): + return self.doRequest(self.url) + self.bootstrapped.addCallback(callDoRequest) + + def cleanup(_): + log.debug('PsiphonTest:cleanup') + self.processDirector.transport.signalProcess('INT') + os.remove(self.command[0]) + return finished + + self.bootstrapped.addErrback(addFailureToReport) + self.bootstrapped.addBoth(cleanup) + return self.bootstrapped + + +

1 0

[ooni-probe/master] add psiphon_install.sh, code should be improved
by art＠torproject.org 29 Apr '16

29 Apr '16

commit c3e0912271313d1d28c3d31bd182586829a1dd52 Author: juga0 <juga> Date: Sun Sep 20 21:23:47 2015 +0000 add psiphon_install.sh, code should be improved --- scripts/psiphon_install.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh new file mode 100644 index 0000000..e03e663 --- /dev/null +++ b/scripts/psiphon_install.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +PSIPHON_PATH=$HOME +PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient + +sudo apt-get install mercurial +cd $PSIPHON_PATH +hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system +# optional, compile their ssh +#cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ +#./configure +#make +#cp ssh ../../../pyclient/ +#cd $PSIPHON_PATH + +# create psi_client.dat +cat <<EOF > psi_generate_dat.py +#!/usr/bin/env python + +import wget +import os +import json + +# Delete 'server_list' if exists +if os.path.exists("server_list"): + # os.remove("server_list") + # os.rename("server_list", "server_list") + pass +else: + # Download 'server_list' + url ="https://psiphon3.com/server_list' + wget.download(url) + +# convert server_list to psi_client.dat +dat = {} +dat["propagation_channel_id"] = "FFFFFFFFFFFFFFFF" +dat["sponsor_id"] = "FFFFFFFFFFFFFFFF" +dat["servers"] = json.load(open('server_list'))['data'].split() +json.dump(dat, open('psi_client.dat', 'w')) +EOF + +chmod +x psi_generate_dat.py +./psi_generate_dat.py +cp psi_client.dat $PSIPHON_PYCLIENT_PATH + +# assuming to be inside a virtualenv +pip install jsonpickle pexpect + +# run psiphon +# cd $PSIPHON_PYCLIENT_PATH +# psi_client.py

1 0

[ooni-probe/master] Skip failing tests
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 4c794e8303eda88f0e50da2ec34a4e6cbabc63b3 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 19:42:29 2015 +0100 Skip failing tests test_http_header_field_manipulation uses an unreachable backend test_sniffing_activated due to non configured packat capture (?) --- ooni/tests/test_oonicli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 9e73061..834fb55 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -151,6 +151,7 @@ class TestRunDirector(ConfigTestCase): @defer.inlineCallbacks def test_http_header_field_manipulation(self): + self.skipTest("This test requires a property configured backend") def verify_function(entry): assert 'agent' in entry assert 'requests' in entry @@ -170,6 +171,7 @@ class TestRunDirector(ConfigTestCase): @defer.inlineCallbacks def test_sniffing_activated(self): + self.skipTest("Not properly set packet capture?") filename = os.path.abspath('test_report.pcap') self.filenames.append(filename) conf_file = os.path.abspath('fake_config.conf')

1 0

[ooni-probe/master] Minor improvements commented on https://github.com/TheTorProject/ooni-probe/pull/410
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9921f4827f58c2282b00903f3169a2ca243df9a5 Author: juga0 <juga> Date: Wed Nov 11 12:48:16 2015 +0000 Minor improvements commented on https://github.com/TheTorProject/ooni-probe/pull/410 * add comment about why using usePTY * add comment about why is needed to run psiphon from an script * fix identation --- ooni/nettests/third_party/psiphon.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/third_party/psiphon.py b/ooni/nettests/third_party/psiphon.py index 6a2e08e..9fec2fd 100644 --- a/ooni/nettests/third_party/psiphon.py +++ b/ooni/nettests/third_party/psiphon.py @@ -25,10 +25,10 @@ class UsageOptions(usage.Options): class PsiphonTest(httpt.HTTPTest, process.ProcessTest): - + """ This class tests Psiphon python client - + test_psiphon: Starts a Psiphon, check if it bootstraps successfully (print a line in stdout). @@ -51,7 +51,7 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): log.debug('PiphonTest._setUp: setting socksproxy') self.localOptions['socksproxy'] = '127.0.0.1:1080' super(PsiphonTest, self)._setUp() - + def setUp(self): log.debug('PsiphonTest.setUp') @@ -77,6 +77,8 @@ class PsiphonTest(httpt.HTTPTest, process.ProcessTest): 'psiphon-circumvention-system/pyclient') log.debug('psiphon path: %s' % self.psiphonpath) + # psi_client.py can not be run directly because the paths in the + # code are relative, so it'll fail to execute from this test x = """ from psi_client import connect connect(False) @@ -107,6 +109,10 @@ connect(False) self.report['psiphon_installed'] = True log.debug("Adding %s to report" % self.report) + # Using pty to see output lines as soon as they get wrotten in the + # buffer, otherwise the test might not see lines until the buffer is + # full with some block size and therefore the test would + # terminate with error finished = self.run(self.command, env=dict(PYTHONPATH=self.psiphonpath), path=self.psiphonpath, @@ -121,7 +127,7 @@ connect(False) self.processDirector.transport.signalProcess('INT') os.remove(self.command[1]) return finished - + self.bootstrapped.addBoth(cleanup) return self.bootstrapped

1 0

[ooni-probe/master] Downgrading Twisted version is not needed
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 9402fe9511b377507d22358eed75cac97eb5f7a0 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 21:50:08 2015 +0100 Downgrading Twisted version is not needed Thanks to @bassosimone --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 7c624dc..bf52114 100644 --- a/README.rst +++ b/README.rst @@ -60,12 +60,6 @@ On unix systems:: sudo pip install ooniprobe -**BUG** Note: Twisted version needs to be >=12.2.0 and <=14.0.0 -unistall/downgrage Twisted:: - - sudo pip uninstall Twisted - sudo pip install 'Twisted>=12.2.0,<=14.0.0' - To install it from the current master run:: sudo pip install https://github.com/TheTorProject/ooni-probe/archive/master.zip

1 0

[ooni-probe/master] Add all supported twisted version, revise build dependencies
by art＠torproject.org 29 Apr '16

29 Apr '16

commit ff417896e2bdedd13ac6e598c471a9ce4df9d0a2 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 20:09:51 2015 +0100 Add all supported twisted version, revise build dependencies Include all Twisted versions >= 12.2 and latest as travis ENV tests Add required build dependencies packages Remove duplicate pip package pydnet (pydumpnet) --- .travis.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99bb48f..db639dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,22 @@ language: python env: - TWISTED=Twisted==12.2 - TWISTED=Twisted==12.3 + - TWISTED=Twisted==13.0 + - TWISTED=Twisted==13.1 - TWISTED=Twisted==13.2 - TWISTED=Twisted==14.0 + - TWISTED=Twisted==14.0.1 + - TWISTED=Twisted==14.0.2 + - TWISTED=Twisted==15.0.0 + - TWISTED=Twisted==15.1.0 + - TWISTED=Twisted==15.2.0 + - TWISTED=Twisted==15.2.1 + - TWISTED=Twisted==15.3.0 + - TWISTED=Twisted==15.4.0 + - TWISTED=Twisted before_install: - - sudo apt-get install tor libpcap-dev libgeoip-dev libdumbnet-dev + - sudo apt-get install tor libpcap-dev libgeoip-dev libdumbnet-dev libffi-dev + libdumbnet-dev libssl-dev - sudo /etc/init.d/tor start python: - "2.7" @@ -13,7 +25,6 @@ python: # the first is for testing pip and the second for setuptools install: - pip install $TWISTED pyOpenSSL coveralls - - pip install https://github.com/hellais/pydnet/archive/master.zip - pip install pyrex-real - pip install -r requirements.txt - pip install -r requirements-dev.txt

1 0

[ooni-probe/master] Remove script to install psiphon
by art＠torproject.org 29 Apr '16

29 Apr '16

commit 867efea7487bd9168a1b7c543954ea2a910587f1 Author: Simone Basso <bassosimone(a)gmail.com> Date: Sun Nov 15 16:11:18 2015 +0100 Remove script to install psiphon We will write a more ad-hoc script to install psiphon as part of the lepidopter install process of OONI. This way we don't need to worry about portability to MacOS and similar issues since we know we'll be on debian. --- scripts/psiphon_install.sh | 51 ---------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/scripts/psiphon_install.sh b/scripts/psiphon_install.sh deleted file mode 100644 index e03e663..0000000 --- a/scripts/psiphon_install.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -PSIPHON_PATH=$HOME -PSIPHON_PYCLIENT_PATH=$PSIPHON_PATH/psiphon-circumvention-system/pyclient - -sudo apt-get install mercurial -cd $PSIPHON_PATH -hg clone https://bitbucket.org/psiphon/psiphon-circumvention-system -# optional, compile their ssh -#cd psiphon-circumvention-system/Server/3rdParty/openssh-5.9p1/ -#./configure -#make -#cp ssh ../../../pyclient/ -#cd $PSIPHON_PATH - -# create psi_client.dat -cat <<EOF > psi_generate_dat.py -#!/usr/bin/env python - -import wget -import os -import json - -# Delete 'server_list' if exists -if os.path.exists("server_list"): - # os.remove("server_list") - # os.rename("server_list", "server_list") - pass -else: - # Download 'server_list' - url ="https://psiphon3.com/server_list' - wget.download(url) - -# convert server_list to psi_client.dat -dat = {} -dat["propagation_channel_id"] = "FFFFFFFFFFFFFFFF" -dat["sponsor_id"] = "FFFFFFFFFFFFFFFF" -dat["servers"] = json.load(open('server_list'))['data'].split() -json.dump(dat, open('psi_client.dat', 'w')) -EOF - -chmod +x psi_generate_dat.py -./psi_generate_dat.py -cp psi_client.dat $PSIPHON_PYCLIENT_PATH - -# assuming to be inside a virtualenv -pip install jsonpickle pexpect - -# run psiphon -# cd $PSIPHON_PYCLIENT_PATH -# psi_client.py

1 0

[ooni-probe/master] Remove duplicate package
by art＠torproject.org 29 Apr '16

29 Apr '16

commit b5781702b1f9e287dd8921585d32835f2d27bd7a Author: anadahz <kojgelo(a)inbox.com> Date: Tue Nov 17 21:44:59 2015 +0100 Remove duplicate package Remove duplicate apt package libdumbnet-dev As spotted by @bassosimone --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 926468a..fb6a033 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ env: - TWISTED=Twisted before_install: - sudo apt-get install tor libpcap-dev libgeoip-dev libdumbnet-dev libffi-dev - libdumbnet-dev libssl-dev + libssl-dev - sudo /etc/init.d/tor start python: - "2.7"

1 0