November 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Update README.rst development instructions
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 305fb1204db3d9f7c668dfbe6992838b1474844d Author: hadiasghari <hadiasghari(a)users.noreply.github.com> Date: Sat Oct 10 14:50:00 2015 -0400 Update README.rst development instructions --- README.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 9cb3042..730d8f0 100644 --- a/README.rst +++ b/README.rst @@ -380,9 +380,17 @@ that is different from master) with:: Setting up development environment ---------------------------------- -On debian based systems this can be done with:: +On Debian based systems a development environment can be setup as follows: (prerequisites include build essentials, python-dev, and tor; for tor see https://www.torproject.org/docs/debian.html.en) - sudo apt-get install libgeoip-dev python-virtualenv virtualenvwrapper - mkvirtualenv ooniprobe + + sudo apt-get install python-pip python-virtualenv virtualenv virtualenvwrapper + sudo apt-get install libgeoip-dev libffi-dev libdumbnet-dev libssl-dev libpcap-dev + git clone https://github.com/TheTorProject/ooni-probe + cd ooni-probe + mkvirtualenv ooniprobe # . ~/.virtualenvs/ooniprobe/bin/activate to access later python setup.py install + pip install -r requirements.txt pip install -r requirements-dev.txt + ooniprobe -s # if all went well, lists available tests + +

1 0

[ooni-probe/master] Merge pull request #417 from hadiasghari/patch-1
by art＠torproject.org 20 Nov '15

20 Nov '15

commit ed0fdf50c127764fd7d25416413b46819d68529b Merge: be134f7 305fb12 Author: anadahz <anadahz(a)users.noreply.github.com> Date: Sat Oct 10 19:06:12 2015 +0000 Merge pull request #417 from hadiasghari/patch-1 Update README.rst development instructions README.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)

1 0

[ooni-probe/master] add handleRead
by art＠torproject.org 20 Nov '15

20 Nov '15

commit f91d1369627eac0c8c42b361f9f2131b62b65f80 Author: juga0 <juga> Date: Sat Sep 19 20:55:48 2015 +0000 add handleRead --- ooni/templates/process.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index a3c74ff..7ec4b2e 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -59,12 +59,14 @@ class ProcessDirector(protocol.ProcessProtocol): self.stdout += data if self.shouldClose(): self.close("condition_met") + self.handleRead(data, None) def errReceived(self, data): log.debug("STDERR: %s" % data) self.stderr += data if self.shouldClose(): self.close("condition_met") + self.handlRead(None, data) def inConnectionLost(self): @@ -84,6 +86,9 @@ class ProcessDirector(protocol.ProcessProtocol): log.debug("Ended %s" % reason) self.finish("process_done") + def handleRead(self, stdout, stderr=None): + pass + class ProcessTest(NetTestCase): name = "Base Process Test" @@ -110,5 +115,10 @@ class ProcessTest(NetTestCase): d = defer.Deferred() d.addCallback(self.processEnded, command) self.processDirector = ProcessDirector(d, finished, self.timeout) + self.processDirector.handleRead = self.handleRead reactor.spawnProcess(self.processDirector, command[0], command, env=env, path=path, usePTY=usePTY) return d + + # handleRead is not an abstract method to be backwards compatible + def handleRead(self, stdout, stderr=None): + pass

1 0

[ooni-probe/master] modify resetTimer and cancelTimer to avoid alreadyCancelled exception
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 2a385f9292755fe66fea4f6952c02c033ff715c6 Author: juga0 <juga> Date: Sat Sep 19 19:24:10 2015 +0000 modify resetTimer and cancelTimer to avoid alreadyCancelled exception --- ooni/templates/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index 3bbd8f6..4939441 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -19,6 +19,7 @@ class ProcessDirector(protocol.ProcessProtocol): def cancelTimer(self): if self.timeout and self.timer: self.timer.cancel() + self.timer = None def close(self, reason=None): self.reason = reason @@ -26,7 +27,7 @@ class ProcessDirector(protocol.ProcessProtocol): def resetTimer(self): if self.timeout is not None: - if self.timer is not None: + if self.timer is not None and self.timer.active(): self.timer.cancel() self.timer = reactor.callLater(self.timeout, self.close,

1 0

[ooni-probe/master] add processDirector as a class attribute
by art＠torproject.org 20 Nov '15

20 Nov '15

commit e9d22b1914df503ae6f238abfc3ece2bac2ddf04 Author: juga0 <juga> Date: Sat Sep 19 20:54:36 2015 +0000 add processDirector as a class attribute --- ooni/templates/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/templates/process.py b/ooni/templates/process.py index efe53d3..a3c74ff 100644 --- a/ooni/templates/process.py +++ b/ooni/templates/process.py @@ -91,6 +91,7 @@ class ProcessTest(NetTestCase): requiresRoot = False timeout = 5 + processDirector = None def _setUp(self): super(ProcessTest, self)._setUp() @@ -108,6 +109,6 @@ class ProcessTest(NetTestCase): def run(self, command, finished=None, env={}, path=None, usePTY=0): d = defer.Deferred() d.addCallback(self.processEnded, command) - processDirector = ProcessDirector(d, finished, self.timeout) + self.processDirector = ProcessDirector(d, finished, self.timeout) reactor.spawnProcess(self.processDirector, command[0], command, env=env, path=path, usePTY=usePTY) return d

1 0

[ooni-probe/master] Implement codename detection without lsb_release
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 23ae055fb90a332799302a483e571406b9169d99 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 12:55:42 2015 +0200 Implement codename detection without lsb_release --- scripts/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index e0f455b..3bfc253 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -76,6 +76,8 @@ if [ -z "$lsb_dist" ] && [ -r /etc/lsb-release ]; then fi if [ -z "$lsb_dist" ] && [ -r /etc/debian_version ]; then lsb_dist='Debian' + distro_version="$(cat /etc/debian_version)" + distro_codename="$(. /etc/os-release && echo "$VERSION" | cut -d '(' -f2 | cut -d ')' -f1)" fi if [ -z "$lsb_dist" ] && [ -r /etc/fedora-release ]; then lsb_dist='Fedora'

1 0

[ooni-probe/master] Add support for installing pluggable transports
by art＠torproject.org 20 Nov '15

20 Nov '15

commit d0b200391219caafe39747a686796ec8b5892762 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 12:24:01 2015 +0200 Add support for installing pluggable transports --- scripts/install.sh | 65 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 9a4928f..e0f455b 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -14,6 +14,7 @@ set -e MIN_DEBIAN_VERSION=8 MIN_UBUNTU_VERSION=11 TOR_DEB_REPO="http://deb.torproject.org/torproject.org" +INSTALL_PT="yes" url='https://get.ooni.io/' @@ -83,6 +84,54 @@ if [ -z "$lsb_dist" ] && [ -r /etc/redhat-release ]; then lsb_dist='Fedora' fi +install_obfs4proxy() { + + if ! command_exists go; then + case "$lsb_dist" in + Fedora) + ( + set -x + $sh_c "yum -y install golang" + ) + ;; + Ubuntu|Debian) + ( + set -x + $sh_c "apt-get install -y -q golang" + ) + ;; + esac + fi + + if command_exists go; then + ( + set -x + export GOPATH=$(mktmp -d) + go get git.torproject.org/pluggable-transports/obfs4.git/obfs4proxy + $sh_c "cp $GOPATH/bin/obfs4proxy /usr/local/bin/obfs4proxy" + $sh_c "chmod +x /usr/local/bin/obfs4proxy" + rm -rf $GOPATH + ) + else + echo >&2 + echo >&2 ' We failed to install go. obfs4proxy will not be installed.' + echo >&2 ' Please follow the instructions on this page to install it manually:' + echo >&2 + echo >&2 ' https://github.com/Yawning/obfs4' + echo >&2 + fi +} + +install_pluggable_transports() { + if [ -z "$INSTALL_PT" ];then + ( + set -x + $sh_c 'pip install obfsproxy fteproxy' + ) + install_obfs4proxy + fi +} + case "$lsb_dist" in Fedora) ( @@ -92,6 +141,7 @@ case "$lsb_dist" in $sh_c 'pip install ooniprobe' ) + install_pluggable_transports non_root_usage exit 0 ;; @@ -133,19 +183,8 @@ case "$lsb_dist" in $sh_c 'pip install ooniprobe' ) fi - - # if [ "$lsb_dist" == 'Ubuntu' ];then - # ( - # set -x - # $sh_c 'apt-get install -y -q python-virtualenv' - # ) - # elif [ "$lsb_dist" == 'Debian' ];then - # ( - # set -x - # $sh_c 'apt-get install -y -q virtualenvwrapper' - # ) - # fi - + + install_pluggable_transports non_root_usage exit 0 ;;

1 0

[ooni-probe/master] Add cloudfronted domain for the tor debian repository
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 9f04a266db99de1518c574ff18080e863255aae3 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 15:33:00 2015 +0200 Add cloudfronted domain for the tor debian repository --- scripts/install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 3bfc253..f55947d 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -64,6 +64,10 @@ elif command_exists busybox && busybox --list-modules | grep -q wget; then curl='busybox wget -qO-' fi +if ! (curl $TOR_DEB_REPO | grep "Apache Server at deb.torproject.org");then + TOR_DEB_REPO="http://d3skbh62gb3f3v.cloudfront.net/torproject.org" +fi + # perform some very rudimentary platform detection lsb_dist='' if command_exists lsb_release; then

1 0

[ooni-probe/master] It seems like censor from country X blocks also when the URL contains "torproject.org"
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 847ef37168e6f2621f78a6df0f99280db1430113 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 16:13:51 2015 +0200 It seems like censor from country X blocks also when the URL contains "torproject.org" --- scripts/install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index cf86e54..b1a2f9d 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -67,7 +67,7 @@ fi if ! (curl $TOR_DEB_REPO | grep "Apache Server at deb.torproject.org");then echo ' The Tor Debian repository deb.torproject.org appears to be blocked.' echo ' Failing over to using the cloudfronted mirror.' - TOR_DEB_REPO="http://d3skbh62gb3f3v.cloudfront.net/torproject.org" + TOR_DEB_REPO="https://d3skbh62gb3f3v.cloudfront.net/torproject.org" fi # perform some very rudimentary platform detection @@ -164,6 +164,13 @@ case "$lsb_dist" in did_apt_get_update=1 fi } + + if [ "$TOR_DEBIAN_REPOSITORY" == "https://*" ];then + ( + set -x + $sh_c 'apt-get install -y apt-transport-https' + ) + fi ( set -x

1 0

[ooni-probe/master] Improve pip installation
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 9e093b2c6200627907e7035f4521c622819d2743 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 17:36:57 2015 +0200 Improve pip installation * Fix some shell bugs --- scripts/install.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 3c9fa82..46a6afa 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -131,7 +131,7 @@ install_obfs4proxy() { } install_pluggable_transports() { - if [ -z "$INSTALL_PT" ];then + if [ "$INSTALL_PT" = "yes" ];then ( set -x $sh_c 'pip install obfsproxy fteproxy' @@ -140,13 +140,18 @@ install_pluggable_transports() { fi } +install_pip() { + $curl https://bootstrap.pypa.io/get-pip.py > /tmp/get-pip.py + $sh_c "python /tmp/get-pip.py > /dev/null 2>&1" +} + case "$lsb_dist" in Fedora) ( set -x $sh_c 'yum -y groupinstall "Development tools"' $sh_c 'yum -y install zlib-devel bzip2-devel openssl-devel sqlite-devel libpcap-devel libffi-devel libevent-devel libgeoip-devel tor' - $sh_c "$curl https://bootstrap.pypa.io/get-pip.py | python" + install_pip $sh_c 'pip install ooniprobe' ) @@ -182,13 +187,15 @@ case "$lsb_dist" in $sh_c 'apt-get update' ) - if [ "$lsb_dist" == 'Debian' ] && + install_pip + + if [ "$lsb_dist" = 'Debian' ] && [ "$(echo $distro_version | cut -d '.' -f1 )" -gt $MIN_DEBIAN_VERSION ]; then ( set -x $sh_c 'apt-get install -y -q ooniprobe' ) - elif [ "$lsb_dist" == 'Ubuntu' ] && + elif [ "$lsb_dist" = 'Ubuntu' ] && [ "$(echo $distro_version | cut -d '.' -f1 )" -gt $MIN_UBUNTU_VERSION ]; then ( set -x @@ -198,7 +205,6 @@ case "$lsb_dist" in ( set -x $sh_c 'apt-get install -y -q curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev libssl-dev libffi-dev libdumbnet-dev' - $sh_c "$curl https://bootstrap.pypa.io/get-pip.py | python" $sh_c 'pip install ooniprobe' ) fi

1 0