November 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge pull request #400 from anadahz/fix/feature/install
by art＠torproject.org 20 Nov '15

20 Nov '15

commit ccc751be92d8b5bb24888434fa60c4de5c246262 Merge: 0ed75b1 2354162 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jul 13 10:45:30 2015 +0200 Merge pull request #400 from anadahz/fix/feature/install Preserve evnironment in su and sudo commands scripts/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Fix wheezy backports uRL
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 85e990cbf78589ac160916b4230d1108f668d455 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jul 13 11:33:31 2015 +0200 Fix wheezy backports uRL --- scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 08cab8c..8ef1c12 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -182,7 +182,7 @@ install_meek() { setup_backports() { - echo "deb http://ftp.de.debian.org/ ${distro_codename}-backports main" > /etc/apt/sources.list.d/stable.list + echo "deb http://ftp.de.debian.org/debian/ ${distro_codename}-backports main" > /etc/apt/sources.list.d/stable.list $sh_c "gpg --keyserver pgpkeys.mit.edu --recv-key A1BD8E9D78F7FE5C3E65D8AF8B48AD6246925553" $sh_c "gpg -a --export A1BD8E9D78F7FE5C3E65D8AF8B48AD6246925553 | apt-key add -" $sh_c "apt-get update"

1 0

[ooni-probe/master] Set PYTHONPATH before running pip
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 01ab8d01caa5231428b807624a8d2199abfeabce Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jul 13 11:22:17 2015 +0200 Set PYTHONPATH before running pip --- scripts/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index baa334a..c6db265 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -238,7 +238,7 @@ install_pluggable_transports() { install_pluggable_transport_deps ( set -x - $sh_c 'pip install obfsproxy fteproxy' + PYTHONPATH=$PYTHONPATH $sh_c 'pip install obfsproxy fteproxy' ) install_obfs4proxy install_meek @@ -258,7 +258,7 @@ case "$lsb_dist" in $sh_c 'yum -y groupinstall "Development tools"' $sh_c 'yum -y install zlib-devel bzip2-devel openssl-devel sqlite-devel libpcap-devel libffi-devel libevent-devel libgeoip-devel tor' install_pip - $sh_c 'pip install ooniprobe' + PYTHONPATH=$PYTHONPATH $sh_c 'pip install ooniprobe' ) install_pluggable_transports @@ -311,7 +311,7 @@ case "$lsb_dist" in ( set -x $sh_c 'apt-get install -y -q curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev libssl-dev libffi-dev libdumbnet-dev' - $sh_c 'pip install ooniprobe' + PYTHONPATH=$PYTHONPATH $sh_c 'pip install ooniprobe' ) fi

1 0

[ooni-probe/master] Set PYTHONPATH path from calling python
by art＠torproject.org 20 Nov '15

20 Nov '15

commit ed26fa9674ed2898afa35110f515c9fb8e0f6381 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jul 13 14:24:36 2015 +0200 Set PYTHONPATH path from calling python --- scripts/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install.sh b/scripts/install.sh index 8ef1c12..89be923 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -11,6 +11,7 @@ set -e TOR_DEB_REPO="http://deb.torproject.org/torproject.org" CLOUDFRONT="no" INSTALL_PT="yes" +PYTHONPATH=$(python -c "import sys; print ':'.join(x for x in sys.path if x)") # These are the minimum ubuntu and debian version required to use the debian # package.

1 0

[ooni-probe/master] Increase the timeout of director tests or travis will be sad
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 88cc41e9f2b1ee7a2f428c4a4ab039edd9436a1d Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jul 27 16:52:50 2015 +0200 Increase the timeout of director tests or travis will be sad --- ooni/tests/test_oonicli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 2ca94e3..9e73061 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -59,6 +59,8 @@ tor: class TestRunDirector(ConfigTestCase): + timeout = 220 + def setUp(self): super(TestRunDirector, self).setUp() if not is_internet_connected():

1 0

[ooni-probe/master] Add support for DNF package manager, add required build dependencies
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 0c636ba35227db9832682ce5de107c545bdbe354 Author: anadahz <kojgelo(a)inbox.com> Date: Fri Jul 31 01:30:40 2015 +0200 Add support for DNF package manager, add required build dependencies - Fedora 22 introduces the next upcoming major version of Yum DNF, this commit adds support for dnf package manager which is the default for Fedora 22 - Add required package dependencies: GeoIP-devel, python-devel, libdnet-devel, gcc-c++ --- scripts/install.sh | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 89be923..016c072 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -108,6 +108,14 @@ elif command_exists mktemp; then mktmp='mktemp' fi +# Fedora 22 introduces the next upcoming major version of Yum DNF +yum='' +if command_exists yum; then + yum='yum' +elif command_exists dnf; then + yum='dnf' +fi + if [ $CLOUDFRONT = "yes" ];then echo ' Using the cloudfronted tor mirror.' TOR_DEB_REPO="https://d3skbh62gb3f3v.cloudfront.net/torproject.org" @@ -194,8 +202,8 @@ install_go() { case "$lsb_dist" in Fedora) ( - set -x - $sh_c "yum -y install golang" + set -x + $sh_c "${yum} -y install golang" ) ;; Ubuntu|Debian) @@ -203,13 +211,13 @@ install_go() { [ "$(echo $distro_version | cut -d '.' -f1 )" -lt $MIN_DEBIAN_VERSION ]; then setup_backports ( - set -x - $sh_c "apt-get install -y -t ${distro_codename}-backports golang" + set -x + $sh_c "apt-get install -y -t ${distro_codename}-backports golang" ) else ( - set -x - $sh_c "apt-get install -y -q golang" + set -x + $sh_c "apt-get install -y -q golang" ) fi ;; @@ -223,14 +231,14 @@ install_pluggable_transport_deps() { case "$lsb_dist" in Fedora) ( - set -x - $sh_c "yum -y install gmp-devel" + set -x + $sh_c "${yum} -y install gmp-devel" ) ;; Ubuntu|Debian) ( - set -x - $sh_c "apt-get install -y -q libgmp-dev" + set -x + $sh_c "apt-get install -y -q libgmp-dev" ) ;; esac @@ -257,15 +265,15 @@ install_pip() { case "$lsb_dist" in Fedora) ( - set -x - $sh_c 'yum -y groupinstall "Development tools"' - $sh_c 'yum -y install zlib-devel bzip2-devel openssl-devel sqlite-devel libpcap-devel libffi-devel libevent-devel libgeoip-devel tor' - install_pip - PYTHONPATH=$PYTHONPATH $sh_c 'pip install ooniprobe' + set -x + $sh_c "${yum} -y groupinstall \"Development tools\"" + $sh_c "${yum} -y install zlib-devel bzip2-devel openssl-devel sqlite-devel libpcap-devel libffi-devel libevent-devel GeoIP-devel tor python-devel libdnet-devel gcc-c++" + install_pip + PYTHONPATH=$PYTHONPATH $sh_c 'pip install ooniprobe' ) - install_pluggable_transports - non_root_usage + install_pluggable_transports + non_root_usage exit 0 ;; @@ -291,8 +299,8 @@ case "$lsb_dist" in ( set -x - $sh_c 'apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-keys A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89' - $sh_c "echo deb $TOR_DEB_REPO $distro_codename main > /etc/apt/sources.list.d/tor.list" + $sh_c 'apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-keys A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89' + $sh_c "echo deb $TOR_DEB_REPO $distro_codename main > /etc/apt/sources.list.d/tor.list" $sh_c 'apt-get update' )

1 0

[ooni-probe/master] Use the latest version of txsocksx that includes a fix for 15.0
by art＠torproject.org 20 Nov '15

20 Nov '15

commit ce4bf8688ad56eb7e2811ad0f5ba0cb536c6dc00 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jul 27 16:52:33 2015 +0200 Use the latest version of txsocksx that includes a fix for 15.0 --- ooni/reporter.py | 2 +- ooni/utils/hacks.py | 27 --------------------------- ooni/utils/trueheaders.py | 5 +++-- requirements.txt | 2 +- 4 files changed, 5 insertions(+), 31 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 36a0e88..574cd1b 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -287,7 +287,7 @@ class OONIBReporter(OReporter): # do this with some deferred kung foo or instantiate the reporter after # tor is started. - from ooni.utils.hacks import SOCKS5Agent + from txsocksx.http import SOCKS5Agent from twisted.internet import reactor if self.collectorAddress.startswith('httpo://'): diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index e001d3f..7ec3d0f 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -58,30 +58,3 @@ def patched_reduce_ex(self, proto): return copy_reg._reconstructor, args, dict else: return copy_reg._reconstructor, args - - -class SOCKS5Agent(SOCKS5AgentOriginal): - """ - This is a quick hack to fix: - https://github.com/habnabit/txsocksx/issues/9 - """ - def _getEndpoint(self, scheme_or_uri, host=None, port=None): - if host is not None: - scheme = scheme_or_uri - else: - scheme = scheme_or_uri.scheme - host = scheme_or_uri.host - port = scheme_or_uri.port - if scheme not in ('http', 'https'): - raise SchemeNotSupported('unsupported scheme', scheme) - endpoint = self.endpointFactory( - host, port, self.proxyEndpoint, **self.endpointArgs) - if scheme == 'https': - if hasattr(self, '_wrapContextFactory'): - tlsPolicy = self._wrapContextFactory(host, port) - elif hasattr(self, '_policyForHTTPS'): - tlsPolicy = self._policyForHTTPS.creatorForNetloc(host, port) - else: - raise NotImplementedError("can't figure out how to make a context factory") - endpoint = self._tlsWrapper(tlsPolicy, endpoint) - return endpoint diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index f015b62..3905949 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -8,11 +8,12 @@ import itertools from copy import copy from twisted.web import client, _newclient, http_headers -from twisted.web._newclient import RequestNotSent, RequestGenerationFailed, TransportProxyProducer, STATUS +from twisted.web._newclient import RequestNotSent, RequestGenerationFailed +from twisted.web._newclient import TransportProxyProducer, STATUS from twisted.internet import reactor from twisted.internet.defer import Deferred, fail, maybeDeferred, failure -from ooni.utils.hacks import SOCKS5Agent +from txsocksx.http import SOCKS5Agent from txsocksx.client import SOCKS5ClientFactory SOCKS5ClientFactory.noisy = False diff --git a/requirements.txt b/requirements.txt index 50e82f2..8b8d5ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML>=3.10 -Twisted>=12.2.0,<=14.0.0 +Twisted>=12.2.0 ipaddr>=2.1.10 pyOpenSSL>=0.13 geoip

1 0

[ooni-probe/master] Add a connect timeout of 20 seconds
by art＠torproject.org 20 Nov '15

20 Nov '15

commit c4f016bfd5a03e43e65caa2e9941f4d775225e29 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 16:05:46 2015 +0200 Add a connect timeout of 20 seconds --- scripts/install.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index f55947d..cf86e54 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -57,14 +57,16 @@ fi curl='' if command_exists curl; then - curl='curl -sSL' + curl='curl --connect-timeout 20 -sSL' elif command_exists wget; then - curl='wget -qO-' + curl='wget --connect-timeout 20 -qO-' elif command_exists busybox && busybox --list-modules | grep -q wget; then - curl='busybox wget -qO-' + curl='busybox wget --connect-timeout 20 -qO-' fi if ! (curl $TOR_DEB_REPO | grep "Apache Server at deb.torproject.org");then + echo ' The Tor Debian repository deb.torproject.org appears to be blocked.' + echo ' Failing over to using the cloudfronted mirror.' TOR_DEB_REPO="http://d3skbh62gb3f3v.cloudfront.net/torproject.org" fi

1 0

[ooni-probe/master] Cosmetic fix
by art＠torproject.org 20 Nov '15

20 Nov '15

commit b1cf6c440c2b2c6f438a79511ce059426b4eabc3 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jul 27 16:53:04 2015 +0200 Cosmetic fix --- ooni/utils/trueheaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index 3905949..ac73970 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -68,7 +68,7 @@ class TrueHeaders(http_headers.Headers): except KeyError: pass - for k, v in itertools.chain(headers_a.getAllRawHeaders(), \ + for k, v in itertools.chain(headers_a.getAllRawHeaders(), headers_b.getAllRawHeaders()): field_names.append(k)

1 0

[ooni-probe/master] Add install.sh script heavily based on docker
by art＠torproject.org 20 Nov '15

20 Nov '15

commit 2af2c9ffd983ea22271322b17319332dd3be7d10 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jul 3 12:03:20 2015 +0200 Add install.sh script heavily based on docker --- scripts/install.sh | 166 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 0000000..9a4928f --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,166 @@ +#!/bin/sh +set -e +# +# This script is meant for quick & easy install via: +# 'curl -sSL https://get.ooni.io/ | sh' +# or: +# 'wget -qO- https://get.ooni.io/ | sh' +# +# It is heavily based upon the get.docker.io script + +# These are the minimum ubuntu and debian version required to use the debian +# package. +# Currently you can only set the major version number. +MIN_DEBIAN_VERSION=8 +MIN_UBUNTU_VERSION=11 +TOR_DEB_REPO="http://deb.torproject.org/torproject.org" + +url='https://get.ooni.io/' + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +non_root_usage() { + your_user=your-user + [ "$user" != 'root' ] && your_user="$user" + echo + echo 'If you would like to run all ooniprobe tests as a non-root user, you should' + echo 'look at using the ooniprobe non root wrapper:' + echo + echo ' https://github.com/TheTorProject/ooni-probe/blob/master/bin/Makefile' + echo +} + +if command_exists ooniprobe; then + echo >&2 'Warning: "ooniprobe" command appears to already exist.' + echo >&2 'Please ensure that you do not already have ooniprobe installed.' + echo >&2 'You may press Ctrl+C now to abort this process and rectify this situation.' + ( set -x; sleep 20 ) +fi + +user="$(id -un 2>/dev/null || true)" + +sh_c='sh -c' +if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo sh -c' + elif command_exists su; then + sh_c='su -c' + else + echo >&2 'Error: this installer needs the ability to run commands as root.' + echo >&2 'We are unable to find either "sudo" or "su" available to make this happen.' + exit 1 + fi +fi + +curl='' +if command_exists curl; then + curl='curl -sSL' +elif command_exists wget; then + curl='wget -qO-' +elif command_exists busybox && busybox --list-modules | grep -q wget; then + curl='busybox wget -qO-' +fi + +# perform some very rudimentary platform detection +lsb_dist='' +if command_exists lsb_release; then + lsb_dist="$(lsb_release -si)" + distro_version="$(lsb_release -rs)" + distro_codename="$(lsb_release -cs)" +fi +if [ -z "$lsb_dist" ] && [ -r /etc/lsb-release ]; then + lsb_dist="$(. /etc/lsb-release && echo "$DISTRIB_ID")" +fi +if [ -z "$lsb_dist" ] && [ -r /etc/debian_version ]; then + lsb_dist='Debian' +fi +if [ -z "$lsb_dist" ] && [ -r /etc/fedora-release ]; then + lsb_dist='Fedora' +fi +if [ -z "$lsb_dist" ] && [ -r /etc/redhat-release ]; then + lsb_dist='Fedora' +fi + +case "$lsb_dist" in + Fedora) + ( + set -x + $sh_c 'yum -y groupinstall "Development tools"' + $sh_c 'yum -y install zlib-devel bzip2-devel openssl-devel sqlite-devel libpcap-devel libffi-devel libevent-devel libgeoip-devel tor python-pip' + $sh_c 'pip install ooniprobe' + ) + + non_root_usage + exit 0 + ;; + + Ubuntu|Debian) + export DEBIAN_FRONTEND=noninteractive + + did_apt_get_update= + apt_get_update() { + if [ -z "$did_apt_get_update" ]; then + ( set -x; $sh_c 'sleep 3; apt-get update' ) + did_apt_get_update=1 + fi + } + + ( + set -x + $sh_c 'apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-keys A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89' + $sh_c "echo deb $TOR_DEB_REPO $distro_codename main > /etc/apt/sources.list.d/tor.list" + $sh_c 'apt-get update' + ) + + if [ "$lsb_dist" == 'Debian' ] && + [ "$(echo $distro_version | cut -d '.' -f1 )" -gt $MIN_DEBIAN_VERSION ]; then + ( + set -x + $sh_c 'apt-get install -y -q ooniprobe' + ) + elif [ "$lsb_dist" == 'Ubuntu' ] && + [ "$(echo $distro_version | cut -d '.' -f1 )" -gt $MIN_UBUNTU_VERSION ]; then + ( + set -x + $sh_c 'apt-get install -y -q ooniprobe' + ) + else + ( + set -x + $sh_c 'apt-get install -y -q curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev libssl-dev libffi-dev libdumbnet-dev' + $sh_c 'pip install ooniprobe' + ) + fi + + # if [ "$lsb_dist" == 'Ubuntu' ];then + # ( + # set -x + # $sh_c 'apt-get install -y -q python-virtualenv' + # ) + # elif [ "$lsb_dist" == 'Debian' ];then + # ( + # set -x + # $sh_c 'apt-get install -y -q virtualenvwrapper' + # ) + # fi + + non_root_usage + exit 0 + ;; + + # Gentoo) + # exit 0 + # ;; +esac + +echo >&2 +echo >&2 ' Either your platform is not easily detectable, is not supported by this' +echo >&2 ' installer script (yet - PRs welcome!), or does not yet have a package for' +echo >&2 ' ooniprobe. Please visit the following URL for more detailed installation' +echo >&2 ' instructions:' +echo >&2 +echo >&2 ' https://ooni.torproject.org/docs/' +echo >&2 +exit 1

1 0