[tor-commits] [tor-browser-bundle/master] Bug 11556: Fix broken bundling due to HTTPS-E 3.5.
gk at torproject.org
gk at torproject.org
Fri Apr 25 07:55:50 UTC 2014
commit ffe165972b6290df94d969675b9f2660cf2eedbb
Author: Georg Koppen <gk at torproject.org>
Date: Fri Apr 25 09:03:02 2014 +0000
Bug 11556: Fix broken bundling due to HTTPS-E 3.5.
Starting with version 3.5 HTTPS-Everywhere uses Python 2.7 for
assembling its rules in a SQLite database. This caused our bundling
step to be broken on all platforms: On Linux (Ubuntu 10.04) there is
no Python 2.7 available via the package management and the Windows and
OS X bundling descriptors missed some necessary packages.
---
gitian/descriptors/linux/gitian-bundle.yml | 10 ++++++++++
gitian/descriptors/linux/gitian-utils.yml | 24 +++++++++++++++++++++---
gitian/descriptors/mac/gitian-bundle.yml | 3 +++
gitian/descriptors/windows/gitian-bundle.yml | 5 ++++-
gitian/fetch-inputs.sh | 6 ++++--
gitian/gpg/LXML.gpg | Bin 0 -> 1450 bytes
gitian/mkbundle-linux.sh | 9 +++++++--
gitian/versions | 6 ++++--
gitian/versions.alpha | 6 ++++--
gitian/versions.beta | 6 ++++--
gitian/versions.nightly | 6 ++++--
11 files changed, 65 insertions(+), 16 deletions(-)
diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml
index 39a7747..c16589a 100644
--- a/gitian/descriptors/linux/gitian-bundle.yml
+++ b/gitian/descriptors/linux/gitian-bundle.yml
@@ -13,6 +13,8 @@ packages:
- "python"
- "rnv"
- "xz-utils"
+- "libxslt1.1"
+- "sqlite3"
reference_datetime: "2000-01-01 00:00:00"
remotes:
- "url": "https://git.torproject.org/tor-launcher.git"
@@ -29,6 +31,10 @@ files:
- "tor-linux64-gbuilt.zip"
- "pluggable-transports-linux32-gbuilt.zip"
- "pluggable-transports-linux64-gbuilt.zip"
+- "python-linux32-utils.zip"
+- "python-linux64-utils.zip"
+- "lxml-linux32-utils.zip"
+- "lxml-linux64-utils.zip"
- "torrc-defaults-appendix-linux"
- "bridge_prefs.js"
- "relativelink-src.zip"
@@ -57,6 +63,10 @@ script: |
mkdir -p tor-browser/Data/Browser/profile.default/extensions/https-everywhere at eff.org
mkdir -p tor-browser/Data/Browser/Caches
mkdir -p tor-browser/Docs/sources/
+ # Preparing Python for HTTPS-Everywhere.
+ unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip
+ export PATH=$INSTDIR/python/bin:$PATH
+ unzip -d $INSTDIR/python/lib/python2.7 lxml-linux$GBUILD_BITS-utils.zip
#
cd tor-launcher
make clean
diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml
index c4052f7..427e69b 100644
--- a/gitian/descriptors/linux/gitian-utils.yml
+++ b/gitian/descriptors/linux/gitian-utils.yml
@@ -15,6 +15,11 @@ packages:
- "faketime"
- "libtool"
- "hardening-wrapper"
+# These packages are needed for Python due to HTTPS-Everywhere >= 3.5.
+- "libsqlite3-dev"
+- "zlib1g-dev"
+- "libxml2-dev"
+- "libxslt-dev"
reference_datetime: "2000-01-01 00:00:00"
remotes:
- "url": "https://github.com/libevent/libevent.git"
@@ -24,6 +29,7 @@ files:
- "dzip.sh"
- "openssl.tar.gz"
- "python.tar.bz2"
+- "lxml.tar.gz"
script: |
INSTDIR="$HOME/install"
source versions
@@ -66,15 +72,27 @@ script: |
cd ..
# Building Python
- # Fx 24 ESR does not work with Python < 2.7 anymore. But 10.04 does only ship
- # with Python 2.6. Thus, we compile 2.7 ourselves...
+ # Fx 24 ESR and HTTPS Everywhere >= 3.5 do not work with Python < 2.7 anymore.
+ # But 10.04 does only ship with Python 2.6. Thus, we compile 2.7 ourselves...
tar xjf python.tar.bz2
cd Python-*
./configure
make $MAKEOPTS altinstall prefix=$INSTDIR/python exec-prefix=$INSTDIR/python
cd ..
- # Grabbing the results
+ # Building the Python lxml module
+ tar xzf lxml.tar.gz
+ cd lxml-*
+ # Make sure we use our freshly built python binary here. Otherwise bad things
+ # may happen when we do so in the bundle step assembling the HTTPS-Everywhere
+ # rules.
+ $INSTDIR/python/bin/python2.7 setup.py build
+ cd build/lib*
+ ~/build/dzip.sh lxml-$LXML_VER-linux$GBUILD_BITS-utils.zip lxml
+ cp *-utils.zip $OUTDIR
+ cd ../../../
+
+ # Grabbing the remaining results
cd $INSTDIR
~/build/dzip.sh openssl-$OPENSSL_VER-linux$GBUILD_BITS-utils.zip openssl
# TODO: It is not guaranteed we have a X.X.XX-style version.
diff --git a/gitian/descriptors/mac/gitian-bundle.yml b/gitian/descriptors/mac/gitian-bundle.yml
index b7ecf3a..263ca57 100644
--- a/gitian/descriptors/mac/gitian-bundle.yml
+++ b/gitian/descriptors/mac/gitian-bundle.yml
@@ -15,6 +15,9 @@ packages:
- "cmake"
- "zlib1g-dev"
- "libbz2-dev"
+# Both packages are needed for assembling the HTTPS-Everywhere rules since 3.5.
+- "python-lxml"
+- "sqlite3"
reference_datetime: "2000-01-01 00:00:00"
remotes:
- "url": "https://git.torproject.org/tor-launcher.git"
diff --git a/gitian/descriptors/windows/gitian-bundle.yml b/gitian/descriptors/windows/gitian-bundle.yml
index fddb611..457fa7b 100644
--- a/gitian/descriptors/windows/gitian-bundle.yml
+++ b/gitian/descriptors/windows/gitian-bundle.yml
@@ -8,11 +8,14 @@ packages:
- "git-core"
- "unzip"
- "zip"
-# This is needed for compiling the StartTorBrowserBundle.exe.
+# This package is needed for compiling the StartTorBrowserBundle.exe.
- "mingw-w64"
- "nsis"
- "faketime"
- "python"
+# Both packages are needed for assembling the HTTPS-Everywhere rules since 3.5.
+- "python-lxml"
+- "sqlite3"
reference_datetime: "2000-01-01 00:00:00"
remotes:
- "url": "https://github.com/moba/tbb-windows-installer.git"
diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh
index 6b8ae99..f80348e 100755
--- a/gitian/fetch-inputs.sh
+++ b/gitian/fetch-inputs.sh
@@ -133,11 +133,12 @@ do
fi
done
-for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP
+for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP LXML
do
PACKAGE="${i}_PACKAGE"
URL="${i}_URL"
- if [ "${i}" == "PYTHON" -o "${i}" == "PYCRYPTO" -o "${i}" == "M2CRYPTO" -o "${i}" == "PYTHON_MSI" ]; then
+ if [ "${i}" == "PYTHON" -o "${i}" == "PYCRYPTO" -o "${i}" == "M2CRYPTO" -o \
+ "${i}" == "PYTHON_MSI" -o "${i}" == "LXML" ]; then
SUFFIX="asc"
else
SUFFIX="sig"
@@ -275,6 +276,7 @@ ln -sf "$M2CRYPTO_PACKAGE" m2crypto.tar.gz
ln -sf "$PY2EXE_PACKAGE" py2exe.exe
ln -sf "$SETUPTOOLS_PACKAGE" setuptools.tar.gz
ln -sf "$GMP_PACKAGE" gmp.tar.bz2
+ln -sf "$LXML_PACKAGE" lxml.tar.gz
# Fetch latest gitian-builder itself
# XXX - this is broken if a non-standard inputs dir is selected using the command line flag.
diff --git a/gitian/gpg/LXML.gpg b/gitian/gpg/LXML.gpg
new file mode 100644
index 0000000..fd5f415
Binary files /dev/null and b/gitian/gpg/LXML.gpg differ
diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh
index b21d9b4..1328a9c 100755
--- a/gitian/mkbundle-linux.sh
+++ b/gitian/mkbundle-linux.sh
@@ -93,8 +93,9 @@ if [ ! -f inputs/openssl-$OPENSSL_VER-linux32-utils.zip -o \
! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux32-utils.zip -o \
! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip -o \
! -f inputs/python-$PYTHON_VER-linux32-utils.zip -o \
- ! -f inputs/python-$PYTHON_VER-linux64-utils.zip ];
-
+ ! -f inputs/python-$PYTHON_VER-linux64-utils.zip -o \
+ ! -f inputs/lxml-$LXML_VER-linux32-utils.zip -o \
+ ! -f inputs/lxml-$LXML_VER-linux64-utils.zip ];
then
echo
echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******"
@@ -116,6 +117,8 @@ then
ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip
ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip
ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip
+ ln -sf lxml-$LXML_VER-linux32-utils.zip lxml-linux32-utils.zip
+ ln -sf lxml-$LXML_VER-linux64-utils.zip lxml-linux64-utils.zip
cd ..
#cp -a result/tor-linux-res.yml $GITIAN_DIR/inputs/
else
@@ -132,6 +135,8 @@ else
ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip
ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip
ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip
+ ln -sf lxml-$LXML_VER-linux32-utils.zip lxml-linux32-utils.zip
+ ln -sf lxml-$LXML_VER-linux64-utils.zip lxml-linux64-utils.zip
cd ..
fi
diff --git a/gitian/versions b/gitian/versions
index 97c6d33..a9c372c 100755
--- a/gitian/versions
+++ b/gitian/versions
@@ -9,8 +9,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1
TOR_TAG=tor-0.2.4.21
TORLAUNCHER_TAG=0.2.4.4
TORBUTTON_TAG=1.6.7.0
-# XXX: 3.5 requires python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=3.5
NSIS_TAG=v0.1
ZLIB_TAG=v1.2.8
LIBEVENT_TAG=release-2.0.21-stable
@@ -37,6 +36,7 @@ TWISTED_VER=13.1.0
M2CRYPTO_VER=0.21.1
PY2EXE_VER=0.6.9
SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -59,6 +59,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -98,3 +99,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE}
M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE}
PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE}
SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE}
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE}
diff --git a/gitian/versions.alpha b/gitian/versions.alpha
index ddfa06b..9c5fa02 100755
--- a/gitian/versions.alpha
+++ b/gitian/versions.alpha
@@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build2
TOR_TAG=tor-0.2.5.3-alpha
TORLAUNCHER_TAG=0.2.5.3
TORBUTTON_TAG=1.6.8.1
-# XXX: 3.5 requires python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=3.5
NSIS_TAG=v0.1
ZLIB_TAG=v1.2.8
LIBEVENT_TAG=release-2.0.21-stable
@@ -38,6 +37,7 @@ TWISTED_VER=13.1.0
M2CRYPTO_VER=0.21.1
PY2EXE_VER=0.6.9
SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -60,6 +60,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -99,3 +100,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE}
M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE}
PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE}
SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE}
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE}
diff --git a/gitian/versions.beta b/gitian/versions.beta
index 7933efd..f6c24a2 100755
--- a/gitian/versions.beta
+++ b/gitian/versions.beta
@@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1
TOR_TAG=tor-0.2.4.21
TORLAUNCHER_TAG=0.2.5.3
TORBUTTON_TAG=1.6.8.1
-# XXX: 3.5 requires python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=3.5
NSIS_TAG=v0.1
ZLIB_TAG=v1.2.8
LIBEVENT_TAG=release-2.0.21-stable
@@ -38,6 +37,7 @@ TWISTED_VER=13.1.0
M2CRYPTO_VER=0.21.1
PY2EXE_VER=0.6.9
SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -60,6 +60,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -99,3 +100,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE}
M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE}
PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE}
SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE}
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE}
diff --git a/gitian/versions.nightly b/gitian/versions.nightly
index 1596918..c8a8d7c 100755
--- a/gitian/versions.nightly
+++ b/gitian/versions.nightly
@@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1
TOR_TAG=master
TORLAUNCHER_TAG=master
TORBUTTON_TAG=master
-# XXX: Master and 3.5 require python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=master
NSIS_TAG=master
ZLIB_TAG=v1.2.8
LIBEVENT_TAG=release-2.0.21-stable
@@ -39,6 +38,7 @@ TWISTED_VER=13.1.0
M2CRYPTO_VER=0.21.1
PY2EXE_VER=0.6.9
SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -61,6 +61,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -100,3 +101,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE}
M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE}
PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE}
SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE}
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE}
More information about the tor-commits
mailing list