[tor-browser-bundle/master] Bug 11556: Fix broken bundling due to HTTPS-E 3.5. - tbb-commits

25 Apr 2014

commit ffe165972b6290df94d969675b9f2660cf2eedbb
Author: Georg Koppen gk@torproject.org
Date:   Fri Apr 25 09:03:02 2014 +0000
Bug 11556: Fix broken bundling due to HTTPS-E 3.5.
Starting with version 3.5 HTTPS-Everywhere uses Python 2.7 for
    assembling its rules in a SQLite database. This caused our bundling
    step to be broken on all platforms: On Linux (Ubuntu 10.04) there is
    no Python 2.7 available via the package management and the Windows and
    OS X bundling descriptors missed some necessary packages.
---
 gitian/descriptors/linux/gitian-bundle.yml   |   10 ++++++++++
 gitian/descriptors/linux/gitian-utils.yml    |   24 +++++++++++++++++++++---
 gitian/descriptors/mac/gitian-bundle.yml     |    3 +++
 gitian/descriptors/windows/gitian-bundle.yml |    5 ++++-
 gitian/fetch-inputs.sh                       |    6 ++++--
 gitian/gpg/LXML.gpg                          |  Bin 0 -> 1450 bytes
 gitian/mkbundle-linux.sh                     |    9 +++++++--
 gitian/versions                              |    6 ++++--
 gitian/versions.alpha                        |    6 ++++--
 gitian/versions.beta                         |    6 ++++--
 gitian/versions.nightly                      |    6 ++++--
 11 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml
index 39a7747..c16589a 100644
--- a/gitian/descriptors/linux/gitian-bundle.yml
+++ b/gitian/descriptors/linux/gitian-bundle.yml
@@ -13,6 +13,8 @@ packages:
 - "python"
 - "rnv"
 - "xz-utils"
+- "libxslt1.1"
+- "sqlite3"
 reference_datetime: "2000-01-01 00:00:00"
 remotes:
 - "url": "https://git.torproject.org/tor-launcher.git"
@@ -29,6 +31,10 @@ files:
 - "tor-linux64-gbuilt.zip"
 - "pluggable-transports-linux32-gbuilt.zip"
 - "pluggable-transports-linux64-gbuilt.zip"
+- "python-linux32-utils.zip"
+- "python-linux64-utils.zip"
+- "lxml-linux32-utils.zip"
+- "lxml-linux64-utils.zip"
 - "torrc-defaults-appendix-linux"
 - "bridge_prefs.js"
 - "relativelink-src.zip"
@@ -57,6 +63,10 @@ script: |
   mkdir -p tor-browser/Data/Browser/profile.default/extensions/https-everywhere@eff.org
   mkdir -p tor-browser/Data/Browser/Caches
   mkdir -p tor-browser/Docs/sources/
+  # Preparing Python for HTTPS-Everywhere.
+  unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip
+  export PATH=$INSTDIR/python/bin:$PATH
+  unzip -d $INSTDIR/python/lib/python2.7 lxml-linux$GBUILD_BITS-utils.zip
   #
   cd tor-launcher
   make clean
diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml
index c4052f7..427e69b 100644
--- a/gitian/descriptors/linux/gitian-utils.yml
+++ b/gitian/descriptors/linux/gitian-utils.yml
@@ -15,6 +15,11 @@ packages:
 - "faketime"
 - "libtool"
 - "hardening-wrapper"
+# These packages are needed for Python due to HTTPS-Everywhere >= 3.5.
+- "libsqlite3-dev"
+- "zlib1g-dev"
+- "libxml2-dev"
+- "libxslt-dev"
 reference_datetime: "2000-01-01 00:00:00"
 remotes:
 - "url": "https://github.com/libevent/libevent.git"
@@ -24,6 +29,7 @@ files:
 - "dzip.sh"
 - "openssl.tar.gz"
 - "python.tar.bz2"
+- "lxml.tar.gz"
 script: |
   INSTDIR="$HOME/install"
   source versions
@@ -66,15 +72,27 @@ script: |
   cd ..
# Building Python
-  # Fx 24 ESR does not work with Python < 2.7 anymore. But 10.04 does only ship
-  # with Python 2.6. Thus, we compile 2.7 ourselves...
+  # Fx 24 ESR and HTTPS Everywhere >= 3.5 do not work with Python < 2.7 anymore.
+  # But 10.04 does only ship with Python 2.6. Thus, we compile 2.7 ourselves...
   tar xjf python.tar.bz2
   cd Python-*
   ./configure
   make $MAKEOPTS altinstall prefix=$INSTDIR/python exec-prefix=$INSTDIR/python
   cd ..
-  # Grabbing the results
+  # Building the Python lxml module
+  tar xzf lxml.tar.gz
+  cd lxml-*
+  # Make sure we use our freshly built python binary here. Otherwise bad things
+  # may happen when we do so in the bundle step assembling the HTTPS-Everywhere
+  # rules.
+  $INSTDIR/python/bin/python2.7 setup.py build
+  cd build/lib*
+  ~/build/dzip.sh lxml-$LXML_VER-linux$GBUILD_BITS-utils.zip lxml
+  cp *-utils.zip $OUTDIR
+  cd ../../../
+
+  # Grabbing the remaining results
   cd $INSTDIR
   ~/build/dzip.sh openssl-$OPENSSL_VER-linux$GBUILD_BITS-utils.zip openssl
   # TODO: It is not guaranteed we have a X.X.XX-style version.
diff --git a/gitian/descriptors/mac/gitian-bundle.yml b/gitian/descriptors/mac/gitian-bundle.yml
index b7ecf3a..263ca57 100644
--- a/gitian/descriptors/mac/gitian-bundle.yml
+++ b/gitian/descriptors/mac/gitian-bundle.yml
@@ -15,6 +15,9 @@ packages:
 - "cmake"
 - "zlib1g-dev"
 - "libbz2-dev"
+# Both packages are needed for assembling the HTTPS-Everywhere rules since 3.5.
+- "python-lxml"
+- "sqlite3"
 reference_datetime: "2000-01-01 00:00:00"
 remotes:
 - "url": "https://git.torproject.org/tor-launcher.git"
diff --git a/gitian/descriptors/windows/gitian-bundle.yml b/gitian/descriptors/windows/gitian-bundle.yml
index fddb611..457fa7b 100644
--- a/gitian/descriptors/windows/gitian-bundle.yml
+++ b/gitian/descriptors/windows/gitian-bundle.yml
@@ -8,11 +8,14 @@ packages:
 - "git-core"
 - "unzip"
 - "zip"
-# This is needed for compiling the StartTorBrowserBundle.exe.
+# This package is needed for compiling the StartTorBrowserBundle.exe.
 - "mingw-w64"
 - "nsis"
 - "faketime"
 - "python"
+# Both packages are needed for assembling the HTTPS-Everywhere rules since 3.5.
+- "python-lxml"
+- "sqlite3"
 reference_datetime: "2000-01-01 00:00:00"
 remotes:
 - "url": "https://github.com/moba/tbb-windows-installer.git"
diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh
index 6b8ae99..f80348e 100755
--- a/gitian/fetch-inputs.sh
+++ b/gitian/fetch-inputs.sh
@@ -133,11 +133,12 @@ do
   fi
 done
-for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP
+for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP LXML
 do
   PACKAGE="${i}_PACKAGE"
   URL="${i}_URL"
-  if [ "${i}" == "PYTHON" -o "${i}" == "PYCRYPTO" -o "${i}" == "M2CRYPTO" -o "${i}" == "PYTHON_MSI" ]; then
+  if [ "${i}" == "PYTHON" -o "${i}" == "PYCRYPTO" -o "${i}" == "M2CRYPTO" -o \
+       "${i}" == "PYTHON_MSI" -o "${i}" == "LXML" ]; then
     SUFFIX="asc"
   else
     SUFFIX="sig"
@@ -275,6 +276,7 @@ ln -sf "$M2CRYPTO_PACKAGE" m2crypto.tar.gz
 ln -sf "$PY2EXE_PACKAGE" py2exe.exe
 ln -sf "$SETUPTOOLS_PACKAGE" setuptools.tar.gz
 ln -sf "$GMP_PACKAGE" gmp.tar.bz2
+ln -sf "$LXML_PACKAGE" lxml.tar.gz
# Fetch latest gitian-builder itself
 # XXX - this is broken if a non-standard inputs dir is selected using the command line flag.
diff --git a/gitian/gpg/LXML.gpg b/gitian/gpg/LXML.gpg
new file mode 100644
index 0000000..fd5f415
Binary files /dev/null and b/gitian/gpg/LXML.gpg differ
diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh
index b21d9b4..1328a9c 100755
--- a/gitian/mkbundle-linux.sh
+++ b/gitian/mkbundle-linux.sh
@@ -93,8 +93,9 @@ if [ ! -f inputs/openssl-$OPENSSL_VER-linux32-utils.zip -o \
      ! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux32-utils.zip -o \
      ! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip -o \
      ! -f inputs/python-$PYTHON_VER-linux32-utils.zip -o \
-     ! -f inputs/python-$PYTHON_VER-linux64-utils.zip ];
-
+     ! -f inputs/python-$PYTHON_VER-linux64-utils.zip -o \
+     ! -f inputs/lxml-$LXML_VER-linux32-utils.zip -o \
+     ! -f inputs/lxml-$LXML_VER-linux64-utils.zip ];
 then
   echo
   echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******"
@@ -116,6 +117,8 @@ then
   ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip
   ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip
   ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip
+  ln -sf lxml-$LXML_VER-linux32-utils.zip lxml-linux32-utils.zip
+  ln -sf lxml-$LXML_VER-linux64-utils.zip lxml-linux64-utils.zip
   cd ..
   #cp -a result/tor-linux-res.yml $GITIAN_DIR/inputs/
 else
@@ -132,6 +135,8 @@ else
   ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip
   ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip
   ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip
+  ln -sf lxml-$LXML_VER-linux32-utils.zip lxml-linux32-utils.zip
+  ln -sf lxml-$LXML_VER-linux64-utils.zip lxml-linux64-utils.zip
   cd ..
 fi
diff --git a/gitian/versions b/gitian/versions
index 97c6d33..a9c372c 100755
--- a/gitian/versions
+++ b/gitian/versions
@@ -9,8 +9,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1
 TOR_TAG=tor-0.2.4.21
 TORLAUNCHER_TAG=0.2.4.4
 TORBUTTON_TAG=1.6.7.0
-# XXX: 3.5 requires python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=3.5
 NSIS_TAG=v0.1
 ZLIB_TAG=v1.2.8
 LIBEVENT_TAG=release-2.0.21-stable
@@ -37,6 +36,7 @@ TWISTED_VER=13.1.0
 M2CRYPTO_VER=0.21.1
 PY2EXE_VER=0.6.9
 SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
 OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -59,6 +59,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
 PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
 SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
 OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -98,3 +99,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/$%7BTWISTED_PACKAGE%7D
 M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/$%7BM2CRYPTO_PACKAGE%7D
 PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/$%7BPY2EXE_VER...
 SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/$%7BSETUPTOOLS_PACKAGE%...
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/$%7BLXML_PACKAGE%7D
diff --git a/gitian/versions.alpha b/gitian/versions.alpha
index ddfa06b..9c5fa02 100755
--- a/gitian/versions.alpha
+++ b/gitian/versions.alpha
@@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build2
 TOR_TAG=tor-0.2.5.3-alpha
 TORLAUNCHER_TAG=0.2.5.3
 TORBUTTON_TAG=1.6.8.1
-# XXX: 3.5 requires python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=3.5
 NSIS_TAG=v0.1
 ZLIB_TAG=v1.2.8
 LIBEVENT_TAG=release-2.0.21-stable
@@ -38,6 +37,7 @@ TWISTED_VER=13.1.0
 M2CRYPTO_VER=0.21.1
 PY2EXE_VER=0.6.9
 SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
 OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -60,6 +60,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
 PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
 SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
 OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -99,3 +100,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/$%7BTWISTED_PACKAGE%7D
 M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/$%7BM2CRYPTO_PACKAGE%7D
 PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/$%7BPY2EXE_VER...
 SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/$%7BSETUPTOOLS_PACKAGE%...
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/$%7BLXML_PACKAGE%7D
diff --git a/gitian/versions.beta b/gitian/versions.beta
index 7933efd..f6c24a2 100755
--- a/gitian/versions.beta
+++ b/gitian/versions.beta
@@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1
 TOR_TAG=tor-0.2.4.21
 TORLAUNCHER_TAG=0.2.5.3
 TORBUTTON_TAG=1.6.8.1
-# XXX: 3.5 requires python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=3.5
 NSIS_TAG=v0.1
 ZLIB_TAG=v1.2.8
 LIBEVENT_TAG=release-2.0.21-stable
@@ -38,6 +37,7 @@ TWISTED_VER=13.1.0
 M2CRYPTO_VER=0.21.1
 PY2EXE_VER=0.6.9
 SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
 OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -60,6 +60,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
 PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
 SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
 OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -99,3 +100,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/$%7BTWISTED_PACKAGE%7D
 M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/$%7BM2CRYPTO_PACKAGE%7D
 PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/$%7BPY2EXE_VER...
 SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/$%7BSETUPTOOLS_PACKAGE%...
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/$%7BLXML_PACKAGE%7D
diff --git a/gitian/versions.nightly b/gitian/versions.nightly
index 1596918..c8a8d7c 100755
--- a/gitian/versions.nightly
+++ b/gitian/versions.nightly
@@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1
 TOR_TAG=master
 TORLAUNCHER_TAG=master
 TORBUTTON_TAG=master
-# XXX: Master and 3.5 require python2.7, see #11556
-HTTPSE_TAG=3.4.5
+HTTPSE_TAG=master
 NSIS_TAG=master
 ZLIB_TAG=v1.2.8
 LIBEVENT_TAG=release-2.0.21-stable
@@ -39,6 +38,7 @@ TWISTED_VER=13.1.0
 M2CRYPTO_VER=0.21.1
 PY2EXE_VER=0.6.9
 SETUPTOOLS_VER=1.4
+LXML_VER=3.3.5
## File names for the source packages
 OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz
@@ -61,6 +61,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2
 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz
 PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe
 SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz
+LXML_PACKAGE=lxml-${LXML_VER}.tar.gz
# Hashes for packages with weak sigs or no sigs
 OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028
@@ -100,3 +101,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/$%7BTWISTED_PACKAGE%7D
 M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/$%7BM2CRYPTO_PACKAGE%7D
 PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/$%7BPY2EXE_VER...
 SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/$%7BSETUPTOOLS_PACKAGE%...
+LXML_URL=https://pypi.python.org/packages/source/l/lxml/$%7BLXML_PACKAGE%7D

    