commit ffe165972b6290df94d969675b9f2660cf2eedbb Author: Georg Koppen <gk@torproject.org> Date: Fri Apr 25 09:03:02 2014 +0000 Bug 11556: Fix broken bundling due to HTTPS-E 3.5. Starting with version 3.5 HTTPS-Everywhere uses Python 2.7 for assembling its rules in a SQLite database. This caused our bundling step to be broken on all platforms: On Linux (Ubuntu 10.04) there is no Python 2.7 available via the package management and the Windows and OS X bundling descriptors missed some necessary packages. --- gitian/descriptors/linux/gitian-bundle.yml | 10 ++++++++++ gitian/descriptors/linux/gitian-utils.yml | 24 +++++++++++++++++++++--- gitian/descriptors/mac/gitian-bundle.yml | 3 +++ gitian/descriptors/windows/gitian-bundle.yml | 5 ++++- gitian/fetch-inputs.sh | 6 ++++-- gitian/gpg/LXML.gpg | Bin 0 -> 1450 bytes gitian/mkbundle-linux.sh | 9 +++++++-- gitian/versions | 6 ++++-- gitian/versions.alpha | 6 ++++-- gitian/versions.beta | 6 ++++-- gitian/versions.nightly | 6 ++++-- 11 files changed, 65 insertions(+), 16 deletions(-) diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml index 39a7747..c16589a 100644 --- a/gitian/descriptors/linux/gitian-bundle.yml +++ b/gitian/descriptors/linux/gitian-bundle.yml @@ -13,6 +13,8 @@ packages: - "python" - "rnv" - "xz-utils" +- "libxslt1.1" +- "sqlite3" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://git.torproject.org/tor-launcher.git" @@ -29,6 +31,10 @@ files: - "tor-linux64-gbuilt.zip" - "pluggable-transports-linux32-gbuilt.zip" - "pluggable-transports-linux64-gbuilt.zip" +- "python-linux32-utils.zip" +- "python-linux64-utils.zip" +- "lxml-linux32-utils.zip" +- "lxml-linux64-utils.zip" - "torrc-defaults-appendix-linux" - "bridge_prefs.js" - "relativelink-src.zip" @@ -57,6 +63,10 @@ script: | mkdir -p tor-browser/Data/Browser/profile.default/extensions/https-everywhere@eff.org mkdir -p tor-browser/Data/Browser/Caches mkdir -p tor-browser/Docs/sources/ + # Preparing Python for HTTPS-Everywhere. + unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip + export PATH=$INSTDIR/python/bin:$PATH + unzip -d $INSTDIR/python/lib/python2.7 lxml-linux$GBUILD_BITS-utils.zip # cd tor-launcher make clean diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml index c4052f7..427e69b 100644 --- a/gitian/descriptors/linux/gitian-utils.yml +++ b/gitian/descriptors/linux/gitian-utils.yml @@ -15,6 +15,11 @@ packages: - "faketime" - "libtool" - "hardening-wrapper" +# These packages are needed for Python due to HTTPS-Everywhere >= 3.5. +- "libsqlite3-dev" +- "zlib1g-dev" +- "libxml2-dev" +- "libxslt-dev" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://github.com/libevent/libevent.git" @@ -24,6 +29,7 @@ files: - "dzip.sh" - "openssl.tar.gz" - "python.tar.bz2" +- "lxml.tar.gz" script: | INSTDIR="$HOME/install" source versions @@ -66,15 +72,27 @@ script: | cd .. # Building Python - # Fx 24 ESR does not work with Python < 2.7 anymore. But 10.04 does only ship - # with Python 2.6. Thus, we compile 2.7 ourselves... + # Fx 24 ESR and HTTPS Everywhere >= 3.5 do not work with Python < 2.7 anymore. + # But 10.04 does only ship with Python 2.6. Thus, we compile 2.7 ourselves... tar xjf python.tar.bz2 cd Python-* ./configure make $MAKEOPTS altinstall prefix=$INSTDIR/python exec-prefix=$INSTDIR/python cd .. - # Grabbing the results + # Building the Python lxml module + tar xzf lxml.tar.gz + cd lxml-* + # Make sure we use our freshly built python binary here. Otherwise bad things + # may happen when we do so in the bundle step assembling the HTTPS-Everywhere + # rules. + $INSTDIR/python/bin/python2.7 setup.py build + cd build/lib* + ~/build/dzip.sh lxml-$LXML_VER-linux$GBUILD_BITS-utils.zip lxml + cp *-utils.zip $OUTDIR + cd ../../../ + + # Grabbing the remaining results cd $INSTDIR ~/build/dzip.sh openssl-$OPENSSL_VER-linux$GBUILD_BITS-utils.zip openssl # TODO: It is not guaranteed we have a X.X.XX-style version. diff --git a/gitian/descriptors/mac/gitian-bundle.yml b/gitian/descriptors/mac/gitian-bundle.yml index b7ecf3a..263ca57 100644 --- a/gitian/descriptors/mac/gitian-bundle.yml +++ b/gitian/descriptors/mac/gitian-bundle.yml @@ -15,6 +15,9 @@ packages: - "cmake" - "zlib1g-dev" - "libbz2-dev" +# Both packages are needed for assembling the HTTPS-Everywhere rules since 3.5. +- "python-lxml" +- "sqlite3" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://git.torproject.org/tor-launcher.git" diff --git a/gitian/descriptors/windows/gitian-bundle.yml b/gitian/descriptors/windows/gitian-bundle.yml index fddb611..457fa7b 100644 --- a/gitian/descriptors/windows/gitian-bundle.yml +++ b/gitian/descriptors/windows/gitian-bundle.yml @@ -8,11 +8,14 @@ packages: - "git-core" - "unzip" - "zip" -# This is needed for compiling the StartTorBrowserBundle.exe. +# This package is needed for compiling the StartTorBrowserBundle.exe. - "mingw-w64" - "nsis" - "faketime" - "python" +# Both packages are needed for assembling the HTTPS-Everywhere rules since 3.5. +- "python-lxml" +- "sqlite3" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://github.com/moba/tbb-windows-installer.git" diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index 6b8ae99..f80348e 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -133,11 +133,12 @@ do fi done -for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP +for i in BINUTILS GCC PYTHON PYCRYPTO M2CRYPTO PYTHON_MSI GMP LXML do PACKAGE="${i}_PACKAGE" URL="${i}_URL" - if [ "${i}" == "PYTHON" -o "${i}" == "PYCRYPTO" -o "${i}" == "M2CRYPTO" -o "${i}" == "PYTHON_MSI" ]; then + if [ "${i}" == "PYTHON" -o "${i}" == "PYCRYPTO" -o "${i}" == "M2CRYPTO" -o \ + "${i}" == "PYTHON_MSI" -o "${i}" == "LXML" ]; then SUFFIX="asc" else SUFFIX="sig" @@ -275,6 +276,7 @@ ln -sf "$M2CRYPTO_PACKAGE" m2crypto.tar.gz ln -sf "$PY2EXE_PACKAGE" py2exe.exe ln -sf "$SETUPTOOLS_PACKAGE" setuptools.tar.gz ln -sf "$GMP_PACKAGE" gmp.tar.bz2 +ln -sf "$LXML_PACKAGE" lxml.tar.gz # Fetch latest gitian-builder itself # XXX - this is broken if a non-standard inputs dir is selected using the command line flag. diff --git a/gitian/gpg/LXML.gpg b/gitian/gpg/LXML.gpg new file mode 100644 index 0000000..fd5f415 Binary files /dev/null and b/gitian/gpg/LXML.gpg differ diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index b21d9b4..1328a9c 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -93,8 +93,9 @@ if [ ! -f inputs/openssl-$OPENSSL_VER-linux32-utils.zip -o \ ! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux32-utils.zip -o \ ! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip -o \ ! -f inputs/python-$PYTHON_VER-linux32-utils.zip -o \ - ! -f inputs/python-$PYTHON_VER-linux64-utils.zip ]; - + ! -f inputs/python-$PYTHON_VER-linux64-utils.zip -o \ + ! -f inputs/lxml-$LXML_VER-linux32-utils.zip -o \ + ! -f inputs/lxml-$LXML_VER-linux64-utils.zip ]; then echo echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******" @@ -116,6 +117,8 @@ then ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip + ln -sf lxml-$LXML_VER-linux32-utils.zip lxml-linux32-utils.zip + ln -sf lxml-$LXML_VER-linux64-utils.zip lxml-linux64-utils.zip cd .. #cp -a result/tor-linux-res.yml $GITIAN_DIR/inputs/ else @@ -132,6 +135,8 @@ else ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip + ln -sf lxml-$LXML_VER-linux32-utils.zip lxml-linux32-utils.zip + ln -sf lxml-$LXML_VER-linux64-utils.zip lxml-linux64-utils.zip cd .. fi diff --git a/gitian/versions b/gitian/versions index 97c6d33..a9c372c 100755 --- a/gitian/versions +++ b/gitian/versions @@ -9,8 +9,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1 TOR_TAG=tor-0.2.4.21 TORLAUNCHER_TAG=0.2.4.4 TORBUTTON_TAG=1.6.7.0 -# XXX: 3.5 requires python2.7, see #11556 -HTTPSE_TAG=3.4.5 +HTTPSE_TAG=3.5 NSIS_TAG=v0.1 ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.21-stable @@ -37,6 +36,7 @@ TWISTED_VER=13.1.0 M2CRYPTO_VER=0.21.1 PY2EXE_VER=0.6.9 SETUPTOOLS_VER=1.4 +LXML_VER=3.3.5 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -59,6 +59,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz +LXML_PACKAGE=lxml-${LXML_VER}.tar.gz # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028 @@ -98,3 +99,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE} M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE} PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE} SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE} +LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE} diff --git a/gitian/versions.alpha b/gitian/versions.alpha index ddfa06b..9c5fa02 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build2 TOR_TAG=tor-0.2.5.3-alpha TORLAUNCHER_TAG=0.2.5.3 TORBUTTON_TAG=1.6.8.1 -# XXX: 3.5 requires python2.7, see #11556 -HTTPSE_TAG=3.4.5 +HTTPSE_TAG=3.5 NSIS_TAG=v0.1 ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.21-stable @@ -38,6 +37,7 @@ TWISTED_VER=13.1.0 M2CRYPTO_VER=0.21.1 PY2EXE_VER=0.6.9 SETUPTOOLS_VER=1.4 +LXML_VER=3.3.5 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -60,6 +60,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz +LXML_PACKAGE=lxml-${LXML_VER}.tar.gz # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028 @@ -99,3 +100,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE} M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE} PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE} SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE} +LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE} diff --git a/gitian/versions.beta b/gitian/versions.beta index 7933efd..f6c24a2 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1 TOR_TAG=tor-0.2.4.21 TORLAUNCHER_TAG=0.2.5.3 TORBUTTON_TAG=1.6.8.1 -# XXX: 3.5 requires python2.7, see #11556 -HTTPSE_TAG=3.4.5 +HTTPSE_TAG=3.5 NSIS_TAG=v0.1 ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.21-stable @@ -38,6 +37,7 @@ TWISTED_VER=13.1.0 M2CRYPTO_VER=0.21.1 PY2EXE_VER=0.6.9 SETUPTOOLS_VER=1.4 +LXML_VER=3.3.5 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -60,6 +60,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz +LXML_PACKAGE=lxml-${LXML_VER}.tar.gz # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028 @@ -99,3 +100,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE} M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE} PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE} SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE} +LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE} diff --git a/gitian/versions.nightly b/gitian/versions.nightly index 1596918..c8a8d7c 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -10,8 +10,7 @@ TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1 TOR_TAG=master TORLAUNCHER_TAG=master TORBUTTON_TAG=master -# XXX: Master and 3.5 require python2.7, see #11556 -HTTPSE_TAG=3.4.5 +HTTPSE_TAG=master NSIS_TAG=master ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.21-stable @@ -39,6 +38,7 @@ TWISTED_VER=13.1.0 M2CRYPTO_VER=0.21.1 PY2EXE_VER=0.6.9 SETUPTOOLS_VER=1.4 +LXML_VER=3.3.5 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz @@ -61,6 +61,7 @@ TWISTED_PACKAGE=Twisted-${TWISTED_VER}.tar.bz2 M2CRYPTO_PACKAGE=M2Crypto-${M2CRYPTO_VER}.tar.gz PY2EXE_PACKAGE=py2exe-${PY2EXE_VER}.win32-py2.7.exe SETUPTOOLS_PACKAGE=setuptools-${SETUPTOOLS_VER}.tar.gz +LXML_PACKAGE=lxml-${LXML_VER}.tar.gz # Hashes for packages with weak sigs or no sigs OPENSSL_HASH=53cb818c3b90e507a8348f4f5eaedb05d8bfe5358aabb508b7263cc670c3e028 @@ -100,3 +101,4 @@ TWISTED_URL=https://pypi.python.org/packages/source/T/Twisted/${TWISTED_PACKAGE} M2CRYPTO_URL=https://pypi.python.org/packages/source/M/M2Crypto/${M2CRYPTO_PACKAGE} PY2EXE_URL=http://softlayer-dal.dl.sourceforge.net/project/py2exe/py2exe/${PY2EXE_VER}/${PY2EXE_PACKAGE} SETUPTOOLS_URL=https://pypi.python.org/packages/source/s/setuptools/${SETUPTOOLS_PACKAGE} +LXML_URL=https://pypi.python.org/packages/source/l/lxml/${LXML_PACKAGE}