commit 2d9e5eea316c3d35a46f55c1150c4873bbbf2d15 Author: Georg Koppen gk@torproject.org Date: Thu Apr 24 21:11:56 2014 +0000
Bug 10120: Part 1: Refactor Linux descriptors.
We build the tools needed for the Linux descriptors separately and preserve them. This does not (yet) include the tools in the Pluggable Transports descriptor which will happen in an other commit.
Additonally, we enhance our clean targets in the Makefile containing not only a target for the newly created utils component now but for the other components as well. --- gitian/Makefile | 23 ++++- gitian/descriptors/linux/gitian-firefox.yml | 29 ++---- gitian/descriptors/linux/gitian-tor.yml | 36 ++----- gitian/descriptors/linux/gitian-utils.yml | 84 +++++++++++++++ gitian/mkbundle-linux.sh | 148 ++++++++++++++++++--------- 5 files changed, 223 insertions(+), 97 deletions(-)
diff --git a/gitian/Makefile b/gitian/Makefile index cec338d..b24ec53 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -72,12 +72,29 @@ prep-beta: ./check-prerequisites.sh $(TORSOCKS) ./fetch-inputs.sh ../../gitian-builder/inputs/ versions.beta
-clean: - rm -f ../../gitian-builder/inputs/*gbuilt* +clean: clean-utils clean-tor clean-browser clean-pt clean-bundle rm -f ../../gitian-builder/inputs/*.yml rm -f ../../gitian-builder/inputs/bundle.inputs rm -f ../../gitian-builder/inputs/versions* - rm -f ../../gitian-builder/inputs/*debug.zip + +clean-utils: + rm -f ../../gitian-builder/inputs/*utils.zip + +clean-tor: + rm -f ../../gitian-builder/inputs/tor-linux* + rm -f ../../gitian-builder/inputs/tor-mac* + rm -f ../../gitian-builder/inputs/tor-win* + +clean-browser: + rm -f ../../gitian-builder/inputs/tor-browser-linux* + rm -f ../../gitian-builder/inputs/tor-browser-mac* + rm -f ../../gitian-builder/inputs/tor-browser-win* + +clean-pt: + rm -f ../../gitian-builder/inputs/pluggable-transports* + +clean-bundle: + rm -f ../../gitian-builder/inputs/bundle-*
vmclean: rm -rf ../../gitian-builder/*.qcow2 diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml index 1f2cf12..7e6c598 100644 --- a/gitian/descriptors/linux/gitian-firefox.yml +++ b/gitian/descriptors/linux/gitian-firefox.yml @@ -29,9 +29,10 @@ remotes: - "url": "https://git.torproject.org/tor-browser.git" "dir": "tor-browser" files: +- "python-linux32-utils.zip" +- "python-linux64-utils.zip" - "re-dzip.sh" - "dzip.sh" -- "python.tar.bz2" - "versions" script: | source versions @@ -55,25 +56,17 @@ script: | mkdir -p $INSTDIR/build/bin/ ln -s /usr/bin/yasm-1 $INSTDIR/build/bin/yasm export PATH=$PATH:$INSTDIR/build/bin + # Preparing Python for Firefox + unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip + # TODO: We might want to have a smarter solution than hard-coding the version. + ln -sf $INSTDIR/python/bin/python2.7 $INSTDIR/python/bin/python + export PATH=$INSTDIR/python/bin:$PATH # mkdir -p $INSTDIR/Browser/ mkdir -p $INSTDIR/Debug/Browser/components # - # Fx 24 ESR does not work with Python < 2.7 anymore. But 10.04 does only ship - # with Python 2.6. Thus, we compile 2.7 ourselves... - if [ ${TORBROWSER_VERSION::3} != "3.0" ]; then - mkdir python - cd python - tar -xjvf ../python.tar.bz2 - Python-*/configure - make - sudo make install - cd .. - fi - # cd tor-browser # .git and the src takes up a lot of useless space, and we need the space to build - #git archive HEAD | tar -x -C $OUTDIR/src rm -rf .git find -type f | xargs touch --date="$REFERENCE_DATETIME" rm -f configure @@ -94,19 +87,17 @@ script: | # Strip and generate debuginfo for the firefox binary that we keep, all *.so # files, and the plugin-container (see ticket #10126) cd $INSTDIR - for LIB in Browser/*.so Browser/webapprt-stub Browser/mozilla-xremote-client Browser/firefox Browser/plugin-container Browser/components/*.so # Browser/updater + for LIB in Browser/*.so Browser/webapprt-stub Browser/mozilla-xremote-client Browser/firefox Browser/plugin-container Browser/components/*.so # Browser/updater do # Build-ID is sometimes non-deterministic, and we use debuglink anyway - objcopy --remove-section=.note.gnu.build-id $LIB + objcopy --remove-section=.note.gnu.build-id $LIB objcopy --only-keep-debug $LIB Debug/$LIB strip $LIB objcopy --add-gnu-debuglink=./Debug/$LIB $LIB done ~/build/re-dzip.sh Browser/omni.ja ~/build/re-dzip.sh Browser/webapprt/omni.ja - if [ -f Browser/browser/omni.ja ]; then # FF24 includes a third .ja - ~/build/re-dzip.sh Browser/browser/omni.ja - fi + ~/build/re-dzip.sh Browser/browser/omni.ja ~/build/dzip.sh tor-browser-linux$GBUILD_BITS-gbuilt.zip Browser/ ~/build/dzip.sh tor-browser-linux$GBUILD_BITS-debug.zip Debug/ cp tor-browser-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/linux/gitian-tor.yml b/gitian/descriptors/linux/gitian-tor.yml index 28b7d13..60a5015 100644 --- a/gitian/descriptors/linux/gitian-tor.yml +++ b/gitian/descriptors/linux/gitian-tor.yml @@ -5,7 +5,7 @@ suites: architectures: - "i386" - "amd64" -packages: +packages: - "git-core" - "unzip" - "zip" @@ -20,8 +20,6 @@ reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://git.torproject.org/tor.git" "dir": "tor" -- "url": "https://github.com/libevent/libevent.git" - "dir": "libevent" files: - "versions" - "bug10297.patch" @@ -32,11 +30,13 @@ files: - "bug11069.patch" - "bug9665.patch" - "dzip.sh" -- "openssl.tar.gz" +- "openssl-linux32-utils.zip" +- "openssl-linux64-utils.zip" +- "libevent-linux32-utils.zip" +- "libevent-linux64-utils.zip" script: | INSTDIR="$HOME/install" source versions - export LIBRARY_PATH="$INSTDIR/lib" export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME export TZ=UTC @@ -53,31 +53,11 @@ script: | mkdir -p $INSTDIR/Data/Tor/ mkdir -p $INSTDIR/Tor/ mkdir -p $INSTDIR/Debug/Tor/ - # - cd libevent - ./autogen.sh - find -type f | xargs touch --date="$REFERENCE_DATETIME" - ./configure --disable-static --prefix=$INSTDIR/libevent - make $MAKEOPTS - make install - cp $INSTDIR/libevent/lib/libevent-2.0.so.5 $INSTDIR/Tor/ - cd .. - # - tar xzf openssl.tar.gz - cd openssl-* - find -type f | xargs touch --date="$REFERENCE_DATETIME" - #./Configure -shared --prefix=$INSTDIR/openssl linux-elf - if [ $GBUILD_BITS == "64" ]; - then - ./config -shared --prefix=$INSTDIR/openssl enable-ec_nistp_64_gcc_128 - else - ./config -shared --prefix=$INSTDIR/openssl - fi - make - make install + unzip -d $INSTDIR openssl-linux$GBUILD_BITS-utils.zip + unzip -d $INSTDIR libevent-linux$GBUILD_BITS-utils.zip cp $INSTDIR/openssl/lib/libssl.so.1.0.0 $INSTDIR/Tor/ cp $INSTDIR/openssl/lib/libcrypto.so.1.0.0 $INSTDIR/Tor/ - cd .. + cp $INSTDIR/libevent/lib/libevent-2.0.so.5 $INSTDIR/Tor/ # chmod 700 $INSTDIR/Tor/*so* # diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml new file mode 100644 index 0000000..c4052f7 --- /dev/null +++ b/gitian/descriptors/linux/gitian-utils.yml @@ -0,0 +1,84 @@ +--- +name: "utils-linux" +suites: +- "lucid" +architectures: +- "i386" +- "amd64" +packages: +- "git-core" +- "unzip" +- "zip" +- "autoconf" +- "autoconf2.13" +- "automake" +- "faketime" +- "libtool" +- "hardening-wrapper" +reference_datetime: "2000-01-01 00:00:00" +remotes: +- "url": "https://github.com/libevent/libevent.git" + "dir": "libevent" +files: +- "versions" +- "dzip.sh" +- "openssl.tar.gz" +- "python.tar.bz2" +script: | + INSTDIR="$HOME/install" + source versions + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME=$REFERENCE_DATETIME + export TZ=UTC + export LC_ALL=C + umask 0022 + # + # Config options for hardening-wrapper + export DEB_BUILD_HARDENING=1 + export DEB_BUILD_HARDENING_STACKPROTECTOR=1 + export DEB_BUILD_HARDENING_FORTIFY=1 + export DEB_BUILD_HARDENING_FORMAT=1 + export DEB_BUILD_HARDENING_PIE=1 + + # Building Libevent + cd libevent + ./autogen.sh + find -type f | xargs touch --date="$REFERENCE_DATETIME" + ./configure --disable-static --prefix=$INSTDIR/libevent + make $MAKEOPTS + make install + cd .. + + # Building OpenSSL + tar xzf openssl.tar.gz + cd openssl-* + find -type f | xargs touch --date="$REFERENCE_DATETIME" + if [ $GBUILD_BITS == "64" ]; + then + ./config -shared --prefix=$INSTDIR/openssl enable-ec_nistp_64_gcc_128 + else + ./config -shared --prefix=$INSTDIR/openssl + fi + # Using $MAKEOPTS breaks the build. Might be the issue mentioned on + # http://cblfs.cross-lfs.org/index.php/OpenSSL. + make + make install + cd .. + + # Building Python + # Fx 24 ESR does not work with Python < 2.7 anymore. But 10.04 does only ship + # with Python 2.6. Thus, we compile 2.7 ourselves... + tar xjf python.tar.bz2 + cd Python-* + ./configure + make $MAKEOPTS altinstall prefix=$INSTDIR/python exec-prefix=$INSTDIR/python + cd .. + + # Grabbing the results + cd $INSTDIR + ~/build/dzip.sh openssl-$OPENSSL_VER-linux$GBUILD_BITS-utils.zip openssl + # TODO: It is not guaranteed we have a X.X.XX-style version. + ~/build/dzip.sh libevent-${LIBEVENT_TAG:8:6}-linux$GBUILD_BITS-utils.zip libevent + ~/build/dzip.sh python-$PYTHON_VER-linux$GBUILD_BITS-utils.zip python + + cp $INSTDIR/*-utils.zip $OUTDIR/ diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index b4e635d..b21d9b4 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -42,7 +42,7 @@ fi cd $GITIAN_DIR export PATH=$PATH:$PWD/libexec
-echo "pref("torbrowser.version", "$TORBROWSER_VERSION-Linux");" > $GITIAN_DIR/inputs/torbrowser.version +echo "pref("torbrowser.version", "$TORBROWSER_VERSION-Linux");" > $GITIAN_DIR/inputs/torbrowser.version echo "$TORBROWSER_VERSION" > $GITIAN_DIR/inputs/bare-version cp -a $WRAPPER_DIR/$VERSIONS_FILE $GITIAN_DIR/inputs/versions
@@ -51,7 +51,7 @@ cp $WRAPPER_DIR/patches/* $GITIAN_DIR/inputs/
cd $WRAPPER_DIR/.. rm -f $GITIAN_DIR/inputs/relativelink-src.zip -$WRAPPER_DIR/build-helpers/dzip.sh $GITIAN_DIR/inputs/relativelink-src.zip ./RelativeLink/ +$WRAPPER_DIR/build-helpers/dzip.sh $GITIAN_DIR/inputs/relativelink-src.zip ./RelativeLink/
cd ./Bundle-Data/ rm -f $GITIAN_DIR/inputs/tbb-docs.zip @@ -65,6 +65,11 @@ $WRAPPER_DIR/build-helpers/dzip.sh $GITIAN_DIR/inputs/linux-skeleton.zip .
cd $WRAPPER_DIR
+# Let's preserve the original $FOO for creating proper symlinks after building +# the utils both if we verify tags and if we don't. + +LIBEVENT_TAG_ORIG=$LIBEVENT_TAG + if [ "z$VERIFY_TAGS" = "z1" ]; then ./verify-tags.sh $GITIAN_DIR/inputs $VERSIONS_FILE || exit 1 @@ -83,35 +88,83 @@ fi
cd $GITIAN_DIR
-if [ ! -f $GITIAN_DIR/inputs/tor-linux32-gbuilt.zip -o ! -f $GITIAN_DIR/inputs/tor-linux64-gbuilt.zip ]; +if [ ! -f inputs/openssl-$OPENSSL_VER-linux32-utils.zip -o \ + ! -f inputs/openssl-$OPENSSL_VER-linux64-utils.zip -o \ + ! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux32-utils.zip -o \ + ! -f inputs/libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip -o \ + ! -f inputs/python-$PYTHON_VER-linux32-utils.zip -o \ + ! -f inputs/python-$PYTHON_VER-linux64-utils.zip ]; + then - echo - echo "****** Starting Tor Component of Linux Bundle (1/4 for Linux) ******" - echo + echo + echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******" + echo
- ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit zlib=$ZLIB_TAG,libevent=$LIBEVENT_TAG,tor=$TOR_TAG $DESCRIPTOR_DIR/linux/gitian-tor.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml if [ $? -ne 0 ]; then #mv var/build.log ./tor-fail-linux.log.`date +%Y%m%d%H%M%S` exit 1 fi - - cp -a build/out/tor-linux*-gbuilt.zip $GITIAN_DIR/inputs/ - cp -a build/out/tor-linux*-debug.zip $GITIAN_DIR/inputs/ + + cd inputs + cp -a ../build/out/*-utils.zip . + ln -sf openssl-$OPENSSL_VER-linux32-utils.zip openssl-linux32-utils.zip + ln -sf openssl-$OPENSSL_VER-linux64-utils.zip openssl-linux64-utils.zip + # TODO: It is not guaranteed we have a X.X.XX-style version. + ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux32-utils.zip libevent-linux32-utils.zip + ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip + ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip + ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip + cd .. #cp -a result/tor-linux-res.yml $GITIAN_DIR/inputs/ else - echo - echo "****** SKIPPING already built Tor Component of Linux Bundle (1/4 for Linux) ******" - echo + echo + echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/5 for Linux) ******" + echo + # We might have built the utilities in the past but maybe the links are + # pointing to the wrong version. Refresh them. + cd inputs + ln -sf openssl-$OPENSSL_VER-linux32-utils.zip openssl-linux32-utils.zip + ln -sf openssl-$OPENSSL_VER-linux64-utils.zip openssl-linux64-utils.zip + # TODO: It is not guaranteed we have a X.X.XX-style version. + ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux32-utils.zip libevent-linux32-utils.zip + ln -sf libevent-${LIBEVENT_TAG_ORIG:8:6}-linux64-utils.zip libevent-linux64-utils.zip + ln -sf python-$PYTHON_VER-linux32-utils.zip python-linux32-utils.zip + ln -sf python-$PYTHON_VER-linux64-utils.zip python-linux64-utils.zip + cd .. +fi + +if [ ! -f inputs/tor-linux32-gbuilt.zip -o \ + ! -f inputs/tor-linux64-gbuilt.zip ]; +then + echo + echo "****** Starting Tor Component of Linux Bundle (2/5 for Linux) ******" + echo + + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor=$TOR_TAG $DESCRIPTOR_DIR/linux/gitian-tor.yml + if [ $? -ne 0 ]; + then + #mv var/build.log ./tor-fail-linux.log.`date +%Y%m%d%H%M%S` + exit 1 + fi
+ cp -a build/out/tor-linux*-gbuilt.zip inputs/ + cp -a build/out/tor-linux*-debug.zip inputs/ + #cp -a result/tor-linux-res.yml $GITIAN_DIR/inputs/ +else + echo + echo "****** SKIPPING already built Tor Component of Linux Bundle (2/5 for Linux) ******" + echo fi
-if [ ! -f $GITIAN_DIR/inputs/tor-browser-linux32-gbuilt.zip -o ! -f $GITIAN_DIR/inputs/tor-browser-linux64-gbuilt.zip ]; +if [ ! -f inputs/tor-browser-linux32-gbuilt.zip -o \ + ! -f inputs/tor-browser-linux64-gbuilt.zip ]; then - echo - echo "****** Starting TorBrowser Component of Linux Bundle (2/4 for Linux) ******" - echo + echo + echo "****** Starting TorBrowser Component of Linux Bundle (3/5 for Linux) ******" + echo
./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml if [ $? -ne 0 ]; @@ -120,20 +173,21 @@ then exit 1 fi
- cp -a build/out/tor-browser-linux*-gbuilt.zip $GITIAN_DIR/inputs/ - cp -a build/out/tor-browser-linux*-debug.zip $GITIAN_DIR/inputs/ + cp -a build/out/tor-browser-linux*-gbuilt.zip inputs/ + cp -a build/out/tor-browser-linux*-debug.zip inputs/ #cp -a result/torbrowser-linux-res.yml $GITIAN_DIR/inputs/ else - echo - echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (2/4 for Linux) ******" - echo + echo + echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/5 for Linux) ******" + echo fi
-if [ ! -f $GITIAN_DIR/inputs/pluggable-transports-linux32-gbuilt.zip -o ! -f $GITIAN_DIR/inputs/pluggable-transports-linux64-gbuilt.zip ]; +if [ ! -f inputs/pluggable-transports-linux32-gbuilt.zip -o \ + ! -f inputs/pluggable-transports-linux64-gbuilt.zip ]; then - echo - echo "****** Starting Pluggable Transports Component of Linux Bundle (3/4 for Linux) ******" - echo + echo + echo "****** Starting Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******" + echo
./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG,fteproxy=$FTEPROXY_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml if [ $? -ne 0 ]; @@ -141,42 +195,42 @@ then #mv var/build.log ./pluggable-transports-fail-linux.log.`date +%Y%m%d%H%M%S` exit 1 fi - - cp -a build/out/pluggable-transports-linux*-gbuilt.zip $GITIAN_DIR/inputs/ - cp -a build/out/pluggable-transports-linux*-debug.zip $GITIAN_DIR/inputs/ + + cp -a build/out/pluggable-transports-linux*-gbuilt.zip inputs/ + cp -a build/out/pluggable-transports-linux*-debug.zip inputs/ #cp -a result/pluggable-transports-linux-res.yml $GITIAN_DIR/inputs/ else - echo - echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (3/4 for Linux) ******" - echo + echo + echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******" + echo fi
-if [ ! -f $GITIAN_DIR/inputs/bundle-linux.gbuilt ]; -then - echo - echo "****** Starting Bundling+Localization of Linux Bundle (4/4 for Linux) ******" - echo - +if [ ! -f inputs/bundle-linux.gbuilt ]; +then + echo + echo "****** Starting Bundling+Localization of Linux Bundle (5/5 for Linux) ******" + echo + cd $WRAPPER_DIR && ./record-inputs.sh $VERSIONS_FILE && cd $GITIAN_DIR - + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit https-everywhere=$HTTPSE_TAG,tor-launcher=$TORLAUNCHER_TAG,torbutton=$TORBUTTON_TAG $DESCRIPTOR_DIR/linux/gitian-bundle.yml if [ $? -ne 0 ]; then #mv var/build.log ./bundle-fail-linux.log.`date +%Y%m%d%H%M%S` exit 1 fi - + mkdir -p $WRAPPER_DIR/$TORBROWSER_VERSION/ cp -a build/out/tor-browser-linux*xz* $WRAPPER_DIR/$TORBROWSER_VERSION/ || exit 1 - cp -a $GITIAN_DIR/inputs/*debug.zip $WRAPPER_DIR/$TORBROWSER_VERSION/ || exit 1 - touch $GITIAN_DIR/inputs/bundle-linux.gbuilt + cp -a inputs/*debug.zip $WRAPPER_DIR/$TORBROWSER_VERSION/ || exit 1 + touch inputs/bundle-linux.gbuilt else - echo - echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (4/4 for Linux) ******" - echo -fi + echo + echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (5/5 for Linux) ******" + echo +fi
-echo +echo echo "****** Linux Bundle complete ******" echo