commit 763784f552779b6dd4e58a76f441a8602b9018ba Author: Georg Koppen <gk@torproject.org> Date: Mon Dec 7 12:33:16 2015 +0000 Bug 15578: Switch Linux descriptors over to Wheezy Besides using Wheezy for building Linux bundles this patch cleans up our usage of libfaketime as well to make it obvious where exactly we still need it. --- gitian/Makefile | 2 +- gitian/README.build | 2 +- gitian/check-prerequisites.sh | 41 ++++++++++---------- gitian/descriptors/linux/gitian-bundle.yml | 6 +-- gitian/descriptors/linux/gitian-firefox.yml | 9 ++--- .../linux/gitian-pluggable-transports.yml | 7 ++-- gitian/descriptors/linux/gitian-tor.yml | 7 ++-- gitian/descriptors/linux/gitian-utils.yml | 38 +++++++++++++----- gitian/make-vms.sh | 29 ++++++++------ 9 files changed, 84 insertions(+), 57 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index 5718d76..36b5ec3 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -119,7 +119,7 @@ clean-bundle: vmclean: rm -rf ../../gitian-builder/*.qcow2 rm -rf ../../gitian-builder/base-* - rm -rf ../../gitian-builder/target-{lucid,precise}* + rm -rf ../../gitian-builder/target-{lucid,wheezy,precise}* distclean: vmclean rm -rf ../../gitian-builder/inputs/* diff --git a/gitian/README.build b/gitian/README.build index f289791..4d01d6a 100644 --- a/gitian/README.build +++ b/gitian/README.build @@ -177,7 +177,7 @@ Known Issues and Quirks: where 'make vmclean' causes the rebuild of two VMs in a row.. This might trigger weird bugs in python-vm-builder.. To rebuild only one set of VMs, use either 'rm ../../gitian-builder/*precise*' (to remove the Windows/Mac - VMs) or 'rm ../../gitian-builder/*lucid*' (to remove the Linux VMs). + VMs) or 'rm ../../gitian-builder/*wheezy*' (to remove the Linux VMs). You probably want to make sure you have no stray qemu processes before rebuilding the VMs or starting a new build, too. 'killall qemu-kvm' is diff --git a/gitian/check-prerequisites.sh b/gitian/check-prerequisites.sh index cc16d0e..a5f8393 100755 --- a/gitian/check-prerequisites.sh +++ b/gitian/check-prerequisites.sh @@ -17,7 +17,7 @@ then VERSION=`cat /etc/issue | grep -Eo '[0-9]{2}' | head -1` if [ "$VERSION" -ge "14" ]; then - dpkg -s ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion torsocks tor 2>/dev/null >/dev/null + dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion torsocks tor 2>/dev/null >/dev/null if [ $? -ne 0 ]; then @@ -25,7 +25,7 @@ then echo echo "Please run:" echo " sudo apt-get install torsocks tor" - echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion" + echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion" exit 1 fi else @@ -45,28 +45,29 @@ then echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync" exit 1 fi - - # python-vm-builder is special as we don't have a Debian package for it. - vmbuilder --help 2>/dev/null >/dev/null - if [ $? -ne 0 ]; - then - echo "The VM tool python-vm-builder is missing." - echo - echo "Please run" - echo 'torsocks wget -U "" http://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12....' - echo 'echo "ec12e0070a007989561bfee5862c89a32c301992dd2771c4d5078ef1b3014f03 vm-builder_0.12.4+bzr489.orig.tar.gz" | sha256sum -c' - echo "# (verification -- must return OK)" - echo "tar -zxvf vm-builder_0.12.4+bzr489.orig.tar.gz" - echo "cd vm-builder-0.12.4+bzr489" - echo "sudo python setup.py install" - echo "cd .." - exit 1 - fi else echo "We need Debian or Ubuntu which seem to be missing. Aborting." exit 1 fi +# vmbuilder is special as we don't have a package for it yet. +# XXX: Make sure an already installed vmbuilder is recent enough. +vmbuilder --help 2>/dev/null >/dev/null +if [ $? -ne 0 ]; +then + echo "The VM tool python-vm-builder is missing." + echo + echo "Please run" + echo 'torsocks wget -U "" https://bugs.launchpad.net/ubuntu/+archive/primary/+files/vm-builder_0.12.4+...' + echo 'echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c' + echo "# (verification -- must return OK)" + echo "tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz" + echo "cd vm-builder-0.12.4+bzr494" + echo "sudo python setup.py install" + echo "cd .." + exit 1 +fi + update_responses_pkg="libyaml-perl libfile-slurp-perl libxml-writer-perl libio-captureoutput-perl libfile-which-perl libparallel-forkmanager-perl libxml-libxml-perl libwww-perl libjson-perl" missing_pkg='' for pkg in $update_responses_pkg @@ -80,7 +81,7 @@ if [ -n "$missing_pkg" ] then echo "You are missing one or more dependencies for the update_responses script" echo "Please run" - echo " sudo apt-get install $missing_pkg" + echo " sudo torsocks apt-get install $missing_pkg" exit 1 fi diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml index 48c68eb..8f282e0 100644 --- a/gitian/descriptors/linux/gitian-bundle.yml +++ b/gitian/descriptors/linux/gitian-bundle.yml @@ -1,7 +1,8 @@ --- name: "bundle-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -106,9 +107,6 @@ script: | cd ../../../ # cd https-everywhere - # Workaround for git not knowing `git submodule -f` in the version shipped in - # 10.04. - sed 's/recursive -f/recursive/' -i makexpi.sh # XXX: Bloody hack to workaround a bug in HTTPS_E's git hash extraction in # makexpi.sh. See https://trac.torproject.org/projects/tor/ticket/10066 # The solution there does not work for us as doing something like diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml index 3c6c1f1..a0fc8e3 100644 --- a/gitian/descriptors/linux/gitian-firefox.yml +++ b/gitian/descriptors/linux/gitian-firefox.yml @@ -1,7 +1,8 @@ --- name: "torbrowser-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -10,9 +11,10 @@ packages: - "autoconf2.13" - "libgtk2.0-dev" - "libdbus-glib-1-dev" -- "yasm-1" +- "yasm" - "libasound2-dev" - "libgstreamer-plugins-base0.10-dev" +- "libxt-dev" - "hardening-wrapper" # To pass configure since ESR 31. - "libpulse-dev" @@ -45,9 +47,6 @@ script: | export DEB_BUILD_HARDENING_FORMAT=1 export DEB_BUILD_HARDENING_PIE=1 # - mkdir -p $INSTDIR/build/bin/ - ln -s /usr/bin/yasm-1 $INSTDIR/build/bin/yasm - export PATH=$PATH:$INSTDIR/build/bin # Preparing Python for Tor Browser unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip # TODO: We might want to have a smarter solution than hard-coding the version. diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml index 25b5a1b..a886572 100644 --- a/gitian/descriptors/linux/gitian-pluggable-transports.yml +++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml @@ -1,7 +1,8 @@ --- name: "pluggable-transports-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -54,7 +55,6 @@ script: | INSTDIR="$HOME/install" PTDIR="$INSTDIR/Tor/PluggableTransports" mkdir -p $PTDIR - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME export TZ=UTC export LC_ALL=C @@ -248,7 +248,8 @@ script: | cp -a obfs4proxy $PTDIR cd ../.. - # Grabbing the results + # Grabbing the results and making sure timestamps don't spoil them + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 cd $INSTDIR ~/build/dzip.sh pluggable-transports-linux$GBUILD_BITS-gbuilt.zip Tor/ Docs/ cp pluggable-transports-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/linux/gitian-tor.yml b/gitian/descriptors/linux/gitian-tor.yml index 630c2e0..f8f83e9 100644 --- a/gitian/descriptors/linux/gitian-tor.yml +++ b/gitian/descriptors/linux/gitian-tor.yml @@ -1,7 +1,8 @@ --- name: "tor-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -27,7 +28,6 @@ files: script: | INSTDIR="$HOME/install" source versions - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME export TZ=UTC export LC_ALL=C @@ -90,7 +90,8 @@ script: | objcopy --add-gnu-debuglink=./Debug/Tor/$LIB $INSTDIR/Tor/$LIB done - # Grabbing the results + # Grabbing the results and making sure timestamps don't spoil them + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 ~/build/dzip.sh tor-linux$GBUILD_BITS-gbuilt.zip Data/ Tor/ ~/build/dzip.sh tor-linux$GBUILD_BITS-debug.zip Debug/ cp tor-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml index a742c4e..c581248 100644 --- a/gitian/descriptors/linux/gitian-utils.yml +++ b/gitian/descriptors/linux/gitian-utils.yml @@ -1,7 +1,8 @@ --- name: "utils-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -21,6 +22,8 @@ packages: - "libxslt-dev" # Needed for passing the Python related part of configure in ESR 31. - "libssl-dev" +# Needed for binutils (64bit) as we are building with PIE enabled. +- "libstdc++6-4.7-pic" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://github.com/libevent/libevent.git" @@ -40,6 +43,7 @@ script: | source versions export TZ=UTC export LC_ALL=C + export FAKETIME=$REFERENCE_DATETIME umask 0022 # Config options for hardening-wrapper @@ -51,6 +55,14 @@ script: | # Building Binutils tar xjf binutils.tar.bz2 + # The libstdc++ shipped by default is non-PIC which breaks the binutils build + # if we build with DEB_BUILD_HARDENING_PIE=1. We need to install a PIC one AND + # make sure it gets used before the non-PIC one would. + if [ $GBUILD_BITS == "64" ]; + then + ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a + export LDFLAGS="-L/home/debian -lstdc++" + fi cd binutils* # We want to use gold as the linker in our toolchain mainly as it is way # faster when linking Tor Browser code (especially libxul). But apart from @@ -62,6 +74,15 @@ script: | cd .. export PATH=$INSTDIR/binutils/bin:$PATH + export LDFLAGS="" + # We need to disable `-Werror=format-security` as GCC does not build with it + # anymore. It seems it got audited for those problems already: + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48817. + export DEB_BUILD_HARDENING_FORMAT=0 + # libfaketime gets into our way when building GCC 4.9.x. See: + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61314 for details. Thus, we + # avoid it for the toolchain and cross our fingers. + # TODO: Test a newer libfaketime than 0.8. # Building GCC tar xjf gcc.tar.bz2 cd gcc-* @@ -71,13 +92,7 @@ script: | make install cd .. - # libfaketime gets into our way when building GCC 4.9.x. See: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61314 for details. Thus, we - # avoid it for the toolchain and cross our fingers. - # TODO: Test a newer libfaketime than 0.8. - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - export FAKETIME=$REFERENCE_DATETIME - + export DEB_BUILD_HARDENING_FORMAT=1 # Building Libevent cd libevent ./autogen.sh @@ -88,6 +103,7 @@ script: | cd .. # Building OpenSSL + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 tar xzf openssl.tar.gz cd openssl-* find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" @@ -108,6 +124,7 @@ script: | cd .. # Building lxml + export LD_PRELOAD="" tar xzf lxml.tar.gz cd lxml-* # Make sure we use our freshly built python binary here. Otherwise bad things @@ -115,7 +132,9 @@ script: | # rules. $INSTDIR/python/bin/python2.7 setup.py build cd build/lib* + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 ~/build/dzip.sh lxml-$LXML_VER-linux$GBUILD_BITS-utils.zip lxml + export LD_PRELOAD="" cp *utils.zip $OUTDIR cd ../../../ @@ -132,7 +151,8 @@ script: | make install cd .. - # Grabbing the remaining results + # Grabbing the remaining results and making sure timestamps don't spoil them + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 cd $INSTDIR ~/build/dzip.sh binutils-$BINUTILS_VER-linux$GBUILD_BITS-utils.zip binutils ~/build/dzip.sh gcc-$GCC_VER-linux$GBUILD_BITS-utils.zip gcc diff --git a/gitian/make-vms.sh b/gitian/make-vms.sh index a3b43b5..469a9ca 100755 --- a/gitian/make-vms.sh +++ b/gitian/make-vms.sh @@ -33,7 +33,14 @@ build_and_test_vm() { export LXC_ARCH=$arch ./bin/make-base-vm --suite $dist --lxc --arch $arch else - ./bin/make-base-vm --suite $dist --arch $arch + if [ "$dist" = "wheezy" ]; + then + export DISTRO=debian + ./bin/make-base-vm --distro debian --suite $dist --arch $arch + else + export DISTRO=ubuntu + ./bin/make-base-vm --suite $dist --arch $arch + fi fi make-clean-vm --suite $dist --arch $arch @@ -56,21 +63,21 @@ build_and_test_vm() { return 0 } -while ! build_and_test_vm lucid i386 32 +while ! build_and_test_vm wheezy i386 32 do - stop-target 32 lucid - rm ./base-lucid-i386* + stop-target 32 wheezy + rm ./base-wheezy-i386* echo - echo "Lucid i386 VM build failed... Trying again" + echo "Wheezy i386 VM build failed... Trying again" echo done -while ! build_and_test_vm lucid amd64 64 +while ! build_and_test_vm wheezy amd64 64 do - stop-target 64 lucid - rm ./base-lucid-amd64* + stop-target 64 wheezy + rm ./base-wheezy-amd64* echo - echo "Lucid amd64 VM build failed... Trying again" + echo "Wheezy amd64 VM build failed... Trying again" echo done @@ -79,7 +86,7 @@ do stop-target 32 precise rm ./base-precise-i386* echo - echo "Lucid amd64 VM build failed... Trying again" + echo "Precise amd64 VM build failed... Trying again" echo done @@ -88,7 +95,7 @@ do stop-target 64 precise rm ./base-precise-amd64* echo - echo "Lucid amd64 VM build failed... Trying again" + echo "Precise amd64 VM build failed... Trying again" echo done