commit b429deeb6d6cd749d11855cfec058e168cddfc3c Author: Georg Koppen <gk@torproject.org> Date: Thu Feb 11 13:04:11 2016 +0000 Revert "Revert "Bug 15578: Switch Linux descriptors over to Wheezy"" This reverts commit a125e1c1bfc3babae9636750b4722b4ab7010ca3. --- gitian/Makefile | 2 +- gitian/README.build | 2 +- gitian/check-prerequisites.sh | 41 +++++++++++----------- gitian/descriptors/linux/gitian-bundle.yml | 6 ++-- gitian/descriptors/linux/gitian-firefox.yml | 9 +++-- .../linux/gitian-pluggable-transports.yml | 7 ++-- gitian/descriptors/linux/gitian-tor.yml | 7 ++-- gitian/descriptors/linux/gitian-utils.yml | 38 +++++++++++++++----- gitian/make-vms.sh | 29 +++++++++------ 9 files changed, 84 insertions(+), 57 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index 5718d76..36b5ec3 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -119,7 +119,7 @@ clean-bundle: vmclean: rm -rf ../../gitian-builder/*.qcow2 rm -rf ../../gitian-builder/base-* - rm -rf ../../gitian-builder/target-{lucid,precise}* + rm -rf ../../gitian-builder/target-{lucid,wheezy,precise}* distclean: vmclean rm -rf ../../gitian-builder/inputs/* diff --git a/gitian/README.build b/gitian/README.build index f289791..4d01d6a 100644 --- a/gitian/README.build +++ b/gitian/README.build @@ -177,7 +177,7 @@ Known Issues and Quirks: where 'make vmclean' causes the rebuild of two VMs in a row.. This might trigger weird bugs in python-vm-builder.. To rebuild only one set of VMs, use either 'rm ../../gitian-builder/*precise*' (to remove the Windows/Mac - VMs) or 'rm ../../gitian-builder/*lucid*' (to remove the Linux VMs). + VMs) or 'rm ../../gitian-builder/*wheezy*' (to remove the Linux VMs). You probably want to make sure you have no stray qemu processes before rebuilding the VMs or starting a new build, too. 'killall qemu-kvm' is diff --git a/gitian/check-prerequisites.sh b/gitian/check-prerequisites.sh index cc16d0e..a5f8393 100755 --- a/gitian/check-prerequisites.sh +++ b/gitian/check-prerequisites.sh @@ -17,7 +17,7 @@ then VERSION=`cat /etc/issue | grep -Eo '[0-9]{2}' | head -1` if [ "$VERSION" -ge "14" ]; then - dpkg -s ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion torsocks tor 2>/dev/null >/dev/null + dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion torsocks tor 2>/dev/null >/dev/null if [ $? -ne 0 ]; then @@ -25,7 +25,7 @@ then echo echo "Please run:" echo " sudo apt-get install torsocks tor" - echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion" + echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion" exit 1 fi else @@ -45,28 +45,29 @@ then echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync" exit 1 fi - - # python-vm-builder is special as we don't have a Debian package for it. - vmbuilder --help 2>/dev/null >/dev/null - if [ $? -ne 0 ]; - then - echo "The VM tool python-vm-builder is missing." - echo - echo "Please run" - echo 'torsocks wget -U "" http://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12....' - echo 'echo "ec12e0070a007989561bfee5862c89a32c301992dd2771c4d5078ef1b3014f03 vm-builder_0.12.4+bzr489.orig.tar.gz" | sha256sum -c' - echo "# (verification -- must return OK)" - echo "tar -zxvf vm-builder_0.12.4+bzr489.orig.tar.gz" - echo "cd vm-builder-0.12.4+bzr489" - echo "sudo python setup.py install" - echo "cd .." - exit 1 - fi else echo "We need Debian or Ubuntu which seem to be missing. Aborting." exit 1 fi +# vmbuilder is special as we don't have a package for it yet. +# XXX: Make sure an already installed vmbuilder is recent enough. +vmbuilder --help 2>/dev/null >/dev/null +if [ $? -ne 0 ]; +then + echo "The VM tool python-vm-builder is missing." + echo + echo "Please run" + echo 'torsocks wget -U "" https://bugs.launchpad.net/ubuntu/+archive/primary/+files/vm-builder_0.12.4+...' + echo 'echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c' + echo "# (verification -- must return OK)" + echo "tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz" + echo "cd vm-builder-0.12.4+bzr494" + echo "sudo python setup.py install" + echo "cd .." + exit 1 +fi + update_responses_pkg="libyaml-perl libfile-slurp-perl libxml-writer-perl libio-captureoutput-perl libfile-which-perl libparallel-forkmanager-perl libxml-libxml-perl libwww-perl libjson-perl" missing_pkg='' for pkg in $update_responses_pkg @@ -80,7 +81,7 @@ if [ -n "$missing_pkg" ] then echo "You are missing one or more dependencies for the update_responses script" echo "Please run" - echo " sudo apt-get install $missing_pkg" + echo " sudo torsocks apt-get install $missing_pkg" exit 1 fi diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml index 7629b4f..a9a0e36 100644 --- a/gitian/descriptors/linux/gitian-bundle.yml +++ b/gitian/descriptors/linux/gitian-bundle.yml @@ -1,7 +1,8 @@ --- name: "bundle-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -107,9 +108,6 @@ script: | cd ../../../ # cd https-everywhere - # Workaround for git not knowing `git submodule -f` in the version shipped in - # 10.04. - sed 's/recursive -f/recursive/' -i makexpi.sh # XXX: Bloody hack to workaround a bug in HTTPS_E's git hash extraction in # makexpi.sh. See https://trac.torproject.org/projects/tor/ticket/10066 # The solution there does not work for us as doing something like diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml index 3c6c1f1..a0fc8e3 100644 --- a/gitian/descriptors/linux/gitian-firefox.yml +++ b/gitian/descriptors/linux/gitian-firefox.yml @@ -1,7 +1,8 @@ --- name: "torbrowser-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -10,9 +11,10 @@ packages: - "autoconf2.13" - "libgtk2.0-dev" - "libdbus-glib-1-dev" -- "yasm-1" +- "yasm" - "libasound2-dev" - "libgstreamer-plugins-base0.10-dev" +- "libxt-dev" - "hardening-wrapper" # To pass configure since ESR 31. - "libpulse-dev" @@ -45,9 +47,6 @@ script: | export DEB_BUILD_HARDENING_FORMAT=1 export DEB_BUILD_HARDENING_PIE=1 # - mkdir -p $INSTDIR/build/bin/ - ln -s /usr/bin/yasm-1 $INSTDIR/build/bin/yasm - export PATH=$PATH:$INSTDIR/build/bin # Preparing Python for Tor Browser unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip # TODO: We might want to have a smarter solution than hard-coding the version. diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml index 25b5a1b..a886572 100644 --- a/gitian/descriptors/linux/gitian-pluggable-transports.yml +++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml @@ -1,7 +1,8 @@ --- name: "pluggable-transports-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -54,7 +55,6 @@ script: | INSTDIR="$HOME/install" PTDIR="$INSTDIR/Tor/PluggableTransports" mkdir -p $PTDIR - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME export TZ=UTC export LC_ALL=C @@ -248,7 +248,8 @@ script: | cp -a obfs4proxy $PTDIR cd ../.. - # Grabbing the results + # Grabbing the results and making sure timestamps don't spoil them + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 cd $INSTDIR ~/build/dzip.sh pluggable-transports-linux$GBUILD_BITS-gbuilt.zip Tor/ Docs/ cp pluggable-transports-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/linux/gitian-tor.yml b/gitian/descriptors/linux/gitian-tor.yml index 630c2e0..f8f83e9 100644 --- a/gitian/descriptors/linux/gitian-tor.yml +++ b/gitian/descriptors/linux/gitian-tor.yml @@ -1,7 +1,8 @@ --- name: "tor-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -27,7 +28,6 @@ files: script: | INSTDIR="$HOME/install" source versions - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME export TZ=UTC export LC_ALL=C @@ -90,7 +90,8 @@ script: | objcopy --add-gnu-debuglink=./Debug/Tor/$LIB $INSTDIR/Tor/$LIB done - # Grabbing the results + # Grabbing the results and making sure timestamps don't spoil them + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 ~/build/dzip.sh tor-linux$GBUILD_BITS-gbuilt.zip Data/ Tor/ ~/build/dzip.sh tor-linux$GBUILD_BITS-debug.zip Debug/ cp tor-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml index 1e3102f..1e04b36 100644 --- a/gitian/descriptors/linux/gitian-utils.yml +++ b/gitian/descriptors/linux/gitian-utils.yml @@ -1,7 +1,8 @@ --- name: "utils-linux" +distro: "debian" suites: -- "lucid" +- "wheezy" architectures: - "amd64" packages: @@ -21,6 +22,8 @@ packages: - "libxslt-dev" # Needed for passing the Python related part of configure in ESR 31. - "libssl-dev" +# Needed for binutils (64bit) as we are building with PIE enabled. +- "libstdc++6-4.7-pic" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://github.com/libevent/libevent.git" @@ -40,6 +43,7 @@ script: | source versions export TZ=UTC export LC_ALL=C + export FAKETIME=$REFERENCE_DATETIME umask 0022 # Config options for hardening-wrapper @@ -51,6 +55,14 @@ script: | # Building Binutils tar xjf binutils.tar.bz2 + # The libstdc++ shipped by default is non-PIC which breaks the binutils build + # if we build with DEB_BUILD_HARDENING_PIE=1. We need to install a PIC one AND + # make sure it gets used before the non-PIC one would. + if [ $GBUILD_BITS == "64" ]; + then + ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a + export LDFLAGS="-L/home/debian -lstdc++" + fi cd binutils* # We want to use gold as the linker in our toolchain mainly as it is way # faster when linking Tor Browser code (especially libxul). But apart from @@ -61,6 +73,15 @@ script: | make install cd .. + export LDFLAGS="" + # We need to disable `-Werror=format-security` as GCC does not build with it + # anymore. It seems it got audited for those problems already: + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48817. + export DEB_BUILD_HARDENING_FORMAT=0 + # libfaketime gets into our way when building GCC 4.9.x. See: + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61314 for details. Thus, we + # avoid it for the toolchain and cross our fingers. + # TODO: Test a newer libfaketime than 0.8. # Building GCC tar xjf gcc.tar.bz2 cd gcc-* @@ -70,13 +91,7 @@ script: | make install cd .. - # libfaketime gets into our way when building GCC 4.9.x. See: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61314 for details. Thus, we - # avoid it for the toolchain and cross our fingers. - # TODO: Test a newer libfaketime than 0.8. - export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 - export FAKETIME=$REFERENCE_DATETIME - + export DEB_BUILD_HARDENING_FORMAT=1 # Building Libevent cd libevent ./autogen.sh @@ -87,6 +102,7 @@ script: | cd .. # Building OpenSSL + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 tar xzf openssl.tar.gz cd openssl-* find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" @@ -107,6 +123,7 @@ script: | cd .. # Building lxml + export LD_PRELOAD="" tar xzf lxml.tar.gz cd lxml-* # Make sure we use our freshly built python binary here. Otherwise bad things @@ -114,7 +131,9 @@ script: | # rules. $INSTDIR/python/bin/python2.7 setup.py build cd build/lib* + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 ~/build/dzip.sh lxml-$LXML_VER-linux$GBUILD_BITS-utils.zip lxml + export LD_PRELOAD="" cp *utils.zip $OUTDIR cd ../../../ @@ -131,7 +150,8 @@ script: | make install cd .. - # Grabbing the remaining results + # Grabbing the remaining results and making sure timestamps don't spoil them + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 cd $INSTDIR ~/build/dzip.sh binutils-$BINUTILS_VER-linux$GBUILD_BITS-utils.zip binutils ~/build/dzip.sh gcc-$GCC_VER-linux$GBUILD_BITS-utils.zip gcc diff --git a/gitian/make-vms.sh b/gitian/make-vms.sh index a3b43b5..469a9ca 100755 --- a/gitian/make-vms.sh +++ b/gitian/make-vms.sh @@ -33,7 +33,14 @@ build_and_test_vm() { export LXC_ARCH=$arch ./bin/make-base-vm --suite $dist --lxc --arch $arch else - ./bin/make-base-vm --suite $dist --arch $arch + if [ "$dist" = "wheezy" ]; + then + export DISTRO=debian + ./bin/make-base-vm --distro debian --suite $dist --arch $arch + else + export DISTRO=ubuntu + ./bin/make-base-vm --suite $dist --arch $arch + fi fi make-clean-vm --suite $dist --arch $arch @@ -56,21 +63,21 @@ build_and_test_vm() { return 0 } -while ! build_and_test_vm lucid i386 32 +while ! build_and_test_vm wheezy i386 32 do - stop-target 32 lucid - rm ./base-lucid-i386* + stop-target 32 wheezy + rm ./base-wheezy-i386* echo - echo "Lucid i386 VM build failed... Trying again" + echo "Wheezy i386 VM build failed... Trying again" echo done -while ! build_and_test_vm lucid amd64 64 +while ! build_and_test_vm wheezy amd64 64 do - stop-target 64 lucid - rm ./base-lucid-amd64* + stop-target 64 wheezy + rm ./base-wheezy-amd64* echo - echo "Lucid amd64 VM build failed... Trying again" + echo "Wheezy amd64 VM build failed... Trying again" echo done @@ -79,7 +86,7 @@ do stop-target 32 precise rm ./base-precise-i386* echo - echo "Lucid amd64 VM build failed... Trying again" + echo "Precise amd64 VM build failed... Trying again" echo done @@ -88,7 +95,7 @@ do stop-target 64 precise rm ./base-precise-amd64* echo - echo "Lucid amd64 VM build failed... Trying again" + echo "Precise amd64 VM build failed... Trying again" echo done