[tbb-commits] [tor-browser-bundle/hardened-builds] Bug 15578: Switch Linux descriptors over to Wheezy

gk at torproject.org gk at torproject.org
Sun Jan 24 12:32:10 UTC 2016


commit 763784f552779b6dd4e58a76f441a8602b9018ba
Author: Georg Koppen <gk at torproject.org>
Date:   Mon Dec 7 12:33:16 2015 +0000

    Bug 15578: Switch Linux descriptors over to Wheezy
    
    Besides using Wheezy for building Linux bundles this patch cleans up our
    usage of libfaketime as well to make it obvious where exactly we still
    need it.
---
 gitian/Makefile                                    |    2 +-
 gitian/README.build                                |    2 +-
 gitian/check-prerequisites.sh                      |   41 ++++++++++----------
 gitian/descriptors/linux/gitian-bundle.yml         |    6 +--
 gitian/descriptors/linux/gitian-firefox.yml        |    9 ++---
 .../linux/gitian-pluggable-transports.yml          |    7 ++--
 gitian/descriptors/linux/gitian-tor.yml            |    7 ++--
 gitian/descriptors/linux/gitian-utils.yml          |   38 +++++++++++++-----
 gitian/make-vms.sh                                 |   29 ++++++++------
 9 files changed, 84 insertions(+), 57 deletions(-)

diff --git a/gitian/Makefile b/gitian/Makefile
index 5718d76..36b5ec3 100644
--- a/gitian/Makefile
+++ b/gitian/Makefile
@@ -119,7 +119,7 @@ clean-bundle:
 vmclean:
 	rm -rf ../../gitian-builder/*.qcow2
 	rm -rf ../../gitian-builder/base-*
-	rm -rf ../../gitian-builder/target-{lucid,precise}*
+	rm -rf ../../gitian-builder/target-{lucid,wheezy,precise}*
 
 distclean: vmclean
 	rm -rf ../../gitian-builder/inputs/*
diff --git a/gitian/README.build b/gitian/README.build
index f289791..4d01d6a 100644
--- a/gitian/README.build
+++ b/gitian/README.build
@@ -177,7 +177,7 @@ Known Issues and Quirks:
      where 'make vmclean' causes the rebuild of two VMs in a row.. This might
      trigger weird bugs in python-vm-builder.. To rebuild only one set of VMs,
      use either 'rm ../../gitian-builder/*precise*' (to remove the Windows/Mac
-     VMs) or 'rm ../../gitian-builder/*lucid*' (to remove the Linux VMs).
+     VMs) or 'rm ../../gitian-builder/*wheezy*' (to remove the Linux VMs).
 
      You probably want to make sure you have no stray qemu processes before
      rebuilding the VMs or starting a new build, too. 'killall qemu-kvm' is
diff --git a/gitian/check-prerequisites.sh b/gitian/check-prerequisites.sh
index cc16d0e..a5f8393 100755
--- a/gitian/check-prerequisites.sh
+++ b/gitian/check-prerequisites.sh
@@ -17,7 +17,7 @@ then
   VERSION=`cat /etc/issue | grep -Eo '[0-9]{2}' | head -1`
   if [ "$VERSION" -ge "14" ];
   then
-    dpkg -s ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion torsocks tor 2>/dev/null >/dev/null
+    dpkg -s ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion torsocks tor 2>/dev/null >/dev/null
 
     if [ $? -ne 0 ];
     then
@@ -25,7 +25,7 @@ then
       echo
       echo "Please run:"
       echo " sudo apt-get install torsocks tor"
-      echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng python-vm-builder qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion"
+      echo " sudo torsocks apt-get install ruby apache2 git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot faketime zip unzip subversion"
       exit 1
     fi
   else
@@ -45,28 +45,29 @@ then
     echo " sudo torsocks apt-get install ruby git apt-cacher-ng qemu-kvm virt-what lxc lxctl fakeroot zip unzip python-cheetah debootstrap parted kpartx rsync"
     exit 1
   fi
-
-  # python-vm-builder is special as we don't have a Debian package for it.
-  vmbuilder --help 2>/dev/null >/dev/null
-  if [ $? -ne 0 ];
-  then
-    echo "The VM tool python-vm-builder is missing."
-    echo
-    echo "Please run"
-    echo 'torsocks wget -U "" http://archive.ubuntu.com/ubuntu/pool/universe/v/vm-builder/vm-builder_0.12.4+bzr489.orig.tar.gz'
-    echo 'echo "ec12e0070a007989561bfee5862c89a32c301992dd2771c4d5078ef1b3014f03  vm-builder_0.12.4+bzr489.orig.tar.gz" | sha256sum -c'
-    echo "# (verification -- must return OK)"
-    echo "tar -zxvf vm-builder_0.12.4+bzr489.orig.tar.gz"
-    echo "cd vm-builder-0.12.4+bzr489"
-    echo "sudo python setup.py install"
-    echo "cd .."
-    exit 1
-  fi
 else
   echo "We need Debian or Ubuntu which seem to be missing. Aborting."
   exit 1
 fi
 
+# vmbuilder is special as we don't have a package for it yet.
+# XXX: Make sure an already installed vmbuilder is recent enough.
+vmbuilder --help 2>/dev/null >/dev/null
+if [ $? -ne 0 ];
+then
+  echo "The VM tool python-vm-builder is missing."
+  echo
+  echo "Please run"
+  echo 'torsocks wget -U "" https://bugs.launchpad.net/ubuntu/+archive/primary/+files/vm-builder_0.12.4+bzr494.orig.tar.gz'
+  echo 'echo "76cbf8c52c391160b2641e7120dbade5afded713afaa6032f733a261f13e6a8e  vm-builder_0.12.4+bzr494.orig.tar.gz" | sha256sum -c'
+  echo "# (verification -- must return OK)"
+  echo "tar -zxvf vm-builder_0.12.4+bzr494.orig.tar.gz"
+  echo "cd vm-builder-0.12.4+bzr494"
+  echo "sudo python setup.py install"
+  echo "cd .."
+  exit 1
+fi
+
 update_responses_pkg="libyaml-perl libfile-slurp-perl libxml-writer-perl libio-captureoutput-perl libfile-which-perl libparallel-forkmanager-perl libxml-libxml-perl libwww-perl libjson-perl"
 missing_pkg=''
 for pkg in $update_responses_pkg
@@ -80,7 +81,7 @@ if [ -n "$missing_pkg" ]
 then
     echo "You are missing one or more dependencies for the update_responses script"
     echo "Please run"
-    echo " sudo apt-get install $missing_pkg"
+    echo " sudo torsocks apt-get install $missing_pkg"
     exit 1
 fi
 
diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml
index 48c68eb..8f282e0 100644
--- a/gitian/descriptors/linux/gitian-bundle.yml
+++ b/gitian/descriptors/linux/gitian-bundle.yml
@@ -1,7 +1,8 @@
 ---
 name: "bundle-linux"
+distro: "debian"
 suites:
-- "lucid"
+- "wheezy"
 architectures:
 - "amd64"
 packages:
@@ -106,9 +107,6 @@ script: |
   cd ../../../
   #
   cd https-everywhere
-  # Workaround for git not knowing `git submodule -f` in the version shipped in
-  # 10.04.
-  sed 's/recursive -f/recursive/' -i makexpi.sh
   # XXX: Bloody hack to workaround a bug in HTTPS_E's git hash extraction in
   # makexpi.sh. See https://trac.torproject.org/projects/tor/ticket/10066
   # The solution there does not work for us as doing something like
diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml
index 3c6c1f1..a0fc8e3 100644
--- a/gitian/descriptors/linux/gitian-firefox.yml
+++ b/gitian/descriptors/linux/gitian-firefox.yml
@@ -1,7 +1,8 @@
 ---
 name: "torbrowser-linux"
+distro: "debian"
 suites:
-- "lucid"
+- "wheezy"
 architectures:
 - "amd64"
 packages:
@@ -10,9 +11,10 @@ packages:
 - "autoconf2.13"
 - "libgtk2.0-dev"
 - "libdbus-glib-1-dev"
-- "yasm-1"
+- "yasm"
 - "libasound2-dev"
 - "libgstreamer-plugins-base0.10-dev"
+- "libxt-dev"
 - "hardening-wrapper"
 # To pass configure since ESR 31.
 - "libpulse-dev"
@@ -45,9 +47,6 @@ script: |
   export DEB_BUILD_HARDENING_FORMAT=1
   export DEB_BUILD_HARDENING_PIE=1
   #
-  mkdir -p $INSTDIR/build/bin/
-  ln -s /usr/bin/yasm-1 $INSTDIR/build/bin/yasm
-  export PATH=$PATH:$INSTDIR/build/bin
   # Preparing Python for Tor Browser
   unzip -d $INSTDIR python-linux$GBUILD_BITS-utils.zip
   # TODO: We might want to have a smarter solution than hard-coding the version.
diff --git a/gitian/descriptors/linux/gitian-pluggable-transports.yml b/gitian/descriptors/linux/gitian-pluggable-transports.yml
index 25b5a1b..a886572 100644
--- a/gitian/descriptors/linux/gitian-pluggable-transports.yml
+++ b/gitian/descriptors/linux/gitian-pluggable-transports.yml
@@ -1,7 +1,8 @@
 ---
 name: "pluggable-transports-linux"
+distro: "debian"
 suites:
-- "lucid"
+- "wheezy"
 architectures:
 - "amd64"
 packages:
@@ -54,7 +55,6 @@ script: |
   INSTDIR="$HOME/install"
   PTDIR="$INSTDIR/Tor/PluggableTransports"
   mkdir -p $PTDIR
-  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   export FAKETIME=$REFERENCE_DATETIME
   export TZ=UTC
   export LC_ALL=C
@@ -248,7 +248,8 @@ script: |
   cp -a obfs4proxy $PTDIR
   cd ../..
 
-  # Grabbing the results
+  # Grabbing the results and making sure timestamps don't spoil them
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   cd $INSTDIR
   ~/build/dzip.sh pluggable-transports-linux$GBUILD_BITS-gbuilt.zip Tor/ Docs/
   cp pluggable-transports-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/
diff --git a/gitian/descriptors/linux/gitian-tor.yml b/gitian/descriptors/linux/gitian-tor.yml
index 630c2e0..f8f83e9 100644
--- a/gitian/descriptors/linux/gitian-tor.yml
+++ b/gitian/descriptors/linux/gitian-tor.yml
@@ -1,7 +1,8 @@
 ---
 name: "tor-linux"
+distro: "debian"
 suites:
-- "lucid"
+- "wheezy"
 architectures:
 - "amd64"
 packages:
@@ -27,7 +28,6 @@ files:
 script: |
   INSTDIR="$HOME/install"
   source versions
-  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   export FAKETIME=$REFERENCE_DATETIME
   export TZ=UTC
   export LC_ALL=C
@@ -90,7 +90,8 @@ script: |
     objcopy --add-gnu-debuglink=./Debug/Tor/$LIB $INSTDIR/Tor/$LIB
   done
 
-  # Grabbing the results
+  # Grabbing the results and making sure timestamps don't spoil them
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   ~/build/dzip.sh tor-linux$GBUILD_BITS-gbuilt.zip Data/ Tor/
   ~/build/dzip.sh tor-linux$GBUILD_BITS-debug.zip Debug/
   cp tor-linux$GBUILD_BITS-gbuilt.zip $OUTDIR/
diff --git a/gitian/descriptors/linux/gitian-utils.yml b/gitian/descriptors/linux/gitian-utils.yml
index a742c4e..c581248 100644
--- a/gitian/descriptors/linux/gitian-utils.yml
+++ b/gitian/descriptors/linux/gitian-utils.yml
@@ -1,7 +1,8 @@
 ---
 name: "utils-linux"
+distro: "debian"
 suites:
-- "lucid"
+- "wheezy"
 architectures:
 - "amd64"
 packages:
@@ -21,6 +22,8 @@ packages:
 - "libxslt-dev"
 # Needed for passing the Python related part of configure in ESR 31.
 - "libssl-dev"
+# Needed for binutils (64bit) as we are building with PIE enabled.
+- "libstdc++6-4.7-pic"
 reference_datetime: "2000-01-01 00:00:00"
 remotes:
 - "url": "https://github.com/libevent/libevent.git"
@@ -40,6 +43,7 @@ script: |
   source versions
   export TZ=UTC
   export LC_ALL=C
+  export FAKETIME=$REFERENCE_DATETIME
   umask 0022
 
   # Config options for hardening-wrapper
@@ -51,6 +55,14 @@ script: |
 
   # Building Binutils
   tar xjf binutils.tar.bz2
+  # The libstdc++ shipped by default is non-PIC which breaks the binutils build
+  # if we build with DEB_BUILD_HARDENING_PIE=1. We need to install a PIC one AND
+  # make sure it gets used before the non-PIC one would.
+  if [ $GBUILD_BITS == "64" ];
+  then
+    ln -s /usr/lib/gcc/x86_64-linux-gnu/4.7/libstdc++_pic.a libstdc++.a
+    export LDFLAGS="-L/home/debian -lstdc++"
+  fi
   cd binutils*
   # We want to use gold as the linker in our toolchain mainly as it is way
   # faster when linking Tor Browser code (especially libxul). But apart from
@@ -62,6 +74,15 @@ script: |
   cd ..
   export PATH=$INSTDIR/binutils/bin:$PATH
 
+  export LDFLAGS=""
+  # We need to disable `-Werror=format-security` as GCC does not build with it
+  # anymore. It seems it got audited for those problems already:
+  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48817.
+  export DEB_BUILD_HARDENING_FORMAT=0
+  # libfaketime gets into our way when building GCC 4.9.x. See:
+  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61314 for details. Thus, we
+  # avoid it for the toolchain and cross our fingers.
+  # TODO: Test a newer libfaketime than 0.8.
   # Building GCC
   tar xjf gcc.tar.bz2
   cd gcc-*
@@ -71,13 +92,7 @@ script: |
   make install
   cd ..
 
-  # libfaketime gets into our way when building GCC 4.9.x. See:
-  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61314 for details. Thus, we
-  # avoid it for the toolchain and cross our fingers.
-  # TODO: Test a newer libfaketime than 0.8.
-  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
-  export FAKETIME=$REFERENCE_DATETIME
-
+  export DEB_BUILD_HARDENING_FORMAT=1
   # Building Libevent
   cd libevent
   ./autogen.sh
@@ -88,6 +103,7 @@ script: |
   cd ..
 
   # Building OpenSSL
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   tar xzf openssl.tar.gz
   cd openssl-*
   find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
@@ -108,6 +124,7 @@ script: |
   cd ..
 
   # Building lxml
+  export LD_PRELOAD=""
   tar xzf lxml.tar.gz
   cd lxml-*
   # Make sure we use our freshly built python binary here. Otherwise bad things
@@ -115,7 +132,9 @@ script: |
   # rules.
   $INSTDIR/python/bin/python2.7 setup.py build
   cd build/lib*
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   ~/build/dzip.sh lxml-$LXML_VER-linux$GBUILD_BITS-utils.zip lxml
+  export LD_PRELOAD=""
   cp *utils.zip $OUTDIR
   cd ../../../
 
@@ -132,7 +151,8 @@ script: |
   make install
   cd ..
 
-  # Grabbing the remaining results
+  # Grabbing the remaining results and making sure timestamps don't spoil them
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   cd $INSTDIR
   ~/build/dzip.sh binutils-$BINUTILS_VER-linux$GBUILD_BITS-utils.zip binutils
   ~/build/dzip.sh gcc-$GCC_VER-linux$GBUILD_BITS-utils.zip gcc
diff --git a/gitian/make-vms.sh b/gitian/make-vms.sh
index a3b43b5..469a9ca 100755
--- a/gitian/make-vms.sh
+++ b/gitian/make-vms.sh
@@ -33,7 +33,14 @@ build_and_test_vm() {
       export LXC_ARCH=$arch
       ./bin/make-base-vm --suite $dist --lxc --arch $arch
     else
-      ./bin/make-base-vm --suite $dist --arch $arch
+      if [ "$dist" = "wheezy" ];
+      then
+        export DISTRO=debian
+        ./bin/make-base-vm --distro debian --suite $dist --arch $arch
+      else
+        export DISTRO=ubuntu
+        ./bin/make-base-vm --suite $dist --arch $arch
+      fi
     fi
 
     make-clean-vm --suite $dist --arch $arch
@@ -56,21 +63,21 @@ build_and_test_vm() {
   return 0
 }
 
-while ! build_and_test_vm lucid i386 32
+while ! build_and_test_vm wheezy i386 32
 do
-  stop-target 32 lucid
-  rm ./base-lucid-i386*
+  stop-target 32 wheezy
+  rm ./base-wheezy-i386*
   echo
-  echo "Lucid i386 VM build failed... Trying again"
+  echo "Wheezy i386 VM build failed... Trying again"
   echo
 done
 
-while ! build_and_test_vm lucid amd64 64
+while ! build_and_test_vm wheezy amd64 64
 do
-  stop-target 64 lucid
-  rm ./base-lucid-amd64*
+  stop-target 64 wheezy
+  rm ./base-wheezy-amd64*
   echo
-  echo "Lucid amd64 VM build failed... Trying again"
+  echo "Wheezy amd64 VM build failed... Trying again"
   echo
 done
 
@@ -79,7 +86,7 @@ do
   stop-target 32 precise
   rm ./base-precise-i386*
   echo
-  echo "Lucid amd64 VM build failed... Trying again"
+  echo "Precise amd64 VM build failed... Trying again"
   echo
 done
 
@@ -88,7 +95,7 @@ do
   stop-target 64 precise
   rm ./base-precise-amd64*
   echo
-  echo "Lucid amd64 VM build failed... Trying again"
+  echo "Precise amd64 VM build failed... Trying again"
   echo
 done
 





More information about the tbb-commits mailing list