[tor-browser-bundle/master] Bug 20352: Adding sandbox to our Gitian build - tbb-commits

6 Dec 2016

commit 522cee6cb9c83be66cc2a6b5bbfefdb3c2bc3217
Author: Georg Koppen gk@torproject.org
Date:   Tue Dec 6 21:20:53 2016 +0000
Bug 20352: Adding sandbox to our Gitian build
On Linux we start building Yawning's sandbox code while producing the
    Linux Tor Browser bundles.
For that to work properly we need to use Debian Jessie images and
    install some packages from backports as not all dependencies are
    available in Wheezy.
---
 gitian/Makefile                             |  6 ++-
 gitian/descriptors/linux/gitian-sandbox.yml | 77 +++++++++++++++++++++++++++++
 gitian/fetch-inputs.sh                      |  3 ++
 gitian/make-vms.sh                          | 22 ++++++++-
 gitian/mkbundle-linux.sh                    | 44 +++++++++++++----
 gitian/versions.nightly                     |  3 ++
 6 files changed, 141 insertions(+), 14 deletions(-)

diff --git a/gitian/Makefile b/gitian/Makefile
index a4a757f..5694cbf 100644
--- a/gitian/Makefile
+++ b/gitian/Makefile
@@ -101,8 +101,7 @@ prep-alpha:
    ./check-prerequisites.sh
    $(TORSOCKS) ./fetch-inputs.sh ../../gitian-builder/inputs/ versions.alpha
-
-clean: clean-utils clean-tor clean-browser clean-pt clean-bundle
+clean: clean-utils clean-tor clean-browser clean-pt clean-sandbox clean-bundle
    rm -f ../../gitian-builder/inputs/*.yml
    rm -f ../../gitian-builder/inputs/bundle.inputs
    rm -f ../../gitian-builder/inputs/versions*
@@ -124,6 +123,9 @@ clean-browser: clean-bundle
 clean-pt: clean-bundle
    rm -f ../../gitian-builder/inputs/pluggable-transports*
+clean-sandbox: clean-bundle
+	rm -f ../../gitian-builder/inputs/sandbox-linux*
+
 clean-bundle:
    rm -f ../../gitian-builder/inputs/bundle-*
diff --git a/gitian/descriptors/linux/gitian-sandbox.yml b/gitian/descriptors/linux/gitian-sandbox.yml
new file mode 100644
index 0000000..0f4f78c
--- /dev/null
+++ b/gitian/descriptors/linux/gitian-sandbox.yml
@@ -0,0 +1,77 @@
+---
+name: "sandbox-linux"
+distro: "debian"
+suites:
+- "jessie"
+architectures:
+- "i386"
+- "amd64"
+packages:
+- "unzip"
+- "zip"
+- "hardening-wrapper"
+# Needed for the sandboxing code
+- "libx11-dev"
+- "pkg-config"
+- "libgtk-3-dev"
+backports_packages:
+- "libseccomp-dev"
+- "libseccomp2"
+reference_datetime: "2000-01-01 00:00:00"
+remotes:
+- "url": "https://github.com/pkg/error"
+  "dir": "errors"
+- "url": "https://github.com/constabulary/gb"
+  "dir": "gb"
+- "url": "https://git.schwanenlied.me/yawning/sandboxed-tor-browser"
+  "dir": "sandbox"
+files:
+- "go-linux32-utils.zip"
+- "go-linux64-utils.zip"
+- "dzip.sh"
+script: |
+  INSTDIR="$HOME/install"
+  mkdir $INSTDIR/sandbox
+  export REFERENCE_DATETIME
+  export TZ=UTC
+  export LC_ALL=C
+  umask 0022
+
+  # Config options for hardening-wrapper for the stub
+  export DEB_BUILD_HARDENING=1
+  export DEB_BUILD_HARDENING_STACKPROTECTOR=1
+  export DEB_BUILD_HARDENING_FORTIFY=1
+  export DEB_BUILD_HARDENING_FORMAT=1
+  export DEB_BUILD_HARDENING_PIE=1
+
+  unzip -d $INSTDIR go-linux$GBUILD_BITS-utils.zip
+  export GOROOT="$INSTDIR/go"
+  export GOPATH="$HOME/go"
+  export PATH="$PATH:$GOROOT/bin:$GOPATH/bin"
+
+  # Building errors
+  cd errors
+  find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
+  mkdir -p "$GOPATH/src/github.com/pkg/"
+  ln -sf "$PWD" "$GOPATH/src/github.com/pkg/errors"
+  go install github.com/pkg/errors
+  cd ..
+
+  # Building gb
+  cd gb
+  find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME"
+  mkdir -p "$GOPATH/src/github.com/constabulary/"
+  ln -sf "$PWD" "$GOPATH/src/github.com/constabulary/gb"
+  go install github.com/constabulary/gb/cmd/gb
+  cd ..
+
+  # XXX: Use own C compiler
+  # Building the sandbox
+  cd sandbox
+  make
+  cp bin/sandboxed-tor-browser $INSTDIR/sandbox
+  cd ..
+
+  cd $INSTDIR
+  ~/build/dzip.sh sandbox-linux$GBUILD_BITS.zip sandbox/
+  cp sandbox-linux$GBUILD_BITS.zip $OUTDIR/
diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh
index bf60ca3..b3d5c26 100755
--- a/gitian/fetch-inputs.sh
+++ b/gitian/fetch-inputs.sh
@@ -293,6 +293,9 @@ goxcrypto             https://go.googlesource.com/crypto  $GO_X_CRYPTO_TAG
 goxnet                https://go.googlesource.com/net  $GO_X_NET_TAG
 obfs4                 https://git.torproject.org/pluggable-transports/obfs4.git $OBFS4_TAG
 noto-fonts            https://github.com/googlei18n/noto-fonts $NOTOFONTS_TAG
+errors                https://github.com/pkg/errors $ERRORS_TAG
+gb                    https://github.com/constabulary/gb $GB_TAG
+sandbox               https://git.schwanenlied.me/yawning/sandboxed-tor-browser $SANDBOX_TAG
 EOF
# HTTPS-Everywhere is special, too. We need to initialize the git submodules and
diff --git a/gitian/make-vms.sh b/gitian/make-vms.sh
index 089865b..7edd2b4 100755
--- a/gitian/make-vms.sh
+++ b/gitian/make-vms.sh
@@ -32,7 +32,7 @@ build_and_test_vm() {
     then
       export LXC_SUITE=$dist
       export LXC_ARCH=$arch
-      if [ "$dist" = "wheezy" ];
+      if [ "$dist" = "wheezy" -o "$dist" = "jessie" ];
       then
         export DISTRO=debian
         ./bin/make-base-vm --distro debian --suite $dist --lxc --arch $arch
@@ -41,7 +41,7 @@ build_and_test_vm() {
         ./bin/make-base-vm --suite $dist --lxc --arch $arch
       fi
     else
-      if [ "$dist" = "wheezy" ];
+      if [ "$dist" = "wheezy" -o "$dist" = "jessie" ];
       then
         export DISTRO=debian
         ./bin/make-base-vm --distro debian --suite $dist --arch $arch
@@ -89,6 +89,24 @@ do
   echo
 done
+while ! build_and_test_vm jessie i386 32
+do
+  stop-target 32 jessie
+  rm ./base-jessie-amd64*
+  echo
+  echo "Jessie i386 VM build failed... Trying again"
+  echo
+done
+
+while ! build_and_test_vm jessie amd64 64
+do
+  stop-target 64 jessie
+  rm ./base-jessie-amd64*
+  echo
+  echo "Jessie amd64 VM build failed... Trying again"
+  echo
+done
+
 while ! build_and_test_vm precise i386 32
 do
   stop-target 32 precise
diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh
index f8aaec1..430b666 100755
--- a/gitian/mkbundle-linux.sh
+++ b/gitian/mkbundle-linux.sh
@@ -78,6 +78,7 @@ die_msg() {
 # the utils both if we verify tags and if we don't.
LIBEVENT_TAG_ORIG=$LIBEVENT_TAG
+SANDBOX_ORIG_TAG=$SANDBOX_TAG
if [ "z$VERIFY_TAGS" = "z1" ];
 then
@@ -112,7 +113,7 @@ if [ ! -f inputs/binutils-$BINUTILS_VER-linux32-utils.zip -o \
      ! -f inputs/gmp-$GMP_VER-linux64-utils.zip ];
 then
   echo
-  echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******"
+  echo "****** Starting Utilities Component of Linux Bundle (1/6 for Linux) ******"
   echo
./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml
@@ -138,7 +139,7 @@ then
   #cp -a result/utils-linux-res.yml inputs/
 else
   echo
-  echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/5 for Linux) ******"
+  echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/6 for Linux) ******"
   echo
   # We might have built the utilities in the past but maybe the links are
   # pointing to the wrong version. Refresh them.
@@ -160,7 +161,7 @@ if [ ! -f inputs/tor-linux32-gbuilt.zip -o \
      ! -f inputs/tor-linux64-gbuilt.zip ];
 then
   echo
-  echo "****** Starting Tor Component of Linux Bundle (2/5 for Linux) ******"
+  echo "****** Starting Tor Component of Linux Bundle (2/6 for Linux) ******"
   echo
./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor=$TOR_TAG $DESCRIPTOR_DIR/linux/gitian-tor.yml
@@ -175,7 +176,7 @@ then
   #cp -a result/tor-linux-res.yml inputs/
 else
   echo
-  echo "****** SKIPPING already built Tor Component of Linux Bundle (2/5 for Linux) ******"
+  echo "****** SKIPPING already built Tor Component of Linux Bundle (2/6 for Linux) ******"
   echo
 fi
@@ -184,7 +185,7 @@ if [ ! -f inputs/tor-browser-linux32-gbuilt.zip -o \
      ! -f inputs/tor-browser-linux64-gbuilt.zip ];
 then
   echo
-  echo "****** Starting TorBrowser Component of Linux Bundle (3/5 for Linux) ******"
+  echo "****** Starting TorBrowser Component of Linux Bundle (3/6 for Linux) ******"
   echo
./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml
@@ -200,7 +201,7 @@ then
   #cp -a result/torbrowser-linux-res.yml inputs/
 else
   echo
-  echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/5 for Linux) ******"
+  echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/6 for Linux) ******"
   echo
 fi
@@ -208,7 +209,7 @@ if [ ! -f inputs/pluggable-transports-linux32-gbuilt.zip -o \
      ! -f inputs/pluggable-transports-linux64-gbuilt.zip ];
 then
   echo
-  echo "****** Starting Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******"
+  echo "****** Starting Pluggable Transports Component of Linux Bundle (4/6 for Linux) ******"
   echo
./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG,goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml
@@ -222,14 +223,35 @@ then
   #cp -a result/pluggable-transports-linux-res.yml inputs/
 else
   echo
-  echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******"
+  echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/6 for Linux) ******"
+  echo
+fi
+
+if [ ! -f inputs/sandbox-linux32.zip -o \
+     ! -f inputs/sandbox-linux64.zip ];
+then
+  echo
+  echo "****** Starting Sandbox Component of Linux Bundle (5/6 for Linux) ******"
+  echo
+
+  ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit errors=$ERRORS_TAG,gb=$GB_TAG,sandbox=$SANDBOX_TAG $DESCRIPTOR_DIR/linux/gitian-sandbox.yml
+  if [ $? -ne 0 ];
+  then
+    exit 1
+  fi
+
+  cp -a build/out/sandbox-linux*.zip inputs/
+  #cp -a result/sandbox-linux-res.yml inputs/
+else
+  echo
+  echo "****** SKIPPING already built Sandbox Component of Linux Bundle (5/6 for Linux) ******"
   echo
 fi
if [ ! -f inputs/bundle-linux.gbuilt ];
 then
   echo
-  echo "****** Starting Bundling+Localization of Linux Bundle (5/5 for Linux) ******"
+  echo "****** Starting Bundling+Localization of Linux Bundle (6/6 for Linux) ******"
   echo
cd $WRAPPER_DIR && ./record-inputs.sh $VERSIONS_FILE && cd $GITIAN_DIR
@@ -246,10 +268,12 @@ then
   cp -a build/out/*.mar $WRAPPER_DIR/$TORBROWSER_BUILDDIR/ || exit 1
   cp -a inputs/mar-tools-linux*.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/ || exit 1
   cp -a inputs/*debug.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/ || exit 1
+  cp -a inputs/sandbox-linux32.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/sandbox-linux32-${SANDBOX_ORIG_TAG}.zip || exit 1
+  cp -a inputs/sandbox-linux64.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/sandbox-linux64-${SANDBOX_ORIG_TAG}.zip || exit 1
   touch inputs/bundle-linux.gbuilt
 else
   echo
-  echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (5/5 for Linux) ******"
+  echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (6/6 for Linux) ******"
   echo
 fi
diff --git a/gitian/versions.nightly b/gitian/versions.nightly
index f487cb5..304284d 100755
--- a/gitian/versions.nightly
+++ b/gitian/versions.nightly
@@ -48,6 +48,9 @@ GO_X_CRYPTO_TAG=master
 GO_X_NET_TAG=master
 OBFS4_TAG=master
 NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2
+ERRORS_TAG=master
+GB_TAG=master
+SANDBOX_TAG=master
GITIAN_TAG=tor-browser-builder-4

    