commit 522cee6cb9c83be66cc2a6b5bbfefdb3c2bc3217 Author: Georg Koppen <gk@torproject.org> Date: Tue Dec 6 21:20:53 2016 +0000 Bug 20352: Adding sandbox to our Gitian build On Linux we start building Yawning's sandbox code while producing the Linux Tor Browser bundles. For that to work properly we need to use Debian Jessie images and install some packages from backports as not all dependencies are available in Wheezy. --- gitian/Makefile | 6 ++- gitian/descriptors/linux/gitian-sandbox.yml | 77 +++++++++++++++++++++++++++++ gitian/fetch-inputs.sh | 3 ++ gitian/make-vms.sh | 22 ++++++++- gitian/mkbundle-linux.sh | 44 +++++++++++++---- gitian/versions.nightly | 3 ++ 6 files changed, 141 insertions(+), 14 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index a4a757f..5694cbf 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -101,8 +101,7 @@ prep-alpha: ./check-prerequisites.sh $(TORSOCKS) ./fetch-inputs.sh ../../gitian-builder/inputs/ versions.alpha - -clean: clean-utils clean-tor clean-browser clean-pt clean-bundle +clean: clean-utils clean-tor clean-browser clean-pt clean-sandbox clean-bundle rm -f ../../gitian-builder/inputs/*.yml rm -f ../../gitian-builder/inputs/bundle.inputs rm -f ../../gitian-builder/inputs/versions* @@ -124,6 +123,9 @@ clean-browser: clean-bundle clean-pt: clean-bundle rm -f ../../gitian-builder/inputs/pluggable-transports* +clean-sandbox: clean-bundle + rm -f ../../gitian-builder/inputs/sandbox-linux* + clean-bundle: rm -f ../../gitian-builder/inputs/bundle-* diff --git a/gitian/descriptors/linux/gitian-sandbox.yml b/gitian/descriptors/linux/gitian-sandbox.yml new file mode 100644 index 0000000..0f4f78c --- /dev/null +++ b/gitian/descriptors/linux/gitian-sandbox.yml @@ -0,0 +1,77 @@ +--- +name: "sandbox-linux" +distro: "debian" +suites: +- "jessie" +architectures: +- "i386" +- "amd64" +packages: +- "unzip" +- "zip" +- "hardening-wrapper" +# Needed for the sandboxing code +- "libx11-dev" +- "pkg-config" +- "libgtk-3-dev" +backports_packages: +- "libseccomp-dev" +- "libseccomp2" +reference_datetime: "2000-01-01 00:00:00" +remotes: +- "url": "https://github.com/pkg/error" + "dir": "errors" +- "url": "https://github.com/constabulary/gb" + "dir": "gb" +- "url": "https://git.schwanenlied.me/yawning/sandboxed-tor-browser" + "dir": "sandbox" +files: +- "go-linux32-utils.zip" +- "go-linux64-utils.zip" +- "dzip.sh" +script: | + INSTDIR="$HOME/install" + mkdir $INSTDIR/sandbox + export REFERENCE_DATETIME + export TZ=UTC + export LC_ALL=C + umask 0022 + + # Config options for hardening-wrapper for the stub + export DEB_BUILD_HARDENING=1 + export DEB_BUILD_HARDENING_STACKPROTECTOR=1 + export DEB_BUILD_HARDENING_FORTIFY=1 + export DEB_BUILD_HARDENING_FORMAT=1 + export DEB_BUILD_HARDENING_PIE=1 + + unzip -d $INSTDIR go-linux$GBUILD_BITS-utils.zip + export GOROOT="$INSTDIR/go" + export GOPATH="$HOME/go" + export PATH="$PATH:$GOROOT/bin:$GOPATH/bin" + + # Building errors + cd errors + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/pkg/" + ln -sf "$PWD" "$GOPATH/src/github.com/pkg/errors" + go install github.com/pkg/errors + cd .. + + # Building gb + cd gb + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/constabulary/" + ln -sf "$PWD" "$GOPATH/src/github.com/constabulary/gb" + go install github.com/constabulary/gb/cmd/gb + cd .. + + # XXX: Use own C compiler + # Building the sandbox + cd sandbox + make + cp bin/sandboxed-tor-browser $INSTDIR/sandbox + cd .. + + cd $INSTDIR + ~/build/dzip.sh sandbox-linux$GBUILD_BITS.zip sandbox/ + cp sandbox-linux$GBUILD_BITS.zip $OUTDIR/ diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index bf60ca3..b3d5c26 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -293,6 +293,9 @@ goxcrypto https://go.googlesource.com/crypto $GO_X_CRYPTO_TAG goxnet https://go.googlesource.com/net $GO_X_NET_TAG obfs4 https://git.torproject.org/pluggable-transports/obfs4.git $OBFS4_TAG noto-fonts https://github.com/googlei18n/noto-fonts $NOTOFONTS_TAG +errors https://github.com/pkg/errors $ERRORS_TAG +gb https://github.com/constabulary/gb $GB_TAG +sandbox https://git.schwanenlied.me/yawning/sandboxed-tor-browser $SANDBOX_TAG EOF # HTTPS-Everywhere is special, too. We need to initialize the git submodules and diff --git a/gitian/make-vms.sh b/gitian/make-vms.sh index 089865b..7edd2b4 100755 --- a/gitian/make-vms.sh +++ b/gitian/make-vms.sh @@ -32,7 +32,7 @@ build_and_test_vm() { then export LXC_SUITE=$dist export LXC_ARCH=$arch - if [ "$dist" = "wheezy" ]; + if [ "$dist" = "wheezy" -o "$dist" = "jessie" ]; then export DISTRO=debian ./bin/make-base-vm --distro debian --suite $dist --lxc --arch $arch @@ -41,7 +41,7 @@ build_and_test_vm() { ./bin/make-base-vm --suite $dist --lxc --arch $arch fi else - if [ "$dist" = "wheezy" ]; + if [ "$dist" = "wheezy" -o "$dist" = "jessie" ]; then export DISTRO=debian ./bin/make-base-vm --distro debian --suite $dist --arch $arch @@ -89,6 +89,24 @@ do echo done +while ! build_and_test_vm jessie i386 32 +do + stop-target 32 jessie + rm ./base-jessie-amd64* + echo + echo "Jessie i386 VM build failed... Trying again" + echo +done + +while ! build_and_test_vm jessie amd64 64 +do + stop-target 64 jessie + rm ./base-jessie-amd64* + echo + echo "Jessie amd64 VM build failed... Trying again" + echo +done + while ! build_and_test_vm precise i386 32 do stop-target 32 precise diff --git a/gitian/mkbundle-linux.sh b/gitian/mkbundle-linux.sh index f8aaec1..430b666 100755 --- a/gitian/mkbundle-linux.sh +++ b/gitian/mkbundle-linux.sh @@ -78,6 +78,7 @@ die_msg() { # the utils both if we verify tags and if we don't. LIBEVENT_TAG_ORIG=$LIBEVENT_TAG +SANDBOX_ORIG_TAG=$SANDBOX_TAG if [ "z$VERIFY_TAGS" = "z1" ]; then @@ -112,7 +113,7 @@ if [ ! -f inputs/binutils-$BINUTILS_VER-linux32-utils.zip -o \ ! -f inputs/gmp-$GMP_VER-linux64-utils.zip ]; then echo - echo "****** Starting Utilities Component of Linux Bundle (1/5 for Linux) ******" + echo "****** Starting Utilities Component of Linux Bundle (1/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/linux/gitian-utils.yml @@ -138,7 +139,7 @@ then #cp -a result/utils-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/5 for Linux) ******" + echo "****** SKIPPING already built Utilities Component of Linux Bundle (1/6 for Linux) ******" echo # We might have built the utilities in the past but maybe the links are # pointing to the wrong version. Refresh them. @@ -160,7 +161,7 @@ if [ ! -f inputs/tor-linux32-gbuilt.zip -o \ ! -f inputs/tor-linux64-gbuilt.zip ]; then echo - echo "****** Starting Tor Component of Linux Bundle (2/5 for Linux) ******" + echo "****** Starting Tor Component of Linux Bundle (2/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor=$TOR_TAG $DESCRIPTOR_DIR/linux/gitian-tor.yml @@ -175,7 +176,7 @@ then #cp -a result/tor-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Tor Component of Linux Bundle (2/5 for Linux) ******" + echo "****** SKIPPING already built Tor Component of Linux Bundle (2/6 for Linux) ******" echo fi @@ -184,7 +185,7 @@ if [ ! -f inputs/tor-browser-linux32-gbuilt.zip -o \ ! -f inputs/tor-browser-linux64-gbuilt.zip ]; then echo - echo "****** Starting TorBrowser Component of Linux Bundle (3/5 for Linux) ******" + echo "****** Starting TorBrowser Component of Linux Bundle (3/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG,faketime=$FAKETIME_TAG $DESCRIPTOR_DIR/linux/gitian-firefox.yml @@ -200,7 +201,7 @@ then #cp -a result/torbrowser-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/5 for Linux) ******" + echo "****** SKIPPING already built TorBrowser Component of Linux Bundle (3/6 for Linux) ******" echo fi @@ -208,7 +209,7 @@ if [ ! -f inputs/pluggable-transports-linux32-gbuilt.zip -o \ ! -f inputs/pluggable-transports-linux64-gbuilt.zip ]; then echo - echo "****** Starting Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******" + echo "****** Starting Pluggable Transports Component of Linux Bundle (4/6 for Linux) ******" echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG,goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG $DESCRIPTOR_DIR/linux/gitian-pluggable-transports.yml @@ -222,14 +223,35 @@ then #cp -a result/pluggable-transports-linux-res.yml inputs/ else echo - echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/5 for Linux) ******" + echo "****** SKIPPING already built Pluggable Transports Component of Linux Bundle (4/6 for Linux) ******" + echo +fi + +if [ ! -f inputs/sandbox-linux32.zip -o \ + ! -f inputs/sandbox-linux64.zip ]; +then + echo + echo "****** Starting Sandbox Component of Linux Bundle (5/6 for Linux) ******" + echo + + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit errors=$ERRORS_TAG,gb=$GB_TAG,sandbox=$SANDBOX_TAG $DESCRIPTOR_DIR/linux/gitian-sandbox.yml + if [ $? -ne 0 ]; + then + exit 1 + fi + + cp -a build/out/sandbox-linux*.zip inputs/ + #cp -a result/sandbox-linux-res.yml inputs/ +else + echo + echo "****** SKIPPING already built Sandbox Component of Linux Bundle (5/6 for Linux) ******" echo fi if [ ! -f inputs/bundle-linux.gbuilt ]; then echo - echo "****** Starting Bundling+Localization of Linux Bundle (5/5 for Linux) ******" + echo "****** Starting Bundling+Localization of Linux Bundle (6/6 for Linux) ******" echo cd $WRAPPER_DIR && ./record-inputs.sh $VERSIONS_FILE && cd $GITIAN_DIR @@ -246,10 +268,12 @@ then cp -a build/out/*.mar $WRAPPER_DIR/$TORBROWSER_BUILDDIR/ || exit 1 cp -a inputs/mar-tools-linux*.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/ || exit 1 cp -a inputs/*debug.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/ || exit 1 + cp -a inputs/sandbox-linux32.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/sandbox-linux32-${SANDBOX_ORIG_TAG}.zip || exit 1 + cp -a inputs/sandbox-linux64.zip $WRAPPER_DIR/$TORBROWSER_BUILDDIR/sandbox-linux64-${SANDBOX_ORIG_TAG}.zip || exit 1 touch inputs/bundle-linux.gbuilt else echo - echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (5/5 for Linux) ******" + echo "****** SKIPPING already built Bundling+Localization of Linux Bundle (6/6 for Linux) ******" echo fi diff --git a/gitian/versions.nightly b/gitian/versions.nightly index f487cb5..304284d 100755 --- a/gitian/versions.nightly +++ b/gitian/versions.nightly @@ -48,6 +48,9 @@ GO_X_CRYPTO_TAG=master GO_X_NET_TAG=master OBFS4_TAG=master NOTOFONTS_TAG=720e34851382ee3c1ef024d8dffb68ffbfb234c2 +ERRORS_TAG=master +GB_TAG=master +SANDBOX_TAG=master GITIAN_TAG=tor-browser-builder-4