[tor-commits] [gitian-builder/tor-browser-builder-3] Bug 15578: Switch over to Wheezy for Linux builds
gk at torproject.org
gk at torproject.org
Fri Jan 22 10:24:24 UTC 2016
commit 15d166d65d006f564bf3c7dbb8780ed0649352ba
Author: Georg Koppen <gk at torproject.org>
Date: Thu Nov 26 06:34:42 2015 +0000
Bug 15578: Switch over to Wheezy for Linux builds
Support for Debian guest VMs was developed by Joseph Bisch. This
commit is largely a backport of this feature adapted to our needs
allowing us to get rid of Ubuntu Lucid which is EOL for a while now.
---
README.md | 22 +++++++++++-
bin/gbuild | 3 ++
bin/make-base-vm | 81 +++++++++++++++++++++++++++++++++++++-------
libexec/copy-from-target | 2 +-
libexec/copy-to-target | 2 +-
libexec/on-target | 2 +-
target-bin/bootstrap-fixup | 18 ++++++++--
7 files changed, 111 insertions(+), 19 deletions(-)
diff --git a/README.md b/README.md
index 2f07c6a..c78e231 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,22 @@ This performs a build inside a VM, with deterministic inputs and outputs. If th
Install virtualbox from http://www.virtualbox.org, and make sure `VBoxManage` is in your `$PATH`.
+## Debian Guests
+
+Gitian now supports Debian guests in addition to Ubuntu guests. Note that this doesn't mean you can allow the builders to choose to use either Debian or Ubuntu guests. The person creating the Gitian descriptor will need to choose a particular distro and suite for the guest and all builders must use that particular distro and suite, otherwise the software won't reproduce for everyone.
+
+The official vmbuilder only includes support for Ubuntu guests, so you need to install [Joseph Bisch's fork of vmbuilder](https://github.com/josephbisch/vmbuilder), which adds a Debian plugin.
+
+To create a Debian guest:
+
+ bin/make-base-vm --distro debian --suite jessie
+
+There is currently no support for LXC Debian guests. There is just KVM support. LXC support for Debian guests is planned to be added soon.
+
+Only Debian Jessie guests have been tested with Gitian. Debian Jessie is the current stable release of Debian at this time. If you have success (or trouble) with other versions of Debian, please let us know.
+
+If you are creating a Gitian descriptor, you can now specify a distro. If no distro is provided, the default is to assume Ubuntu. Since Ubuntu is assumed, older Gitian descriptors that don't specify a distro will still work as they always have.
+
## Create the base VM for use in further builds
**NOTE:** requires `sudo`, please review the script
@@ -88,6 +104,10 @@ If you have everything set-up properly, you should be able to:
PATH=$PATH:$(pwd)/libexec
make-clean-vm --suite lucid --arch i386
+ # on-target needs $DISTRO to be set to debian if using a Debian guest
+ # (when running gbuild, $DISTRO is set based on the descriptor, so this line isn't needed)
+ DiSTRO=debian
+
# For LXC:
LXC_ARCH=i386 LXC_SUITE=lucid on-target ls -la
@@ -128,7 +148,7 @@ After you've merged everybody's signatures, verify them:
* Log files are captured to the _var_ directory
* You can run the utilities in libexec by running `PATH="libexec:$PATH"`
* To start the target VM run `start-target 32 lucid-i386` or `start-target 64 lucid-amd64`
-* To ssh into the target run `on-target` or `on-target -u root`
+* To ssh into the target run `on-target` (after setting $DISTRO to debian if using a Debian guest) or `on-target -u root`
* On the target, the _build_ directory contains the code as it is compiled and _install_ contains intermediate libraries
* By convention, the script in `<package>.yml` starts with any environment setup you would need to manually compile things on the target
diff --git a/bin/gbuild b/bin/gbuild
index d256986..0171ccf 100755
--- a/bin/gbuild
+++ b/bin/gbuild
@@ -180,10 +180,13 @@ FileUtils.mkdir_p(result_dir)
package_name = build_desc["name"] or raise "must supply name"
package_name = sanitize(package_name, "package name")
+distro = build_desc["distro"] || "ubuntu"
suites = build_desc["suites"] or raise "must supply suites"
archs = build_desc["architectures"] or raise "must supply architectures"
reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
+ENV['DISTRO'] = distro
+
desc_sum = `sha256sum #{build_desc_file}`
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
in_sums << desc_sum
diff --git a/bin/make-base-vm b/bin/make-base-vm
index 66a3704..1aa8eac 100755
--- a/bin/make-base-vm
+++ b/bin/make-base-vm
@@ -1,10 +1,10 @@
#!/bin/sh
set -e
+DISTRO=ubuntu
SUITE=lucid
ARCH=amd64
-MIRROR=http://${MIRROR_HOST:-127.0.0.1}:3142/archive.ubuntu.com/ubuntu
-SECURITY_MIRROR=http://${MIRROR_HOST:-127.0.0.1}:3142/security.ubuntu.com/ubuntu
+MIRROR_BASE=http://${MIRROR_HOST:-127.0.0.1}:3142
LXC=0
VBOX=0
@@ -13,11 +13,12 @@ usage() {
echo "Make a base client."
echo
cat << EOF
- --help display this help and exit
- --suite U build suite U instead of lucid
- --arch A build architecture A (e.g. i386) instead of amd64
- --lxc use lxc instead of kvm
- --vbox use VirtualBox instead of kvm
+ --help display this help and exit
+ --distro D build distro D (e.g. debian) instead of ubuntu
+ --suite U build suite U instead of lucid
+ --arch A build architecture A (e.g. i386) instead of amd64
+ --lxc use lxc instead of kvm
+ --vbox use VirtualBox instead of kvm
EOF
}
@@ -28,6 +29,10 @@ if [ $# != 0 ] ; then
usage
exit 0
;;
+ --distro|-d)
+ DISTRO="$2"
+ shift 2
+ ;;
--suite|-s)
SUITE="$2"
shift 2
@@ -55,6 +60,19 @@ if [ $# != 0 ] ; then
done
fi
+if [ $DISTRO = "debian" -a $LXC = "1" ]; then
+ echo "There is no support for Debian guests using LXC currently. Please use KVM or another distro for now."
+ exit 1
+fi
+
+if [ $DISTRO = "ubuntu" ]; then
+ MIRROR=$MIRROR_BASE/archive.ubuntu.com/ubuntu
+ SECURITY_MIRROR=$MIRROR_BASE/security.ubuntu.com/ubuntu
+elif [ $DISTRO = "debian" ]; then
+ MIRROR=$MIRROR_BASE/ftp.debian.org/debian
+ SECURITY_MIRROR=$MIRROR_BASE/security.debian.org/
+fi
+
mkdir -p var
if [ ! -e var/id_dsa ]; then
@@ -68,7 +86,47 @@ if [ $ARCH = "amd64" -a $SUITE = "hardy" ]; then
FLAVOUR=server
fi
-addpkg=openssh-server,pciutils,build-essential,git-core,subversion,lxc
+if [ $DISTRO = "debian" -a $ARCH = "amd64" ]; then
+ FLAVOUR=amd64
+elif [ $DISTRO = "debian" -a $ARCH = "i386" -a \($SUITE = "squeeze" -o $SUITE = "lenny" -o $SUITE = "etch" -o $SUITE = "sarge" -o $SUITE = "woody" -o $SUITE = "potato" -o $SUITE = "slink" -o $SUITE = "hamm" -o $SUITE = "bo" -o $SUITE = "rex" -o $SUITE = "buzz"\) ]; then
+ FLAVOUR=686
+elif [ $DISTRO = "debian" ]; then
+ FLAVOUR=686-pae
+fi
+
+LOCALE_PKG=language-pack-en
+if [ $DISTRO = "debian" ]; then
+ LOCALE_PKG=locales
+fi
+
+addpkg=pciutils,build-essential,git-core,subversion,$LOCALE_PKG,wget,lsb-release
+
+if [ $DISTRO = "ubuntu" ]; then
+ # Need comma at end to work around an issue with apt for Debian <= Wheezy regarding empty strings
+ #
+ # If we left the comma down below when adding KERNEL_PKG to addpkg, the fact that KERNEL_PKG is undefined
+ # if DISTRO is debian would result in two commas in a row (,,), which is interpreted by apt-get as the
+ # package with the name empty string (""). This triggers a bug with apt versions < 1.0.3. So by adding the
+ # comma to the end of KERNEL_PKG, we are including that comma if the distro is ubuntu (and therefore we do
+ # have a kernel package that needs to be installed). If KERNEL_PKG is not set (i.e. we have Debian as the
+ # distro), then we don't add that extra comma and therefore, we don't end up with two commas in a row.
+ #
+ # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=744940
+ # http://anonscm.debian.org/cgit/apt/apt.git/commit/?h=1.0.3&id=d99854cac4065bc7b337815fb2116269d58dab73
+ KERNEL_PKG=linux-image-generic,
+fi
+
+GRUB_PKG=grub
+if [ $DISTRO = "ubuntu" ]; then
+ GRUB_PKG=grub-pc
+fi
+
+if [ $LXC = "1" ]; then
+ addpkg=$addpkg,lxc
+else
+ # Lack of comma after KERNEL_PKG is not a typo
+ addpkg=$addpkg,${KERNEL_PKG}${GRUB_PKG},openssh-server
+fi
# Remove cron to work around vmbuilder issue when umounting /dev on target
removepkg=cron
@@ -84,8 +142,8 @@ if [ $VBOX = "1" ]; then
vagrant ssh "$NAME" -c "sudo mkdir -p /root/.ssh && sudo chmod 700 /root/.ssh"
vagrant ssh "$NAME" -c "sudo sh -c 'cat >> /root/.ssh/authorized_keys'" < var/id_dsa.pub
- vagrant ssh "$NAME" -c "sudo -u ubuntu mkdir -p /home/ubuntu/.ssh && sudo -u ubuntu chmod 700 /home/ubuntu/.ssh"
- vagrant ssh "$NAME" -c "sudo sh -c 'cat >> /home/ubuntu/.ssh/authorized_keys'" < var/id_dsa.pub
+ vagrant ssh "$NAME" -c "sudo -u $DISTRO mkdir -p /home/$DISTRO/.ssh && sudo -u $DISTRO chmod 700 /home/$DISTRO/.ssh"
+ vagrant ssh "$NAME" -c "sudo sh -c 'cat >> /home/$DISTRO/.ssh/authorized_keys'" < var/id_dsa.pub
VBoxManage snapshot "Gitian-$NAME" take "Gitian-Clean"
vagrant suspend "$NAME"
@@ -99,7 +157,7 @@ if [ -e $OUT.qcow2 ]; then
fi
rm -rf $OUT
-sudo vmbuilder kvm ubuntu --rootsize 15360 --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --security-mirror=$SECURITY_MIRROR --dest=$OUT --flavour=$FLAVOUR --firstboot=`pwd`/target-bin/bootstrap-fixup
+sudo vmbuilder kvm $DISTRO --rootsize 15360 --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --security-mirror=$SECURITY_MIRROR --dest=$OUT --flavour=$FLAVOUR --firstboot=`pwd`/target-bin/bootstrap-fixup
mv $OUT/*.qcow2 $OUT.qcow2
rm -rf $OUT
@@ -114,4 +172,3 @@ if [ $LXC = "1" ]; then
rm -f $OUT.raw
# bootstrap-fixup is done in libexec/make-clean-vm
fi
-
diff --git a/libexec/copy-from-target b/libexec/copy-from-target
index f82da66..5372bb9 100755
--- a/libexec/copy-from-target
+++ b/libexec/copy-from-target
@@ -2,7 +2,7 @@
. gconfig
-TUSER=ubuntu
+TUSER=${DISTRO:-ubuntu}
QUIET_FLAG=
usage() {
diff --git a/libexec/copy-to-target b/libexec/copy-to-target
index f9d900e..df45812 100755
--- a/libexec/copy-to-target
+++ b/libexec/copy-to-target
@@ -2,7 +2,7 @@
. gconfig
-TUSER=ubuntu
+TUSER=${DISTRO:-ubuntu}
QUIET_FLAG=
usage() {
diff --git a/libexec/on-target b/libexec/on-target
index ae653bd..fe56fee 100755
--- a/libexec/on-target
+++ b/libexec/on-target
@@ -4,7 +4,7 @@ set -e
. gconfig
-TUSER=ubuntu
+TUSER=${DISTRO:-ubuntu}
usage() {
echo "Usage: ${0##*/} [OPTION]... <command>"
diff --git a/target-bin/bootstrap-fixup b/target-bin/bootstrap-fixup
index ccbb977..d371072 100755
--- a/target-bin/bootstrap-fixup
+++ b/target-bin/bootstrap-fixup
@@ -2,7 +2,8 @@
set -e
-. /etc/lsb-release
+DISTRIB_NAME=`lsb_release -is`
+DISTRIB_CODENAME=`lsb_release -cs`
ip=`hostname --all-ip-addresses | cut -d ' ' -f1 | cut -d. -f1-3`
@@ -14,5 +15,16 @@ else
MIRROR_HOST_ON_GUEST=${MIRROR_HOST_ON_GUEST:-10.0.2.2}
fi
-echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME main universe" > $1/etc/apt/sources.list
-echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME-updates main universe" >> $1/etc/apt/sources.list
+if [ $DISTRIB_NAME = "Ubuntu" ]; then
+ echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME main universe" > $1/etc/apt/sources.list
+ echo "deb http://$MIRROR_HOST_ON_GUEST:3142/security.ubuntu.com/ubuntu $DISTRIB_CODENAME-security main universe" >> $1/etc/apt/sources.list
+ echo "deb http://$MIRROR_HOST_ON_GUEST:3142/archive.ubuntu.com/ubuntu $DISTRIB_CODENAME-updates main universe" >> $1/etc/apt/sources.list
+elif [ $DISTRIB_NAME = "Debian" ]; then
+ echo "deb http://$MIRROR_HOST_ON_GUEST:3142/ftp.debian.org/debian $DISTRIB_CODENAME main" > $1/etc/apt/sources.list
+ echo "deb http://$MIRROR_HOST_ON_GUEST:3142/security.debian.org/ $DISTRIB_CODENAME/updates main" >> $1/etc/apt/sources.list
+ echo "deb http://$MIRROR_HOST_ON_GUEST:3142/ftp.debian.org/debian $DISTRIB_CODENAME-updates main" >> $1/etc/apt/sources.list
+ # grub-legacy conflicts grub-pc dependencies
+ # No grub-legacy on Ubuntu, just on Debian
+ # Work around bcron-run conflict due to cron being removed
+ apt-get purge -y grub-legacy bcron-run &> /dev/null
+fi
More information about the tor-commits
mailing list