commit d852329ac0979d8005e9e8bdd9b3f8a049fc2db4 Author: Kathy Brade <brade@pearlcrescent.com> Date: Wed Dec 17 16:57:28 2014 -0500 Bug 13379: Sign our MAR files. In the Linux mar-tools package, include certutil, signmar, and the required NSS and NSPR libraries. Change update_responses to select the action to be done based on the program name, thereby separating the incremental MAR file generation (done via a new 'gen_incrementals' symlink) from the htdocs generation. Also added an 'update_responses' make target. Thanks to boklm for help with this. Add a series of 'signmar' make targets which run a new signmars.sh script, which does the following: 1. Prompts for an NSS password (the script assumes that an nssdb directory exists that contains the signing key/cert). 2. Moves original (unsigned) MAR files to a TORBROWSER_VERSION-unsigned/ directory. 3. Signs each of the MAR files, placing the signed files in the TORBROWSER_VERSION/ directory. You may set NSS_DB_DIR and/or NSS_CERTNAME before running signmars.sh; the defaults are ./nssdb and marsigner --- gitian/Makefile | 15 +++ gitian/descriptors/linux/gitian-firefox.yml | 11 +++ gitian/signmars.sh | 135 +++++++++++++++++++++++++++ tools/update-responses/gen_incrementals | 1 + tools/update-responses/update_responses | 34 +++++-- 5 files changed, 189 insertions(+), 7 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index 2c17656..9613db6 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -33,6 +33,21 @@ build-beta: ./hash-bundles.sh versions.beta incrementals: + ../tools/update-responses/gen_incrementals + +signmars: + ./signmars.sh versions + +signmars-alpha: + ./signmars.sh versions.alpha + +signmars-beta: + ./signmars.sh versions.beta + +signmars-nightly: + ./signmars.sh versions.nightly + +update_responses: ../tools/update-responses/update_responses hash: diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml index 5f96b6e..a1abf2e 100644 --- a/gitian/descriptors/linux/gitian-firefox.yml +++ b/gitian/descriptors/linux/gitian-firefox.yml @@ -136,12 +136,23 @@ script: | rm -f $INSTDIR/Browser/*.chk # # Make MAR-based update tools available for use during the bundle phase. + # Note that mar and mbsdiff are standalone tools, compiled for the build + # host's architecture. We also include signmar, certutil, and the libraries + # they require; these utilities and libraries are built for the target + # architecture. MARTOOLS=~/build/mar-tools mkdir -p $MARTOOLS cp -p config/createprecomplete.py $MARTOOLS/ cp -p tools/update-packaging/*.sh $MARTOOLS/ cp -p obj-*/dist/host/bin/mar $MARTOOLS/ cp -p obj-*/dist/host/bin/mbsdiff $MARTOOLS/ + cp -p obj-*/modules/libmar/tool/signmar $MARTOOLS/ + cp -p obj-*/security/nss/cmd/certutil/certutil $MARTOOLS/ + NSS_LIBS="libfreebl3.so libmozsqlite3.so libnss3.so libnssdbm3.so libnssutil3.so libsmime3.so libsoftokn3.so libssl3.so" + NSPR_LIBS="libnspr4.so libplc4.so libplds4.so" + for LIB in $NSS_LIBS $NSPR_LIBS; do + cp -p obj-*/dist/bin/$LIB $MARTOOLS/ + done cd ~/build zip -r mar-tools-linux${GBUILD_BITS}.zip mar-tools cp -p mar-tools-linux${GBUILD_BITS}.zip $OUTDIR/ diff --git a/gitian/signmars.sh b/gitian/signmars.sh new file mode 100755 index 0000000..fdb531e --- /dev/null +++ b/gitian/signmars.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# +# You may set NSS_DB_DIR and/or NSS_CERTNAME before invoking this script. + +set -e +set -u + +WRAPPER_DIR=$(dirname "$0") +WRAPPER_DIR=$(readlink -e "$WRAPPER_DIR") + +if [ -z "${NSS_DB_DIR+x}" ]; then + NSS_DB_DIR=$WRAPPER_DIR/nssdb +fi + +if [ -z "${NSS_CERTNAME+x}" ]; then + NSS_CERTNAME=marsigner +fi + +# Incorporate definitions from the versions file. +if [ -z "$1" ]; then + VERSIONS_FILE=$WRAPPER_DIR/versions +else + VERSIONS_FILE=$1 +fi + +if ! [ -e $VERSIONS_FILE ]; then + echo >&2 "Error: $VERSIONS_FILE file does not exist" + exit 1 +fi + +. $VERSIONS_FILE + +export LC_ALL=C + +# Check some prerequisites. +if [ ! -r "$NSS_DB_DIR/cert8.db" ]; then + >&2 echo "Please create and populate the $NSS_DB_DIR directory" + exit 2 +fi + +OSNAME="" +ARCH="$(uname -s)-$(uname -m)" +case $ARCH in + Linux-x86_64) + OSNAME="linux64" + ;; + Linux-i*86) + OSNAME="linux32" + ;; + *) + >&2 echo "Unsupported architecture $ARCH" + exit 2 +esac + +# Extract the MAR tools so we can use the signmar program. +MARTOOLS_TMP_DIR=$(mktemp -d) +trap "rm -rf $MARTOOLS_TMP_DIR" EXIT +MARTOOLS_ZIP="$WRAPPER_DIR/../../gitian-builder/inputs/mar-tools-${OSNAME}.zip" +cd $MARTOOLS_TMP_DIR +unzip -q "$MARTOOLS_ZIP" +cd $WRAPPER_DIR +export PATH="$MARTOOLS_TMP_DIR/mar-tools:$PATH" +if [ -z "${LD_LIBRARY_PATH+x}" ]; then + export LD_LIBRARY_PATH="$MARTOOLS_TMP_DIR/mar-tools" +else + export LD_LIBRARY_PATH="$MARTOOLS_TMP_DIR/mar-tools:$LD_LIBRARY_PATH" +fi + +# Prompt for the NSS password. +# TODO: Test that the entered NSS password is correct. But how? Unfortunately, +# both certutil and signmar keep trying to read a new password when they are +# given an incorrect one. +read -s -p "NSS password:" NSSPASS +echo "" + +# Sign each MAR file. +# +# Our strategy is to first move all .mar files out of the TORBROWSER_VERSION +# directory into a TORBROWSER_VERSION-unsigned/ directory. Details: +# If a file has not been signed, we move it to the -unsigned/ directory. +# If a file has already been signed and a file with the same name exists in +# the -unsigned/ directory, we just delete the signed file. +# If a file has already been signed but no corresponding file exists in +# the -unsigned/ directory, we report an error and exit. +# +# Once the above is done, the -unsigned/ directory contains a set of .mar +# files that need to be signed, so we go ahead and sign them one-by-one. +SIGNED_DIR="$WRAPPER_DIR/$TORBROWSER_VERSION" +UNSIGNED_DIR="$WRAPPER_DIR/${TORBROWSER_VERSION}-unsigned" +mkdir -p "$UNSIGNED_DIR" +cd "$SIGNED_DIR" +for marfile in *.mar; do + if [ ! -f "$marfile" ]; then + continue; + fi + + # First, we check for an existing signature. The signmar -T output will + # include a line like "Signature block found with N signatures". + SIGINFO_PREFIX="Signature block found with " + SIGINFO=$(signmar -T "$marfile" | grep "^${SIGINFO_PREFIX}") + SIGCOUNT=0 + if [ ! -z "$SIGINFO" ]; then + SIGCOUNT=$(echo $SIGINFO | sed -e "s/${SIGINFO_PREFIX}//" -e 's/\([0-9]*\).*$/\1/') + fi + if [ $SIGCOUNT -eq 0 ]; then + # No signature; move this .mar file to the -unsigned/ directory. + mv "$marfile" "$UNSIGNED_DIR/" + elif [ -e "$UNSIGNED_DIR/$marfile" ]; then + # We have an -unsigned/ copy; discard this file. + rm "$marfile" + else + >&2 echo "Error: $SIGNED_DIR/$marfile is already signed but $UNSIGNED_DIR/$marfile is missing" + # TODO: Try to remove the existing signature(s) from marfile? + exit 1 + fi +done + +# Use signmar to sign each .mar file that is now in the -unsigned directory. +TMPMAR="$SIGNED_DIR/tmp.mar" +trap "rm -f $TMPMAR" EXIT +cd "$UNSIGNED_DIR" +COUNT=0 +for marfile in *.mar; do + if [ ! -f "$marfile" ]; then + continue; + fi + echo "$NSSPASS" | signmar -d "$NSS_DB_DIR" -n "$NSS_CERTNAME" -s \ + "$marfile" "$TMPMAR" + mv "$TMPMAR" "$SIGNED_DIR/$marfile" + COUNT=$((COUNT + 1)) +done + +echo "The $COUNT MAR files located in $SIGNED_DIR/ have been signed." +echo "The unsigned (original) MAR files are in $UNSIGNED_DIR/" diff --git a/tools/update-responses/gen_incrementals b/tools/update-responses/gen_incrementals new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/gen_incrementals @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index b28b575..5b85037 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -1,6 +1,7 @@ #!/usr/bin/perl -w use strict; +use English; use FindBin; use YAML qw(LoadFile); use File::Slurp; @@ -13,6 +14,7 @@ use File::Which; use POSIX qw(setlocale LC_ALL); use IO::CaptureOutput qw(capture_exec); use Parallel::ForkManager; +use File::Basename; # Set umask and locale to provide a consistent environment for MAR file # generation, etc. @@ -158,7 +160,7 @@ sub create_incremental_mar { $pm->finish; } -sub create_missing_incremental_mars { +sub create_missing_incremental_mars_for_version { my ($config, $version) = @_; my $pm = Parallel::ForkManager->new(get_nbprocs); $pm->run_on_finish(sub { $_[2]->(@_) }); @@ -178,6 +180,14 @@ sub create_missing_incremental_mars { $pm->wait_all_children; } +sub create_all_missing_incremental_mars { + my ($config) = @_; + foreach my $version (values %{$config->{channels}}) { + get_version_files($config, $version); + create_missing_incremental_mars_for_version($config, $version); + } +} + sub get_config { my ($config, $version, $os, $name) = @_; return $config->{versions}{$version}{$os}{$name} @@ -224,7 +234,6 @@ sub write_responses { my ($config) = @_; foreach my $version (values %{$config->{channels}}) { get_version_files($config, $version); - create_missing_incremental_mars($config, $version); my $files = $config->{versions}{$version}{files}; my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {}; foreach my $old_os (keys %$migrate_archs) { @@ -309,8 +318,19 @@ sub extract_martools { $ENV{PATH} .= ":$martools_tmpdir/mar-tools"; } -extract_martools; -check_deps; -write_responses($config); -write_htaccess($config); -clean_htdocs; +my %actions = ( + update_responses => sub { + write_responses(@_); + write_htaccess(@_); + clean_htdocs; + }, + gen_incrementals => sub { + extract_martools; + check_deps; + create_all_missing_incremental_mars(@_); + }, +); + +my $action = fileparse($PROGRAM_NAME); +exit_error "Unknown action $action" unless $actions{$action}; +$actions{$action}->($config);