[tbb-commits] [tor-browser-bundle/master] Bug 13379: Sign our MAR files.

brade at torproject.org brade at torproject.org
Wed Dec 17 22:07:58 UTC 2014


commit d852329ac0979d8005e9e8bdd9b3f8a049fc2db4
Author: Kathy Brade <brade at pearlcrescent.com>
Date:   Wed Dec 17 16:57:28 2014 -0500

    Bug 13379: Sign our MAR files.
    
    In the Linux mar-tools package, include certutil, signmar, and the required
      NSS and NSPR libraries.
    Change update_responses to select the action to be done based on the
      program name, thereby separating the incremental MAR file generation (done
      via a new 'gen_incrementals' symlink) from the htdocs generation.  Also added
      an 'update_responses' make target.  Thanks to boklm for help with this.
    Add a series of 'signmar' make targets which run a new signmars.sh script,
    which does the following:
      1. Prompts for an NSS password (the script assumes that an nssdb directory
         exists that contains the signing key/cert).
      2. Moves original (unsigned) MAR files to a TORBROWSER_VERSION-unsigned/
         directory.
      3. Signs each of the MAR files, placing the signed files in the
         TORBROWSER_VERSION/ directory.
      You may set NSS_DB_DIR and/or NSS_CERTNAME before running signmars.sh; the
      defaults are ./nssdb and marsigner
---
 gitian/Makefile                             |   15 +++
 gitian/descriptors/linux/gitian-firefox.yml |   11 +++
 gitian/signmars.sh                          |  135 +++++++++++++++++++++++++++
 tools/update-responses/gen_incrementals     |    1 +
 tools/update-responses/update_responses     |   34 +++++--
 5 files changed, 189 insertions(+), 7 deletions(-)

diff --git a/gitian/Makefile b/gitian/Makefile
index 2c17656..9613db6 100644
--- a/gitian/Makefile
+++ b/gitian/Makefile
@@ -33,6 +33,21 @@ build-beta:
 	./hash-bundles.sh versions.beta
 
 incrementals:
+	../tools/update-responses/gen_incrementals
+
+signmars:
+	./signmars.sh versions
+
+signmars-alpha:
+	./signmars.sh versions.alpha
+
+signmars-beta:
+	./signmars.sh versions.beta
+
+signmars-nightly:
+	./signmars.sh versions.nightly
+
+update_responses:
 	../tools/update-responses/update_responses
 
 hash:
diff --git a/gitian/descriptors/linux/gitian-firefox.yml b/gitian/descriptors/linux/gitian-firefox.yml
index 5f96b6e..a1abf2e 100644
--- a/gitian/descriptors/linux/gitian-firefox.yml
+++ b/gitian/descriptors/linux/gitian-firefox.yml
@@ -136,12 +136,23 @@ script: |
   rm -f $INSTDIR/Browser/*.chk
   #
   # Make MAR-based update tools available for use during the bundle phase.
+  # Note that mar and mbsdiff are standalone tools, compiled for the build
+  # host's architecture.  We also include signmar, certutil, and the libraries
+  # they require; these utilities and libraries are built for the target
+  # architecture.
   MARTOOLS=~/build/mar-tools
   mkdir -p $MARTOOLS
   cp -p config/createprecomplete.py $MARTOOLS/
   cp -p tools/update-packaging/*.sh $MARTOOLS/
   cp -p obj-*/dist/host/bin/mar $MARTOOLS/
   cp -p obj-*/dist/host/bin/mbsdiff $MARTOOLS/
+  cp -p obj-*/modules/libmar/tool/signmar $MARTOOLS/
+  cp -p obj-*/security/nss/cmd/certutil/certutil $MARTOOLS/
+  NSS_LIBS="libfreebl3.so libmozsqlite3.so libnss3.so libnssdbm3.so libnssutil3.so libsmime3.so libsoftokn3.so libssl3.so"
+  NSPR_LIBS="libnspr4.so libplc4.so libplds4.so"
+  for LIB in $NSS_LIBS $NSPR_LIBS; do
+    cp -p obj-*/dist/bin/$LIB $MARTOOLS/
+  done
   cd ~/build
   zip -r mar-tools-linux${GBUILD_BITS}.zip mar-tools
   cp -p mar-tools-linux${GBUILD_BITS}.zip $OUTDIR/
diff --git a/gitian/signmars.sh b/gitian/signmars.sh
new file mode 100755
index 0000000..fdb531e
--- /dev/null
+++ b/gitian/signmars.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+#
+#
+# You may set NSS_DB_DIR and/or NSS_CERTNAME before invoking this script.
+
+set -e
+set -u
+
+WRAPPER_DIR=$(dirname "$0")
+WRAPPER_DIR=$(readlink -e "$WRAPPER_DIR")
+
+if [ -z "${NSS_DB_DIR+x}" ]; then
+  NSS_DB_DIR=$WRAPPER_DIR/nssdb
+fi
+
+if [ -z "${NSS_CERTNAME+x}" ]; then
+  NSS_CERTNAME=marsigner
+fi
+
+# Incorporate definitions from the versions file.
+if [ -z "$1" ]; then
+  VERSIONS_FILE=$WRAPPER_DIR/versions
+else
+  VERSIONS_FILE=$1
+fi
+
+if ! [ -e $VERSIONS_FILE ]; then
+  echo >&2 "Error: $VERSIONS_FILE file does not exist"
+  exit 1
+fi
+
+. $VERSIONS_FILE
+
+export LC_ALL=C
+
+# Check some prerequisites.
+if [ ! -r "$NSS_DB_DIR/cert8.db" ]; then
+  >&2 echo "Please create and populate the $NSS_DB_DIR directory"
+  exit 2
+fi
+
+OSNAME=""
+ARCH="$(uname -s)-$(uname -m)"
+case $ARCH in
+  Linux-x86_64)
+    OSNAME="linux64"
+    ;;
+  Linux-i*86)
+    OSNAME="linux32"
+    ;;
+  *)
+    >&2 echo "Unsupported architecture $ARCH"
+    exit 2
+esac
+
+# Extract the MAR tools so we can use the signmar program.
+MARTOOLS_TMP_DIR=$(mktemp -d)
+trap "rm -rf $MARTOOLS_TMP_DIR" EXIT
+MARTOOLS_ZIP="$WRAPPER_DIR/../../gitian-builder/inputs/mar-tools-${OSNAME}.zip"
+cd $MARTOOLS_TMP_DIR
+unzip -q "$MARTOOLS_ZIP"
+cd $WRAPPER_DIR
+export PATH="$MARTOOLS_TMP_DIR/mar-tools:$PATH"
+if [ -z "${LD_LIBRARY_PATH+x}" ]; then
+  export LD_LIBRARY_PATH="$MARTOOLS_TMP_DIR/mar-tools"
+else
+  export LD_LIBRARY_PATH="$MARTOOLS_TMP_DIR/mar-tools:$LD_LIBRARY_PATH"
+fi
+
+# Prompt for the NSS password.
+# TODO: Test that the entered NSS password is correct.  But how?  Unfortunately,
+# both certutil and signmar keep trying to read a new password when they are
+# given an incorrect one.
+read -s -p "NSS password:" NSSPASS
+echo ""
+
+# Sign each MAR file.
+#
+# Our strategy is to first move all .mar files out of the TORBROWSER_VERSION
+# directory into a TORBROWSER_VERSION-unsigned/ directory.  Details:
+#   If a file has not been signed, we move it to the -unsigned/ directory.
+#   If a file has already been signed and a file with the same name exists in
+#     the -unsigned/ directory, we just delete the signed file.
+#   If a file has already been signed but no corresponding file exists in
+#     the -unsigned/ directory, we report an error and exit.
+#
+# Once the above is done,  the -unsigned/ directory contains a set of .mar
+# files that need to be signed, so we go ahead and sign them one-by-one.
+SIGNED_DIR="$WRAPPER_DIR/$TORBROWSER_VERSION"
+UNSIGNED_DIR="$WRAPPER_DIR/${TORBROWSER_VERSION}-unsigned"
+mkdir -p "$UNSIGNED_DIR"
+cd "$SIGNED_DIR"
+for marfile in *.mar; do
+  if [ ! -f "$marfile" ]; then
+    continue;
+  fi
+
+  # First, we check for an existing signature.  The signmar -T output will
+  # include a line like "Signature block found with N signatures".
+  SIGINFO_PREFIX="Signature block found with "
+  SIGINFO=$(signmar -T "$marfile" | grep "^${SIGINFO_PREFIX}")
+  SIGCOUNT=0
+  if [ ! -z "$SIGINFO" ]; then
+    SIGCOUNT=$(echo $SIGINFO | sed -e "s/${SIGINFO_PREFIX}//" -e 's/\([0-9]*\).*$/\1/')
+  fi
+  if [ $SIGCOUNT -eq 0 ]; then
+    # No signature; move this .mar file to the -unsigned/ directory.
+    mv "$marfile" "$UNSIGNED_DIR/"
+  elif [ -e "$UNSIGNED_DIR/$marfile" ]; then
+    # We have an -unsigned/ copy; discard this file.
+    rm "$marfile"
+  else
+    >&2 echo "Error: $SIGNED_DIR/$marfile is already signed but $UNSIGNED_DIR/$marfile is missing"
+    # TODO: Try to remove the existing signature(s) from marfile?
+    exit 1
+  fi
+done
+
+# Use signmar to sign each .mar file that is now in the -unsigned directory.
+TMPMAR="$SIGNED_DIR/tmp.mar"
+trap "rm -f $TMPMAR" EXIT
+cd "$UNSIGNED_DIR"
+COUNT=0
+for marfile in *.mar; do
+  if [ ! -f "$marfile" ]; then
+    continue;
+  fi
+  echo "$NSSPASS" | signmar -d "$NSS_DB_DIR" -n "$NSS_CERTNAME" -s \
+      "$marfile" "$TMPMAR"
+  mv "$TMPMAR" "$SIGNED_DIR/$marfile"
+  COUNT=$((COUNT + 1))
+done
+
+echo "The $COUNT MAR files located in $SIGNED_DIR/ have been signed."
+echo "The unsigned (original) MAR files are in $UNSIGNED_DIR/"
diff --git a/tools/update-responses/gen_incrementals b/tools/update-responses/gen_incrementals
new file mode 120000
index 0000000..3766925
--- /dev/null
+++ b/tools/update-responses/gen_incrementals
@@ -0,0 +1 @@
+update_responses
\ No newline at end of file
diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses
index b28b575..5b85037 100755
--- a/tools/update-responses/update_responses
+++ b/tools/update-responses/update_responses
@@ -1,6 +1,7 @@
 #!/usr/bin/perl -w
 
 use strict;
+use English;
 use FindBin;
 use YAML qw(LoadFile);
 use File::Slurp;
@@ -13,6 +14,7 @@ use File::Which;
 use POSIX qw(setlocale LC_ALL);
 use IO::CaptureOutput qw(capture_exec);
 use Parallel::ForkManager;
+use File::Basename;
 
 # Set umask and locale to provide a consistent environment for MAR file
 # generation, etc.
@@ -158,7 +160,7 @@ sub create_incremental_mar {
     $pm->finish;
 }
 
-sub create_missing_incremental_mars {
+sub create_missing_incremental_mars_for_version {
     my ($config, $version) = @_;
     my $pm = Parallel::ForkManager->new(get_nbprocs);
     $pm->run_on_finish(sub { $_[2]->(@_) });
@@ -178,6 +180,14 @@ sub create_missing_incremental_mars {
     $pm->wait_all_children;
 }
 
+sub create_all_missing_incremental_mars {
+    my ($config) = @_;
+    foreach my $version (values %{$config->{channels}}) {
+        get_version_files($config, $version);
+        create_missing_incremental_mars_for_version($config, $version);
+    }
+}
+
 sub get_config {
     my ($config, $version, $os, $name) = @_;
     return $config->{versions}{$version}{$os}{$name}
@@ -224,7 +234,6 @@ sub write_responses {
     my ($config) = @_;
     foreach my $version (values %{$config->{channels}}) {
         get_version_files($config, $version);
-        create_missing_incremental_mars($config, $version);
         my $files = $config->{versions}{$version}{files};
         my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {};
         foreach my $old_os (keys %$migrate_archs) {
@@ -309,8 +318,19 @@ sub extract_martools {
     $ENV{PATH} .= ":$martools_tmpdir/mar-tools";
 }
 
-extract_martools;
-check_deps;
-write_responses($config);
-write_htaccess($config);
-clean_htdocs;
+my %actions = (
+    update_responses => sub {
+        write_responses(@_);
+        write_htaccess(@_);
+        clean_htdocs;
+    },
+    gen_incrementals => sub {
+        extract_martools;
+        check_deps;
+        create_all_missing_incremental_mars(@_);
+    },
+);
+
+my $action = fileparse($PROGRAM_NAME);
+exit_error "Unknown action $action" unless $actions{$action};
+$actions{$action}->($config);



More information about the tbb-commits mailing list