[tor-commits] [tor-browser-bundle/master] fetch-inputs: implement proper gpg checking and partial script cleanup
mikeperry at torproject.org
mikeperry at torproject.org
Wed Jul 10 18:00:12 UTC 2013
commit 1913aee57276738ad65bfc34d3177f375b8e0d90
Author: Peter Palfrader <peter at palfrader.org>
Date: Wed Jul 10 18:56:59 2013 +0200
fetch-inputs: implement proper gpg checking and partial script cleanup
---
gitian/fetch-inputs.sh | 269 +++++++++++++++++++++++-------------------------
gitian/gpg/OPENSSL.gpg | Bin 0 -> 4316 bytes
2 files changed, 126 insertions(+), 143 deletions(-)
diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh
index 21728e5..cd8b4cd 100755
--- a/gitian/fetch-inputs.sh
+++ b/gitian/fetch-inputs.sh
@@ -3,90 +3,143 @@
# fetch-inputs.sh - Fetch our inputs from the source mirror
#
-. ./versions
-
+MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/
+set -e
+set -u
umask 0022
-export WRAPPER_DIR=$PWD
+if ! [ -e ./versions ]; then
+ echo >&2 "Error: ./versions file does not exist"
+ exit 1
+fi
+
+. ./versions
+
+WRAPPER_DIR=$(dirname "$0")
+WRAPPER_DIR=$(readlink -f "$WRAPPER_DIR")
-if [ -z "$1" ]; then
- INPUTS_DIR=$PWD/../../gitian-builder/inputs
+if [ "$#" -gt 1 ]; then
+ echo >&2 "Usage: $0 [<inputsdir>]"
+ exit 1
+elif [ "$#" = 1 ]; then
+ INPUTS_DIR="$1"
else
- INPUTS_DIR=$1
+ INPUTS_DIR="$PWD/../../gitian-builder/inputs"
fi
-if [ -n $INPUTS_DIR -a ! -d $INPUTS_DIR ];
-then
- mkdir $INPUTS_DIR
-fi
+mkdir -p "$INPUTS_DIR"
+cd "$INPUTS_DIR"
-if [ -n $INPUTS_DIR -a -d $INPUTS_DIR ]; then
- cd $INPUTS_DIR
-fi
-MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/
+##############################################################################
+CLEANUP=$(tempfile)
+trap "bash '$CLEANUP'; rm -f '$CLEANUP'" EXIT
-gpg --import $WRAPPER_DIR/gpg/*
+verify() {
+ local file="$1"; shift
+ local keyring="$1"; shift
-# Get package files from mirror
-for i in OPENSSL TOOLCHAIN4 OSXSDK # OBFSPROXY
-do
- PACKAGE=${i}"_PACKAGE"
- URL=${MIRROR_URL}${!PACKAGE}
- wget -N ${URL} #>& /dev/null
- if [ $? -ne 0 ]; then
- echo "$i url ${URL} is broken!"
- mv ${!PACKAGE} ${!PACKAGE}".removed"
+ local f
+ for f in "$file" "$file.asc" "$keyring"; do
+ if ! [ -e "$f" ]; then
+ echo >&2 "Error: Required file $f does not exist."; exit 1
+ fi
+ done
+
+ local tmpfile=$(tempfile)
+ echo "rm -f '$tmpfile'" >> "$CLEANUP"
+ local gpghome=$(mktemp -d)
+ echo "rm -rf '$gpghome'" >> "$CLEANUP"
+ exec 3> "$tmpfile"
+
+ GNUPGHOME="$gpghome" gpg --no-options --no-default-keyring --trust-model=always --keyring="$keyring" --status-fd=3 --verify "$file.asc" "$file" >/dev/null 2>&1
+ if grep -q '^\[GNUPG:\] GOODSIG ' "$tmpfile"; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+get() {
+ local file="$1"; shift
+ local url="$1"; shift
+
+ if ! wget -N "$url" >& /dev/null; then
+ echo >&2 "Error: Cannot download $url"
+ mv "${file}" "${file}.DLFAILED"
exit 1
fi
-done
+}
+
+update_git() {
+ local dir="$1"; shift
+ local url="$1"; shift
+ local tag="${1:-}"
+
+ if [ -d "$dir/.git" ];
+ then
+ (cd "$dir" && git fetch origin && git fetch --tags origin)
+ else
+ if ! git clone "$url"; then
+ echo >&2 "Error: Cloning $url failed"
+ exit 1
+ fi
+ fi
+
+ if [ -n "$tag" ]; then
+ (cd "$dir" && git checkout "$tag")
+ fi
+}
+
+##############################################################################
+# Get package files from mirror
# Get+verify sigs that exist
-# XXX: This doesn't cover everything. See #8525
for i in OPENSSL # OBFSPROXY
do
- PACKAGE=${i}"_PACKAGE"
- URL=${MIRROR_URL}${!PACKAGE}
- if [ ! -f ${!PACKAGE}".asc" ]; then
- wget -N ${URL}".asc" >& /dev/null
- if [ $? -ne 0 ]; then
- echo "$i GPG sig url ${URL} is broken!"
- mv ${!PACKAGE} ${!PACKAGE}".nogpg"
- exit 1
- fi
- fi
- gpg ${!PACKAGE}".asc" >& /dev/null
- if [ $? -ne 0 ]; then
- echo "$i GPG signature is broken for ${URL}"
- mv ${!PACKAGE} ${!PACKAGE}".badgpg"
+ PACKAGE="${i}_PACKAGE"
+ URL="${MIRROR_URL}${!PACKAGE}"
+ get "${!PACKAGE}" "$URL"
+ get "${!PACKAGE}.asc" "$URL.asc"
+
+ if ! verify "${!PACKAGE}" "$WRAPPER_DIR/gpg/$i.gpg"; then
+ echo "$i: GPG signature is broken for ${URL}"
+ mv "${!PACKAGE}" "${!PACKAGE}.badgpg"
exit 1
fi
done
+# XXX: This doesn't cover everything. See #8525
+for i in TOOLCHAIN4 OSXSDK
+do
+ PACKAGE="${i}_PACKAGE"
+ URL="${MIRROR_URL}${!PACKAGE}"
+ get "${!PACKAGE}" "${MIRROR_URL}${!PACKAGE}"
+ echo >&2 "Warning, not verifying signature for $i"
+done
+
# Verify packages with weak or no signatures via multipath downloads
# (OpenSSL is signed with MD5, and OSXSDK is not signed at all)
mkdir -p verify
cd verify
for i in OPENSSL OSXSDK
do
- URL=${i}"_URL"
- PACKAGE=${i}"_PACKAGE"
- wget -N --no-remove-listing ${!URL} >& /dev/null
- if [ $? -ne 0 ]; then
+ URL="${i}_URL"
+ PACKAGE="${i}_PACKAGE"
+ if ! wget -N --no-remove-listing "${!URL}"; then
echo "$i url ${!URL} is broken!"
- mv ${!PACKAGE} ${!PACKAGE}".removed"
+ mv "${!PACKAGE}" "${!PACKAGE}.removed"
exit 1
fi
done
# XXX: Google won't allow wget -N.. We need to re-download the whole
# TOOLCHAIN4 each time :/
-rm -f $TOOLCHAIN4_PACKAGE
-wget $TOOLCHAIN4_URL
+rm -f "$TOOLCHAIN4_PACKAGE"
+wget "$TOOLCHAIN4_URL"
for i in OPENSSL OSXSDK TOOLCHAIN4
do
- PACKAGE=${i}"_PACKAGE"
- diff ${!PACKAGE} ../${!PACKAGE}
- if [ $? -ne 0 ]; then
+ PACKAGE="${i}_PACKAGE"
+ if ! diff "${!PACKAGE}" "../${!PACKAGE}"; then
echo "Package ${!PACKAGE} differs from our mirror's version!"
exit 1
fi
@@ -102,7 +155,7 @@ if [ ! -f mingw-w64-svn-snapshot-r5830.zip ];
then
svn co -r 5830 https://mingw-w64.svn.sourceforge.net/svnroot/mingw-w64/trunk mingw-w64-svn || exit 1
# XXX: Path
- ZIPOPTS="-x*/.svn/*" $WRAPPER_DIR/build-helpers/dzip.sh mingw-w64-svn-snapshot-r5830.zip mingw-w64-svn
+ ZIPOPTS="-x*/.svn/*" "$WRAPPER_DIR/build-helpers/dzip.sh" mingw-w64-svn-snapshot-r5830.zip mingw-w64-svn
fi
mkdir -p linux-langpacks
@@ -112,25 +165,26 @@ mkdir -p mac-langpacks
for i in $BUNDLE_LOCALES
do
cd linux-langpacks
- wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/linux-i686/xpi/$i.xpi
+ wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/linux-i686/xpi/$i.xpi"
cd ..
cd win32-langpacks
- wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/win32/xpi/$i.xpi
+ wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/win32/xpi/$i.xpi"
cd ..
cd mac-langpacks
- wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/mac/xpi/$i.xpi
+ wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/mac/xpi/$i.xpi"
cd ..
done
-$WRAPPER_DIR/build-helpers/dzip.sh win32-langpacks.zip win32-langpacks
-$WRAPPER_DIR/build-helpers/dzip.sh linux-langpacks.zip linux-langpacks
-$WRAPPER_DIR/build-helpers/dzip.sh mac-langpacks.zip mac-langpacks
+"$WRAPPER_DIR/build-helpers/dzip.sh" win32-langpacks.zip win32-langpacks
+"$WRAPPER_DIR/build-helpers/dzip.sh" linux-langpacks.zip linux-langpacks
+"$WRAPPER_DIR/build-helpers/dzip.sh" mac-langpacks.zip mac-langpacks
-ln -sf $NOSCRIPT_PACKAGE noscript at noscript.net.xpi
-ln -sf $PDFJS_PACKAGE uriloader at pdf.js.xpi
-ln -sf $OPENSSL_PACKAGE openssl.tar.gz
+ln -sf "$NOSCRIPT_PACKAGE" noscript at noscript.net.xpi
+ln -sf "$PDFJS_PACKAGE" uriloader at pdf.js.xpi
+ln -sf "$OPENSSL_PACKAGE" openssl.tar.gz
# Fetch latest gitian-builder itself
+# XXX - this is broken if a non-standard inputs dir is selected using the command line flag.
cd ..
git remote set-url origin https://git.torproject.org/builders/gitian-builder.git
git fetch origin
@@ -138,89 +192,18 @@ git fetch --tags origin
git checkout tor-browser-builder-2
cd inputs
-if [ -d tbb-windows-installer/.git ];
-then
- cd tbb-windows-installer
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://github.com/moba/tbb-windows-installer.git || exit 1
-fi
-
-if [ -d zlib/.git ];
-then
- cd zlib
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://github.com/madler/zlib.git || exit 1
-fi
-
-if [ -d libevent/.git ];
-then
- cd libevent
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://github.com/libevent/libevent.git || exit 1
-fi
-
-if [ -d tor-launcher/.git ];
-then
- cd tor-launcher
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://git.torproject.org/tor-launcher.git || exit 1
-fi
-
-if [ -d tor/.git ];
-then
- cd tor
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://git.torproject.org/tor.git || exit 1
-fi
-
-if [ -d torbutton/.git ];
-then
- cd torbutton
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://git.torproject.org/torbutton.git || exit 1
-fi
-
-if [ -d https-everywhere/.git ];
-then
- cd https-everywhere
- git fetch origin
- git fetch --tags origin
- cd ..
-else
- git clone https://git.torproject.org/https-everywhere.git || exit 1
-fi
-
-if [ -d tor-browser/.git ];
-then
- cd tor-browser
- git fetch origin
- git fetch --tags origin
- git checkout $TORBROWSER_TAG
- cd ..
-else
- git clone https://git.torproject.org/tor-browser.git || exit 1
- cd tor-browser
- git checkout $TORBROWSER_TAG
- cd ..
-fi
+while read dir url tag; do
+ update_git "$dir" "$url" "$tag"
+done << EOF
+tbb-windows-installer https://github.com/moba/tbb-windows-installer.git
+zlib https://github.com/madler/zlib.git
+libevent https://github.com/libevent/libevent.git
+tor-launcher https://git.torproject.org/tor-launcher.git
+tor https://git.torproject.org/tor.git
+torbutton https://git.torproject.org/torbutton.git
+https-everywhere https://git.torproject.org/https-everywhere.git
+tor-browser https://git.torproject.org/tor-browser.git $TORBROWSER_TAG
+EOF
exit 0
diff --git a/gitian/gpg/OPENSSL.gpg b/gitian/gpg/OPENSSL.gpg
new file mode 100644
index 0000000..1b282b7
Binary files /dev/null and b/gitian/gpg/OPENSSL.gpg differ
More information about the tor-commits
mailing list