commit 1913aee57276738ad65bfc34d3177f375b8e0d90 Author: Peter Palfrader <peter@palfrader.org> Date: Wed Jul 10 18:56:59 2013 +0200 fetch-inputs: implement proper gpg checking and partial script cleanup --- gitian/fetch-inputs.sh | 269 +++++++++++++++++++++++------------------------- gitian/gpg/OPENSSL.gpg | Bin 0 -> 4316 bytes 2 files changed, 126 insertions(+), 143 deletions(-) diff --git a/gitian/fetch-inputs.sh b/gitian/fetch-inputs.sh index 21728e5..cd8b4cd 100755 --- a/gitian/fetch-inputs.sh +++ b/gitian/fetch-inputs.sh @@ -3,90 +3,143 @@ # fetch-inputs.sh - Fetch our inputs from the source mirror # -. ./versions - +MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/ +set -e +set -u umask 0022 -export WRAPPER_DIR=$PWD +if ! [ -e ./versions ]; then + echo >&2 "Error: ./versions file does not exist" + exit 1 +fi + +. ./versions + +WRAPPER_DIR=$(dirname "$0") +WRAPPER_DIR=$(readlink -f "$WRAPPER_DIR") -if [ -z "$1" ]; then - INPUTS_DIR=$PWD/../../gitian-builder/inputs +if [ "$#" -gt 1 ]; then + echo >&2 "Usage: $0 [<inputsdir>]" + exit 1 +elif [ "$#" = 1 ]; then + INPUTS_DIR="$1" else - INPUTS_DIR=$1 + INPUTS_DIR="$PWD/../../gitian-builder/inputs" fi -if [ -n $INPUTS_DIR -a ! -d $INPUTS_DIR ]; -then - mkdir $INPUTS_DIR -fi +mkdir -p "$INPUTS_DIR" +cd "$INPUTS_DIR" -if [ -n $INPUTS_DIR -a -d $INPUTS_DIR ]; then - cd $INPUTS_DIR -fi -MIRROR_URL=https://people.torproject.org/~mikeperry/mirrors/sources/ +############################################################################## +CLEANUP=$(tempfile) +trap "bash '$CLEANUP'; rm -f '$CLEANUP'" EXIT -gpg --import $WRAPPER_DIR/gpg/* +verify() { + local file="$1"; shift + local keyring="$1"; shift -# Get package files from mirror -for i in OPENSSL TOOLCHAIN4 OSXSDK # OBFSPROXY -do - PACKAGE=${i}"_PACKAGE" - URL=${MIRROR_URL}${!PACKAGE} - wget -N ${URL} #>& /dev/null - if [ $? -ne 0 ]; then - echo "$i url ${URL} is broken!" - mv ${!PACKAGE} ${!PACKAGE}".removed" + local f + for f in "$file" "$file.asc" "$keyring"; do + if ! [ -e "$f" ]; then + echo >&2 "Error: Required file $f does not exist."; exit 1 + fi + done + + local tmpfile=$(tempfile) + echo "rm -f '$tmpfile'" >> "$CLEANUP" + local gpghome=$(mktemp -d) + echo "rm -rf '$gpghome'" >> "$CLEANUP" + exec 3> "$tmpfile" + + GNUPGHOME="$gpghome" gpg --no-options --no-default-keyring --trust-model=always --keyring="$keyring" --status-fd=3 --verify "$file.asc" "$file" >/dev/null 2>&1 + if grep -q '^\[GNUPG:\] GOODSIG ' "$tmpfile"; then + return 0 + else + return 1 + fi +} + +get() { + local file="$1"; shift + local url="$1"; shift + + if ! wget -N "$url" >& /dev/null; then + echo >&2 "Error: Cannot download $url" + mv "${file}" "${file}.DLFAILED" exit 1 fi -done +} + +update_git() { + local dir="$1"; shift + local url="$1"; shift + local tag="${1:-}" + + if [ -d "$dir/.git" ]; + then + (cd "$dir" && git fetch origin && git fetch --tags origin) + else + if ! git clone "$url"; then + echo >&2 "Error: Cloning $url failed" + exit 1 + fi + fi + + if [ -n "$tag" ]; then + (cd "$dir" && git checkout "$tag") + fi +} + +############################################################################## +# Get package files from mirror # Get+verify sigs that exist -# XXX: This doesn't cover everything. See #8525 for i in OPENSSL # OBFSPROXY do - PACKAGE=${i}"_PACKAGE" - URL=${MIRROR_URL}${!PACKAGE} - if [ ! -f ${!PACKAGE}".asc" ]; then - wget -N ${URL}".asc" >& /dev/null - if [ $? -ne 0 ]; then - echo "$i GPG sig url ${URL} is broken!" - mv ${!PACKAGE} ${!PACKAGE}".nogpg" - exit 1 - fi - fi - gpg ${!PACKAGE}".asc" >& /dev/null - if [ $? -ne 0 ]; then - echo "$i GPG signature is broken for ${URL}" - mv ${!PACKAGE} ${!PACKAGE}".badgpg" + PACKAGE="${i}_PACKAGE" + URL="${MIRROR_URL}${!PACKAGE}" + get "${!PACKAGE}" "$URL" + get "${!PACKAGE}.asc" "$URL.asc" + + if ! verify "${!PACKAGE}" "$WRAPPER_DIR/gpg/$i.gpg"; then + echo "$i: GPG signature is broken for ${URL}" + mv "${!PACKAGE}" "${!PACKAGE}.badgpg" exit 1 fi done +# XXX: This doesn't cover everything. See #8525 +for i in TOOLCHAIN4 OSXSDK +do + PACKAGE="${i}_PACKAGE" + URL="${MIRROR_URL}${!PACKAGE}" + get "${!PACKAGE}" "${MIRROR_URL}${!PACKAGE}" + echo >&2 "Warning, not verifying signature for $i" +done + # Verify packages with weak or no signatures via multipath downloads # (OpenSSL is signed with MD5, and OSXSDK is not signed at all) mkdir -p verify cd verify for i in OPENSSL OSXSDK do - URL=${i}"_URL" - PACKAGE=${i}"_PACKAGE" - wget -N --no-remove-listing ${!URL} >& /dev/null - if [ $? -ne 0 ]; then + URL="${i}_URL" + PACKAGE="${i}_PACKAGE" + if ! wget -N --no-remove-listing "${!URL}"; then echo "$i url ${!URL} is broken!" - mv ${!PACKAGE} ${!PACKAGE}".removed" + mv "${!PACKAGE}" "${!PACKAGE}.removed" exit 1 fi done # XXX: Google won't allow wget -N.. We need to re-download the whole # TOOLCHAIN4 each time :/ -rm -f $TOOLCHAIN4_PACKAGE -wget $TOOLCHAIN4_URL +rm -f "$TOOLCHAIN4_PACKAGE" +wget "$TOOLCHAIN4_URL" for i in OPENSSL OSXSDK TOOLCHAIN4 do - PACKAGE=${i}"_PACKAGE" - diff ${!PACKAGE} ../${!PACKAGE} - if [ $? -ne 0 ]; then + PACKAGE="${i}_PACKAGE" + if ! diff "${!PACKAGE}" "../${!PACKAGE}"; then echo "Package ${!PACKAGE} differs from our mirror's version!" exit 1 fi @@ -102,7 +155,7 @@ if [ ! -f mingw-w64-svn-snapshot-r5830.zip ]; then svn co -r 5830 https://mingw-w64.svn.sourceforge.net/svnroot/mingw-w64/trunk mingw-w64-svn || exit 1 # XXX: Path - ZIPOPTS="-x*/.svn/*" $WRAPPER_DIR/build-helpers/dzip.sh mingw-w64-svn-snapshot-r5830.zip mingw-w64-svn + ZIPOPTS="-x*/.svn/*" "$WRAPPER_DIR/build-helpers/dzip.sh" mingw-w64-svn-snapshot-r5830.zip mingw-w64-svn fi mkdir -p linux-langpacks @@ -112,25 +165,26 @@ mkdir -p mac-langpacks for i in $BUNDLE_LOCALES do cd linux-langpacks - wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/l... + wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/l..." cd .. cd win32-langpacks - wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/w... + wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/w..." cd .. cd mac-langpacks - wget -N https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/m... + wget -N "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$FIREFOX_LANG_VER/m..." cd .. done -$WRAPPER_DIR/build-helpers/dzip.sh win32-langpacks.zip win32-langpacks -$WRAPPER_DIR/build-helpers/dzip.sh linux-langpacks.zip linux-langpacks -$WRAPPER_DIR/build-helpers/dzip.sh mac-langpacks.zip mac-langpacks +"$WRAPPER_DIR/build-helpers/dzip.sh" win32-langpacks.zip win32-langpacks +"$WRAPPER_DIR/build-helpers/dzip.sh" linux-langpacks.zip linux-langpacks +"$WRAPPER_DIR/build-helpers/dzip.sh" mac-langpacks.zip mac-langpacks -ln -sf $NOSCRIPT_PACKAGE noscript@noscript.net.xpi -ln -sf $PDFJS_PACKAGE uriloader@pdf.js.xpi -ln -sf $OPENSSL_PACKAGE openssl.tar.gz +ln -sf "$NOSCRIPT_PACKAGE" noscript@noscript.net.xpi +ln -sf "$PDFJS_PACKAGE" uriloader@pdf.js.xpi +ln -sf "$OPENSSL_PACKAGE" openssl.tar.gz # Fetch latest gitian-builder itself +# XXX - this is broken if a non-standard inputs dir is selected using the command line flag. cd .. git remote set-url origin https://git.torproject.org/builders/gitian-builder.git git fetch origin @@ -138,89 +192,18 @@ git fetch --tags origin git checkout tor-browser-builder-2 cd inputs -if [ -d tbb-windows-installer/.git ]; -then - cd tbb-windows-installer - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://github.com/moba/tbb-windows-installer.git || exit 1 -fi - -if [ -d zlib/.git ]; -then - cd zlib - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://github.com/madler/zlib.git || exit 1 -fi - -if [ -d libevent/.git ]; -then - cd libevent - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://github.com/libevent/libevent.git || exit 1 -fi - -if [ -d tor-launcher/.git ]; -then - cd tor-launcher - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://git.torproject.org/tor-launcher.git || exit 1 -fi - -if [ -d tor/.git ]; -then - cd tor - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://git.torproject.org/tor.git || exit 1 -fi - -if [ -d torbutton/.git ]; -then - cd torbutton - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://git.torproject.org/torbutton.git || exit 1 -fi - -if [ -d https-everywhere/.git ]; -then - cd https-everywhere - git fetch origin - git fetch --tags origin - cd .. -else - git clone https://git.torproject.org/https-everywhere.git || exit 1 -fi - -if [ -d tor-browser/.git ]; -then - cd tor-browser - git fetch origin - git fetch --tags origin - git checkout $TORBROWSER_TAG - cd .. -else - git clone https://git.torproject.org/tor-browser.git || exit 1 - cd tor-browser - git checkout $TORBROWSER_TAG - cd .. -fi +while read dir url tag; do + update_git "$dir" "$url" "$tag" +done << EOF +tbb-windows-installer https://github.com/moba/tbb-windows-installer.git +zlib https://github.com/madler/zlib.git +libevent https://github.com/libevent/libevent.git +tor-launcher https://git.torproject.org/tor-launcher.git +tor https://git.torproject.org/tor.git +torbutton https://git.torproject.org/torbutton.git +https-everywhere https://git.torproject.org/https-everywhere.git +tor-browser https://git.torproject.org/tor-browser.git $TORBROWSER_TAG +EOF exit 0 diff --git a/gitian/gpg/OPENSSL.gpg b/gitian/gpg/OPENSSL.gpg new file mode 100644 index 0000000..1b282b7 Binary files /dev/null and b/gitian/gpg/OPENSSL.gpg differ