commit e0ef86ec6822abfb17554defaf287e926f3a8206 Author: Georg Koppen <gk@torproject.org> Date: Thu May 15 14:23:43 2014 +0000 Bug 10120: Part 3: Refactor Mac descriptors. We build the tools needed for the Mac descriptors separately and preserve them. This does not include the tools in the Pluggable Transports descriptor. Using our newly built OpenSSL there, e.g., will happen in an other commit. --- gitian/descriptors/mac/gitian-tor.yml | 76 +++++------------- gitian/descriptors/mac/gitian-utils.yml | 70 +++++++++++++++++ gitian/mkbundle-mac.sh | 131 +++++++++++++++++++------------ 3 files changed, 170 insertions(+), 107 deletions(-) diff --git a/gitian/descriptors/mac/gitian-tor.yml b/gitian/descriptors/mac/gitian-tor.yml index 68905e7..b8a761b 100644 --- a/gitian/descriptors/mac/gitian-tor.yml +++ b/gitian/descriptors/mac/gitian-tor.yml @@ -4,7 +4,7 @@ suites: - "precise" architectures: - "i386" -packages: +packages: - "git-core" - "unzip" - "zip" @@ -13,18 +13,12 @@ packages: - "automake" - "faketime" - "libtool" -#- "libssl0.9.8" reference_datetime: "2000-01-01 00:00:00" remotes: - "url": "https://git.torproject.org/tor.git" "dir": "tor" -- "url": "https://github.com/libevent/libevent.git" - "dir": "libevent" -- "url": "https://github.com/madler/zlib.git" - "dir": "zlib" files: - "versions" -- "openssl.tar.gz" - "bug10297.patch" - "bug5018.patch" - "bug9229.patch" @@ -37,63 +31,32 @@ files: - "apple-uni-sdk-10.6_20110407-0.flosoft1_i386.deb" - "multiarch-darwin11-cctools127.2-gcc42-5666.3-llvmgcc42-2336.1-Linux-120724.tar.xz" - "dzip.sh" -#- "apple-x86-gcc_4.2.1~5646.1flosoft2_i386.deb" -#- "apple-x86-odcctools_758.159-0flosoft11_i386.deb" +- "openssl-mac32-utils.zip" +- "libevent-mac32-utils.zip" script: | INSTDIR="$HOME/install" source versions - export LIBRARY_PATH="$INSTDIR/lib" export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 export FAKETIME=$REFERENCE_DATETIME export TZ=UTC export LC_ALL=C umask 0022 # - #export CFLAGS="-isysroot /usr/lib/apple/SDKs/MacOSX10.6.sdk/" - #export CXXFLAGS=$CFLAGS - #export LDFLAGS=$CFLAGS - export PATH="$PATH:/usr/apple-osx/bin/" - export CFLAGS="-I/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/include/ -I/usr/lib/gcc/i686-apple-darwin10/4.2.1/include/ -I. -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -mmacosx-version-min=10.5" - export LDFLAGS="-L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -mmacosx-version-min=10.5" - # mkdir -p $INSTDIR/bin/ mkdir -p $INSTDIR/TorBrowserBundle.app/Tor/ mkdir -p $INSTDIR/TorBrowserBundle.app/Data/Tor/ mkdir -p $OUTDIR/ # sudo dpkg -i *.deb - cd /usr - sudo tar -Jxvf /home/ubuntu/build/multiarch-darwin*tar.xz - cd ~/build - # For OpenSSL - sudo ln -s /usr/apple-osx/bin/apple-osx-gcc /usr/apple-osx/bin/i686-apple-darwin11-cc - # - #cd zlib - #find -type f | xargs touch --date="$REFERENCE_DATETIME" - #make BINARY_PATH=$INSTDIR/zlib/lib INCLUDE_PATH=$INSTDIR/zlib/include LIBRARY_PATH=$INSTDIR/zlib/lib PREFIX=i686-apple-darwin10- $MAKEOPTS SHARED_MODE=1 - #make BINARY_PATH=$INSTDIR/zlib/lib INCLUDE_PATH=$INSTDIR/zlib/include LIBRARY_PATH=$INSTDIR/zlib/lib PREFIX=i686-apple-darwin10- $MAKEOPTS SHARED_MODE=1 install - #cp $INSTDIR/zlib/lib/*.dylib $INSTDIR/Tor/ - #cd .. - # - tar xzf openssl.tar.gz - cd openssl-* - find -type f | xargs touch --date="$REFERENCE_DATETIME" - ./Configure --cross-compile-prefix=i686-apple-darwin11- $CFLAGS darwin-i386-cc --prefix=$INSTDIR/openssl #enable-ec_nistp_64_gcc_128 - make # SHARED_LDFLAGS="-shared -dynamiclib -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/" - make install - #cp $INSTDIR/openssl/lib/*.dylib $INSTDIR/TorBrowserBundle.app/Tor/ - cd .. - # - cd libevent - ./autogen.sh - find -type f | xargs touch --date="$REFERENCE_DATETIME" - ./configure --disable-static --host=i686-apple-darwin11 --prefix=$INSTDIR/libevent - make $MAKEOPTS - make install + tar xaf multiarch-darwin*tar.xz + export PATH="$PATH:$HOME/build/apple-osx/bin/" + unzip -d $INSTDIR openssl-mac32-utils.zip + unzip -d $INSTDIR libevent-mac32-utils.zip cp $INSTDIR/libevent/lib/libevent-*.dylib $INSTDIR/TorBrowserBundle.app/Tor/ LIBEVENT_FILE=`basename $INSTDIR/libevent/lib/libevent-*.dylib` - cd .. # + export CFLAGS="-I/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/include/ -I/usr/lib/gcc/i686-apple-darwin10/4.2.1/include/ -I. -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -mmacosx-version-min=10.5" + export LDFLAGS="-L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -mmacosx-version-min=10.5" cd tor git update-index --refresh -q export GIT_COMMITTER_NAME="nobody" @@ -118,22 +81,19 @@ script: | fi fi mkdir -p $OUTDIR/src - git archive HEAD | tar -x -C $OUTDIR/src + #git archive HEAD | tar -x -C $OUTDIR/src ./autogen.sh find -type f | xargs touch --date="$REFERENCE_DATETIME" - ./configure --enable-static-openssl --disable-asciidoc --host=i686-apple-darwin11 --with-libevent-dir=$INSTDIR/libevent --with-openssl-dir=$INSTDIR/openssl --prefix=$INSTDIR --with-zlib-dir=$INSTDIR/zlib/ + ./configure --enable-static-openssl --disable-asciidoc --host=i686-apple-darwin11 --with-libevent-dir=$INSTDIR/libevent --with-openssl-dir=$INSTDIR/openssl --prefix=$INSTDIR make $MAKEOPTS make install - cd .. - cp $INSTDIR/bin/tor $INSTDIR/TorBrowserBundle.app/Tor/ - cp $INSTDIR/share/tor/geoip $INSTDIR/TorBrowserBundle.app/Data/Tor/ - cd $INSTDIR/TorBrowserBundle.app/Tor/ - i686-apple-darwin11-install_name_tool -change /home/ubuntu/install/libevent/lib/$LIBEVENT_FILE @executable_path/$LIBEVENT_FILE tor + cd $INSTDIR + cp bin/tor TorBrowserBundle.app/Tor/ + cp share/tor/geoip TorBrowserBundle.app/Data/Tor/ + cd TorBrowserBundle.app/Tor/ + i686-apple-darwin11-install_name_tool -change $INSTDIR/libevent/lib/$LIBEVENT_FILE @executable_path/$LIBEVENT_FILE tor # XXX: Geoip files.. # - #cp -a /usr/lib/gcc/i686-apple-darwin10/4.6/libgcc_s_sjlj-1.dll $INSTDIR/Tor/ - #cp -a /usr/lib/gcc/i686-apple-darwin10/4.6/libssp*.dll $INSTDIR/Tor/ - # cd $INSTDIR - ~/build/dzip.sh tor-mac$GBUILD_BITS-gbuilt.zip TorBrowserBundle.app - cp tor-mac$GBUILD_BITS-gbuilt.zip $OUTDIR/ + ~/build/dzip.sh tor-mac32-gbuilt.zip TorBrowserBundle.app + cp tor-mac32-gbuilt.zip $OUTDIR/ diff --git a/gitian/descriptors/mac/gitian-utils.yml b/gitian/descriptors/mac/gitian-utils.yml new file mode 100644 index 0000000..1caf8f0 --- /dev/null +++ b/gitian/descriptors/mac/gitian-utils.yml @@ -0,0 +1,70 @@ +--- +name: "utils-mac" +suites: +- "precise" +architectures: +- "i386" +packages: +- "git-core" +- "unzip" +- "zip" +- "autoconf" +- "autoconf2.13" +- "automake" +- "faketime" +- "libtool" +reference_datetime: "2000-01-01 00:00:00" +remotes: +- "url": "https://github.com/libevent/libevent.git" + "dir": "libevent" +files: +- "apple-uni-sdk-10.6_20110407-0.flosoft1_i386.deb" +- "multiarch-darwin11-cctools127.2-gcc42-5666.3-llvmgcc42-2336.1-Linux-120724.tar.xz" +- "openssl.tar.gz" +- "versions" +- "dzip.sh" +script: | + INSTDIR="$HOME/install" + source versions + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME=$REFERENCE_DATETIME + export TZ=UTC + export LC_ALL=C + umask 0022 + + sudo dpkg -i *.deb + tar xaf multiarch-darwin*tar.xz + export PATH="$PATH:$HOME/build/apple-osx/bin/" + # For OpenSSL + ln -s ~/build/apple-osx/bin/apple-osx-gcc ~/build/apple-osx/bin/i686-apple-darwin11-cc + + export CFLAGS="-I/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/include/ -I/usr/lib/gcc/i686-apple-darwin10/4.2.1/include/ -I. -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -mmacosx-version-min=10.5" + export LDFLAGS="-L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/ -L/usr/lib/apple/SDKs/MacOSX10.6.sdk/usr/lib/system/ -mmacosx-version-min=10.5" + + # Building Libevent + cd libevent + ./autogen.sh + find -type f | xargs touch --date="$REFERENCE_DATETIME" + ./configure --disable-static --host=i686-apple-darwin11 --prefix=$INSTDIR/libevent + make $MAKEOPTS + make install + cd .. + + # Building OpenSSL + tar xzf openssl.tar.gz + cd openssl-* + find -type f | xargs touch --date="$REFERENCE_DATETIME" + # TODO: Add enable-ec_nistp_64_gcc_128 for 64bit OS X. + ./Configure --cross-compile-prefix=i686-apple-darwin11- $CFLAGS darwin-i386-cc --prefix=$INSTDIR/openssl + # Using $MAKEOPTS breaks the build. Might be the issue mentioned on + # http://cblfs.cross-lfs.org/index.php/OpenSSL. + make + make install + cd .. + + # Grabbing the results + cd $INSTDIR + ~/build/dzip.sh openssl-$OPENSSL_VER-mac32-utils.zip openssl + ~/build/dzip.sh libevent-${LIBEVENT_TAG#release-}-mac32-utils.zip libevent + + cp *utils.zip $OUTDIR/ diff --git a/gitian/mkbundle-mac.sh b/gitian/mkbundle-mac.sh index d34002d..fb1d32f 100755 --- a/gitian/mkbundle-mac.sh +++ b/gitian/mkbundle-mac.sh @@ -42,7 +42,7 @@ fi cd $GITIAN_DIR export PATH=$PATH:$PWD/libexec -echo "pref(\"torbrowser.version\", \"$TORBROWSER_VERSION-MacOS\");" > $GITIAN_DIR/inputs/torbrowser.version +echo "pref(\"torbrowser.version\", \"$TORBROWSER_VERSION-MacOS\");" > $GITIAN_DIR/inputs/torbrowser.version echo "$TORBROWSER_VERSION" > $GITIAN_DIR/inputs/bare-version cp -a $WRAPPER_DIR/$VERSIONS_FILE $GITIAN_DIR/inputs/versions @@ -51,7 +51,7 @@ cp $WRAPPER_DIR/patches/* $GITIAN_DIR/inputs/ cd $WRAPPER_DIR/.. rm -f $GITIAN_DIR/inputs/relativelink-src.zip -$WRAPPER_DIR/build-helpers/dzip.sh $GITIAN_DIR/inputs/relativelink-src.zip ./RelativeLink/ +$WRAPPER_DIR/build-helpers/dzip.sh $GITIAN_DIR/inputs/relativelink-src.zip ./RelativeLink/ cd ./Bundle-Data/ rm -f $GITIAN_DIR/inputs/tbb-docs.zip @@ -79,6 +79,11 @@ die_msg() { exit 1 } +# Let's preserve the original $FOO for creating proper symlinks after building +# the utils both if we verify tags and if we don't. + +LIBEVENT_TAG_ORIG=$LIBEVENT_TAG + if [ "z$VERIFY_TAGS" = "z1" ]; then ./verify-tags.sh $GITIAN_DIR/inputs $VERSIONS_FILE || die_msg "You should run 'make prep' to ensure your inputs are up to date" @@ -100,32 +105,64 @@ fi cd $GITIAN_DIR -if [ ! -f $GITIAN_DIR/inputs/tor-mac32-gbuilt.zip ]; +if [ ! -f inputs/openssl-$OPENSSL_VER-mac32-utils.zip -o \ + ! -f inputs/libevent-${LIBEVENT_TAG_ORIG#release-}-mac32-utils.zip ]; +then + echo + echo "****** Starting Utilities Component of Mac Bundle (1/5 for Max) ******" + echo + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libevent=$LIBEVENT_TAG $DESCRIPTOR_DIR/mac/gitian-utils.yml + if [ $? -ne 0 ]; + then + #mv var/build.log ./utils-fail-mac.log.`date +%Y%m%d%H%M%S` + exit 1 + fi + + cd inputs + cp -a ../build/out/*-utils.zip . + ln -sf openssl-$OPENSSL_VER-mac32-utils.zip openssl-mac32-utils.zip + ln -sf libevent-${LIBEVENT_TAG_ORIG#release-}-mac32-utils.zip libevent-mac32-utils.zip + cd .. + #cp -a result/utils-mac-res.yml inputs/ +else + echo + echo "****** SKIPPING already built Utilities Component of Mac Bundle (1/5 for + Mac) ******" + echo + # We might have built the utilities in the past but maybe the links are + # pointing to the wrong version. Refresh them. + cd inputs + ln -sf openssl-$OPENSSL_VER-mac32-utils.zip openssl-mac32-utils.zip + ln -sf libevent-${LIBEVENT_TAG_ORIG#release-}-mac32-utils.zip libevent-mac32-utils.zip + cd .. +fi + +if [ ! -f inputs/tor-mac32-gbuilt.zip ]; then - echo - echo "****** Starting Tor Component of Mac Bundle (1/4 for Mac) ******" - echo + echo + echo "****** Starting Tor Component of Mac Bundle (2/5 for Mac) ******" + echo - ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit zlib=$ZLIB_TAG,libevent=$LIBEVENT_TAG,tor=$TOR_TAG $DESCRIPTOR_DIR/mac/gitian-tor.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor=$TOR_TAG $DESCRIPTOR_DIR/mac/gitian-tor.yml if [ $? -ne 0 ]; then #mv var/build.log ./tor-fail-mac.log.`date +%Y%m%d%H%M%S` exit 1 fi - - cp -a build/out/tor-mac*-gbuilt.zip $GITIAN_DIR/inputs/ - #cp -a result/tor-mac-res.yml $GITIAN_DIR/inputs/ + + cp -a build/out/tor-mac*-gbuilt.zip inputs/ + #cp -a result/tor-mac-res.yml inputs/ else - echo - echo "****** SKIPPING already built Tor Component of Mac Bundle (1/4 for Mac) ******" - echo + echo + echo "****** SKIPPING already built Tor Component of Mac Bundle (2/5 for Mac) ******" + echo fi -if [ ! -f $GITIAN_DIR/inputs/tor-browser-mac32-gbuilt.zip ]; +if [ ! -f inputs/tor-browser-mac32-gbuilt.zip ]; then - echo - echo "****** Starting TorBrowser Component of Mac Bundle (2/4 for Mac) ******" - echo + echo + echo "****** Starting TorBrowser Component of Mac Bundle (3/5 for Mac) ******" + echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit tor-browser=$TORBROWSER_TAG $DESCRIPTOR_DIR/mac/gitian-firefox.yml if [ $? -ne 0 ]; @@ -134,19 +171,19 @@ then exit 1 fi - cp -a build/out/tor-browser-mac*-gbuilt.zip $GITIAN_DIR/inputs/ - #cp -a result/torbrowser-mac-res.yml $GITIAN_DIR/inputs/ + cp -a build/out/tor-browser-mac*-gbuilt.zip inputs/ + #cp -a result/torbrowser-mac-res.yml inputs/ else - echo - echo "****** SKIPPING already built TorBrowser Component of Mac Bundle (2/4 for Mac) ******" - echo + echo + echo "****** SKIPPING already built TorBrowser Component of Mac Bundle (3/5 for Mac) ******" + echo fi -if [ ! -f $GITIAN_DIR/inputs/pluggable-transports-mac32-gbuilt.zip ]; +if [ ! -f inputs/pluggable-transports-mac32-gbuilt.zip ]; then - echo - echo "****** Starting Pluggable Transports Component of Mac Bundle (3/4 for Mac) ******" - echo + echo + echo "****** Starting Pluggable Transports Component of Mac Bundle (4/5 for Mac) ******" + echo ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit pyptlib=$PYPTLIB_TAG,obfsproxy=$OBFSPROXY_TAG,flashproxy=$FLASHPROXY_TAG,libfte=$LIBFTE_TAG,fteproxy=$FTEPROXY_TAG,txsocksx=$TXSOCKSX_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml if [ $? -ne 0 ]; @@ -155,44 +192,40 @@ then exit 1 fi - cp -a build/out/pluggable-transports-mac*-gbuilt.zip $GITIAN_DIR/inputs/ - #cp -a result/pluggable-transports-mac-res.yml $GITIAN_DIR/inputs/ + cp -a build/out/pluggable-transports-mac*-gbuilt.zip inputs/ + #cp -a result/pluggable-transports-mac-res.yml inputs/ else - echo - echo "****** SKIPPING already built Pluggable Transports Component of Mac Bundle (3/4 for Mac) ******" - echo + echo + echo "****** SKIPPING already built Pluggable Transports Component of Mac Bundle (4/5 for Mac) ******" + echo fi -if [ ! -f $GITIAN_DIR/inputs/bundle-mac.gbuilt ]; -then - echo - echo "****** Starting Bundling+Localization Component of Mac Bundle (4/4 for Mac) ******" - echo - +if [ ! -f inputs/bundle-mac.gbuilt ]; +then + echo + echo "****** Starting Bundling+Localization Component of Mac Bundle (5/5 for Mac) ******" + echo + cd $WRAPPER_DIR && ./record-inputs.sh $VERSIONS_FILE && cd $GITIAN_DIR - + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit libdmg-hfsplus=$LIBDMG_TAG,https-everywhere=$HTTPSE_TAG,torbutton=$TORBUTTON_TAG,tor-launcher=$TORLAUNCHER_TAG $DESCRIPTOR_DIR/mac/gitian-bundle.yml if [ $? -ne 0 ]; then #mv var/build.log ./bundle-fail-mac.log.`date +%Y%m%d%H%M%S` exit 1 fi - - #cp -a build/out/*.dmg $WRAPPER_DIR + mkdir -p $WRAPPER_DIR/$TORBROWSER_VERSION/ cp -a build/out/* $WRAPPER_DIR/$TORBROWSER_VERSION/ || exit 1 - touch $GITIAN_DIR/inputs/bundle-mac.gbuilt + touch inputs/bundle-mac.gbuilt else - echo - echo "****** SKIPPING already built Bundling+Localization Component of Mac Bundle (4/4 for Mac) ******" - echo + echo + echo "****** SKIPPING already built Bundling+Localization Component of Mac Bundle (5/5 for Mac) ******" + echo fi -echo +echo echo "****** Mac Bundle complete ******" -echo - - -# FIXME: docs +echo