commit 5c4ec887367ad46464f35a35fc630b00e1b3180f Author: Nicolas Vigier boklm@torproject.org Date: Sat Aug 7 09:50:52 2021 +0200

Bug 40307: Update macOS toolchain to switch to mozilla91

Based on work from sysrqb (tor-browser-build!342). --- projects/cctools/config | 2 +- projects/firefox/mozconfig-osx-x86_64 | 2 +- projects/libtapi/build | 3 +- projects/libtapi/config | 2 +- projects/macosx-toolchain/build | 75 ++++++++++++++++++++++++++++------- projects/macosx-toolchain/config | 10 ++--- projects/ninja/build | 2 +- projects/node/config | 4 ++ projects/rust/build | 2 +- projects/rust/config | 2 +- rbm.conf | 11 ++--- 11 files changed, 83 insertions(+), 32 deletions(-)

diff --git a/projects/cctools/config b/projects/cctools/config index 280e692..2725cfe 100644 --- a/projects/cctools/config +++ b/projects/cctools/config @@ -3,7 +3,7 @@ filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' version: '[% c("abbrev") %]' # This can be found in # taskcluster/ci/fetch/toolchains.yml -git_hash: f2890afb8a52944dd8265e489a0691c7b0afdd54 +git_hash: 30518813875aed656aa7f18b6d485feee25f8f87 git_url: https://github.com/tpoechtrager/cctools-port

var: diff --git a/projects/firefox/mozconfig-osx-x86_64 b/projects/firefox/mozconfig-osx-x86_64 index 4609839..6cbc76c 100644 --- a/projects/firefox/mozconfig-osx-x86_64 +++ b/projects/firefox/mozconfig-osx-x86_64 @@ -4,7 +4,7 @@ TOOLCHAIN_DIR=/var/tmp/dist/macosx-toolchain mk_add_options "export LD_LIBRARY_PATH=$TOOLCHAIN_DIR/clang/lib"

CROSS_CCTOOLS_PATH=$TOOLCHAIN_DIR/cctools -CROSS_SYSROOT=$TOOLCHAIN_DIR/MacOSX10.11.sdk +CROSS_SYSROOT=$TOOLCHAIN_DIR/MacOSX10.12.sdk CROSS_PRIVATE_FRAMEWORKS=$CROSS_SYSROOT/System/Library/PrivateFrameworks HARDENING_FLAGS="-Werror=format -Werror=format-security -fstack-protector-strong -D_FORTIFY_SOURCE=2" FLAGS="-target x86_64-apple-darwin -B $CROSS_CCTOOLS_PATH/bin -isysroot $CROSS_SYSROOT $HARDENING_FLAGS" diff --git a/projects/libtapi/build b/projects/libtapi/build index a0d9c3e..0d1bd1f 100644 --- a/projects/libtapi/build +++ b/projects/libtapi/build @@ -36,7 +36,8 @@ cmake $builddir/src/llvm \ -DCMAKE_INSTALL_PREFIX=$distdir \ -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64" \ -DTAPI_REPOSITORY_STRING=$TAPI_REPOSITORY \ - -DTAPI_FULL_VERSION=$TAPI_VERSION + -DTAPI_FULL_VERSION=$TAPI_VERSION \ + -DPYTHON_EXECUTABLE=/usr/bin/python3

ninja clangBasic ninja libtapi install-libtapi install-tapi-headers diff --git a/projects/libtapi/config b/projects/libtapi/config index 6c4f5d0..17aaac3 100644 --- a/projects/libtapi/config +++ b/projects/libtapi/config @@ -3,7 +3,7 @@ filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' version: '[% c("abbrev") %]' # This can be found in # taskcluster/ci/fetch/toolchains.yml -git_hash: 3efb201881e7a76a21e0554906cf306432539cef +git_hash: b4f141a16c7c82f75973429c92242a8c7de3ddb1 git_url: https://github.com/tpoechtrager/apple-libtapi

var: diff --git a/projects/macosx-toolchain/build b/projects/macosx-toolchain/build index 5716a9e..c4d1937 100644 --- a/projects/macosx-toolchain/build +++ b/projects/macosx-toolchain/build @@ -9,31 +9,33 @@ tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %] tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %] tar -C $distdir -xf [% c('input_files_by_name/SDK') %] tar -C $distdir -xf [% c('input_files_by_name/cctools') %] -tar -C $builddir -xf [% c('input_files_by_name/clang-source') %] +tar -C $builddir -xf [% c('input_files_by_name/llvm-project') %]

+arch=x86_64 clangdir=/var/tmp/dist/clang/bin cctoolsdir=$distdir/cctools/bin -sysrootdir=$distdir/MacOSX10.11.sdk/ +sysrootdir=$distdir/MacOSX[% c("version") %].sdk/ +target=x86_64-apple-darwin # We still need to put the cctoolsdir on the path. That's because of `lipo`. See # the respective comment in the cctools build script. export PATH="/var/tmp/dist/ninja:/var/tmp/dist/cmake/bin:$cctoolsdir:$PATH" export MACOSX_DEPLOYMENT_TARGET=[% c("var/macosx_deployment_target") %] +export MACH_USE_SYSTEM_PYTHON=1

-cd $builddir/clang-source/projects +cd $builddir/clang-source patch -p1 < $rootdir/compiler-rt-cross-compile.patch patch -p1 < $rootdir/compiler-rt-no-codesign.patch cd .. -mkdir build -cd build +mkdir build_clang +pushd build_clang # We follow quite closely Mozilla's build/build-clang/build-clang.py with the -# clang-macosx64.json flavor. This currently implies disabling Xray as it -# requires an OSX 10.12 SDK (see: https://bugs.llvm.org/show_bug.cgi?id=38959). +# clang-macosx64.json flavor. cmake -GNinja \ -DCMAKE_C_COMPILER=$clangdir/clang \ -DCMAKE_CXX_COMPILER=$clangdir/clang++ \ -DCMAKE_ASM_COMPILER=$clangdir/clang \ -DCMAKE_LINKER=$clangdir/clang \ - -DCMAKE_AR=$cctoolsdir/x86_64-apple-darwin-ar \ + -DCMAKE_AR=$cctoolsdir/$target-ar \ -DCMAKE_C_FLAGS="[% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \ -DCMAKE_CXX_FLAGS="-stdlib=libc++ [% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \ -DCMAKE_ASM_FLAGS="[% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \ @@ -43,15 +45,15 @@ cmake -GNinja \ -DCMAKE_INSTALL_PREFIX=$distdir/clang \ -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64" \ -DLLVM_ENABLE_ASSERTIONS=OFF \ - -DPYTHON_EXECUTABLE=/usr/bin/python2.7 \ + -DPYTHON_EXECUTABLE=/usr/bin/python3 \ -DLLVM_TOOL_LIBCXX_BUILD=ON \ -DLLVM_ENABLE_BINDINGS=OFF \ -DLLVM_ENABLE_LIBXML2=FORCE_ON \ -DLLVM_LINK_LLVM_DYLIB=ON \ - -DCMAKE_RANLIB=$cctoolsdir/x86_64-apple-darwin-ranlib \ - -DCMAKE_LIBTOOL=$cctoolsdir/x86_64-apple-darwin-libtool \ + -DCMAKE_RANLIB=$cctoolsdir/$target-ranlib \ + -DCMAKE_LIBTOOL=$cctoolsdir/$target-libtool \ -DCMAKE_SYSTEM_NAME=Darwin \ - -DCMAKE_SYSTEM_VERSION=10.10 \ + -DCMAKE_SYSTEM_VERSION=$MACOSX_DEPLOYMENT_TARGET \ -DLLVM_ENABLE_THREADS=OFF \ -DCOMPILER_RT_BUILD_XRAY=OFF \ -DLIBCXXABI_LIBCXX_INCLUDES=$builddir/clang-source/projects/libcxx/include \ @@ -61,10 +63,53 @@ cmake -GNinja \ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY \ -DCMAKE_MACOSX_RPATH=ON \ - -DCMAKE_OSX_ARCHITECTURES=x86_64 \ - -DDARWIN_osx_ARCHS=x86_64 \ + -DCMAKE_OSX_ARCHITECTURES=$arch \ + -DDARWIN_macosx_OVERRIDE_SDK_VERSION=$MACOSX_DEPLOYMENT_TARGET \ + -DDARWIN_osx_ARCHS=$arch \ -DDARWIN_osx_SYSROOT=$sysrootdir \ - -DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin $builddir/clang-source + -DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin $builddir/clang-source/llvm + +ninja install -v + +popd +mkdir build_compiler_rt +pushd build_compiler_rt + +# And we build compiler-rt by following taskcluster/scripts/misc/build-compiler-rt.sh. +cmake -GNinja \ + -DCMAKE_C_COMPILER=$clangdir/clang \ + -DCMAKE_CXX_COMPILER=$clangdir/clang++ \ + -DCMAKE_C_COMPILER_TARGET=$target \ + -DCMAKE_CXX_COMPILER_TARGET=$target \ + -DCMAKE_ASM_COMPILER_TARGET=$target \ + -DCMAKE_AR=$clangdir/llvm-ar \ + -DCMAKE_RANLIB=$clangdir/llvm-ranlib \ + -DCMAKE_C_FLAGS="[% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \ + -DCMAKE_CXX_FLAGS="-stdlib=libc++ [% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \ + -DCMAKE_ASM_FLAGS="[% c('var/FLAGS') %] -I$sysrootdir/usr/include -iframework $sysrootdir/System/Library/Frameworks" \ + -DCMAKE_EXE_LINKER_FLAGS="[% c('var/LDFLAGS') %]" \ + -DCMAKE_SHARED_LINKER_FLAGS="[% c('var/LDFLAGS') %]" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=OFF \ + -DCMAKE_INSTALL_PREFIX=$distdir/clang/lib/clang/[% pc("clang", "version") %]/ \ + -DLLVM_CONFIG_PATH=$clangdir/llvm-config \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DCMAKE_LINKER=$cctoolsdir/$target-ld \ + -DCMAKE_LIPO=$cctoolsdir/lipo \ + -DCMAKE_SYSTEM_NAME=Darwin \ + -DCMAKE_SYSTEM_VERSION=$MACOSX_DEPLOYMENT_TARGET \ + -DDARWIN_macosx_OVERRIDE_SDK_VERSION=$MACOSX_DEPLOYMENT_TARGET \ + -DDARWIN_osx_ARCHS=$arch \ + -DDARWIN_osx_SYSROOT=$sysrootdir \ + -DDARWIN_osx_BUILTIN_ARCHS=$arch \ + -DCMAKE_OSX_SYSROOT=$sysrootdir \ + -DCMAKE_FIND_ROOT_PATH=$sysrootdir \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \ + -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY \ + -DDARWIN_osx_BUILTIN_ARCHS=$arch \ + $builddir/clang-source/compiler-rt + ninja install -v

# We now have a native macosx64 toolchain. diff --git a/projects/macosx-toolchain/config b/projects/macosx-toolchain/config index 5f9a109..b42d190 100644 --- a/projects/macosx-toolchain/config +++ b/projects/macosx-toolchain/config @@ -1,13 +1,13 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-1-[% c("var/build_id") %].tar.gz' -version: 10.11 +version: 10.12 var: container: use_container: 1 setup: | mkdir -p /var/tmp/dist tar -C /var/tmp/dist -xf $rootdir/[% c("compiler_tarfile") %] - export sysrootdir=/var/tmp/dist/[% project %]/MacOSX10.11.sdk/ + export sysrootdir=/var/tmp/dist/[% project %]/MacOSX[% c("version") %].sdk/ export clangdir=/var/tmp/dist/[% project %]/clang export cctoolsdir=/var/tmp/dist/[% project %]/cctools/bin export PATH="$clangdir/bin:$cctoolsdir:$PATH" @@ -30,8 +30,8 @@ input_files: # build/macosx/cross-mozconfig.common - name: SDK URL: 'https://people.torproject.org/~gk/mirrors/sources/MacOSX%5B% c("version") %].sdk.tar.bz2' - sha256sum: d11e410d757d023be66d2fe7eaa85a1d232da3ac04f177eff3d8b8275cd9ffca - - name: clang-source - project: clang-source + sha256sum: d2ef4ae4f85675c08795d33f283e307b6299184d78d4b10bdbea39df6dca7d94 + - name: llvm-project + project: llvm-project - filename: compiler-rt-cross-compile.patch - filename: compiler-rt-no-codesign.patch diff --git a/projects/ninja/build b/projects/ninja/build index 9fc0eaa..527f1b6 100644 --- a/projects/ninja/build +++ b/projects/ninja/build @@ -10,7 +10,7 @@ mkdir /var/tmp/build tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %]

-[% IF c("var/windows") -%] +[% IF c("var/windows") || c("var/osx") -%] # Starting with Debian Bullseye, /usr/bin/python does not exist, so # we need to use python3 explicitly sed -i 's/env python$/env python3/' configure.py diff --git a/projects/node/config b/projects/node/config index ce058cf..2682fad 100644 --- a/projects/node/config +++ b/projects/node/config @@ -16,6 +16,10 @@ targets: var: arch_deps: - python2 + osx-x86_64: + var: + arch_deps: + - python2

input_files: - project: container-image diff --git a/projects/rust/build b/projects/rust/build index 69bd0f4..213e1c1 100644 --- a/projects/rust/build +++ b/projects/rust/build @@ -31,7 +31,7 @@ export PATH="$distdir-rust-old/bin:$PATH" cat > $distdir/helper/x86_64-apple-darwin-clang << 'EOF' #!/bin/sh BASEDIR=/var/tmp/dist/macosx-toolchain -$BASEDIR/cctools/bin/x86_64-apple-darwin-clang -target x86_64-apple-darwin -B $BASEDIR/cctools/bin -isysroot $BASEDIR/MacOSX10.11.sdk/ -Wl,-syslibroot,$BASEDIR/MacOSX10.11.sdk/ -Wl,-dead_strip -Wl,-pie "$@" +$BASEDIR/cctools/bin/x86_64-apple-darwin-clang -target x86_64-apple-darwin -B $BASEDIR/cctools/bin -isysroot $sysrootdir -Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie "$@" EOF

chmod +x $distdir/helper/x86_64-apple-darwin-clang diff --git a/projects/rust/config b/projects/rust/config index 4995f1c..2a6657f 100644 --- a/projects/rust/config +++ b/projects/rust/config @@ -67,7 +67,7 @@ targets: - libssl-dev - pkg-config - zlib1g-dev - configure_opt: --enable-local-rust --enable-vendor --enable-extended --release-channel=stable --sysconfdir=etc --target=x86_64-apple-darwin --set=target.x86_64-apple-darwin.cc=x86_64-apple-darwin-clang + configure_opt: --enable-local-rust --enable-vendor --enable-extended --release-channel=stable --sysconfdir=etc --target=x86_64-unknown-linux-gnu,x86_64-apple-darwin --set=target.x86_64-apple-darwin.cc=x86_64-apple-darwin-clang

windows: var: diff --git a/rbm.conf b/rbm.conf index c67fdc9..be891d0 100644 --- a/rbm.conf +++ b/rbm.conf @@ -476,28 +476,29 @@ targets:

torbrowser-osx-x86_64: - osx-x86_64 + - ff91esr osx-x86_64: arch: x86_64 var: osx: 1 osname: osx-x86_64 container: - suite: buster + suite: bullseye arch: amd64 compiler: 'macosx-toolchain' configure_opt: '--host=x86_64-apple-darwin CC="x86_64-apple-darwin-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin-clang++ [% c("var/FLAGS") %]" [% c("var/configure_opt_project") %]' FLAGS: "-target x86_64-apple-darwin -B $cctoolsdir -isysroot $sysrootdir" LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie" - macosx_deployment_target: '10.9' + macosx_deployment_target: '10.12' locale_ja: ja-JP-mac - # We only support RLBox on the nightly channel for now - rlbox: '[% c("var/nightly") %]' + rlbox: 0 # We only build snowflake on the alpha and nightly # channels for now. snowflake: '[% c("var/alpha") || c("var/nightly") %]' deps: - build-essential - - python + - python3 + - python3-distutils - automake - libtool - zip