GitLab

at 2025-04-02T15:01:36+02:00

Bug 41393 - Create special clang packaged for Linux

Building Firefox requires compiler-rt wasm32 library to be
packaged alongside clang.

We were doing this step inside the firefox and geckoview
projects, but that meant one can't simply download the clang
toolchain and use it for dev builds.

MacOS and Windows already have separate projects to package
clang with other libraries, so a step to include the wasm32
compiler-rt lib was added to each of them.

 +#!/bin/bash
 +[% c("var/set_default_env") -%]
 +distdir=/var/tmp/dist/[% project %]
 +mkdir -p /var/tmp/dist
++
 +tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %]
 +tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-compiler-rt') %]
++
 +[% IF c("var/android") -%]
 +  tar -C /var/tmp/dist -xf [% c('input_files_by_name/llvm-runtimes') %]
 +  cp -r /var/tmp/dist/llvm-runtimes/* /var/tmp/dist/clang
 +[% END -%]
++
 +mkdir $distdir
 +mv /var/tmp/dist/clang/* $distdir/
++
 +cd /var/tmp/dist
 +[% c('tar', {
 +    tar_src => [ project ],
 +    tar_args => '-caf ' _ dest_dir _ '/' _ c('filename'),
 +  }) %]

 +# vim: filetype=yaml sw=2
 +version: '[% c("var/llvm_version") %]'
 +filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.[% c("compress_tar") %]'
 +container:
 +  use_container: 1
++
 +input_files:
 +  - project: container-image
 +  - name: clang
 +    project: clang
 +  - project: wasi-sysroot
 +    name: wasi-compiler-rt
 +    pkg_type: build_compiler_rt
 +  - project: llvm-runtimes
 +    name: llvm-runtimes
 +    enable: '[% c("var/android") %]'

  This is the Clang compiler.
 -For Linux and Android, we use it only to build Firefox and GeckoView.
 -For Windows and macOS, we use it to build everything, but we do not use this
 +For Linux and Android, we use it only to build Firefox and GeckoView, but we do
 +not use this project directly. Instead, we use `clang-linux`,
 +that adds platform-specific libraries and tools.
 +For Windows and macOS, we use it to build everything, but we also do not use this
  project directly. Instead, we use `mingw-w64-clang` and `macosx_toolchain`, that
  also add platform-specific libraries and tools.

@@ -37,7 +37,7 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/node/bi
    [% END -%]
    # Use clang for everything on Linux now if we don't build with ASan.
    [% IF ! c("var/asan") -%]
 -    export PATH="/var/tmp/dist/clang/bin:$PATH"
 +    export PATH="/var/tmp/dist/clang-linux/bin:$PATH"
    [% END -%]
    [% IF c("var/linux-i686") -%]
      # Exporting `PKG_CONFIG_PATH` in the mozconfig file is causing build
@@ -51,20 +51,7 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/node/bi
  [% IF c("var/rlbox") -%]
    tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %]
 -  # XXX: We need the libclang_rt.builtins-wasm32.a in our clang lib directory.
 -  # Copy it over.
 -  # https://searchfox.org/mozilla-central/source/build/build-clang/build-clang.py#890,
 -  # include it directly in our clang
 -  [% IF c("var/macos") -%]
 -    rtdir=/var/tmp/dist/macosx-toolchain/clang/lib/clang/[% pc("llvm-project", "var/llvm_major") %]/lib/wasi
 -  [% ELSIF c("var/windows") -%]
 -    rtdir=/var/tmp/dist/mingw-w64-clang/lib/clang/[% pc("llvm-project", "var/llvm_major") %]/lib/wasi
 -  [% ELSE -%]
 -    rtdir=/var/tmp/dist/clang/lib/clang/[% pc("llvm-project", "var/llvm_major") %]/lib/wasi
 -  [% END -%]
 -  mkdir -p $rtdir
 -  cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
 -  export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
 +  export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot
  [% END -%]
  tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]

@@ -206,7 +206,7 @@ input_files:
    - project: python
      name: python
      enable: '[% c("var/linux") %]'
 -  - project: clang
 +  - project: clang-linux
      name: clang
      enable: '[% c("var/linux") %]'
    - project: fxc2

brizental pushed to branch main at The Tor Project / Applications / tor-browser-build

Commits:

17 changed files:

Changes:

 +This projects packages extra libraries and tools alongside clang for Firefox developement
 +on Linux and Android. It is the default compiler for 'firefox' and 'geckoview' projects
 +on Linux and Android.

@@ -5,21 +5,12 @@ tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %]
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/cbindgen') %]
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/nasm') %]
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %]
 -export LLVM_CONFIG="/var/tmp/dist/clang/bin/llvm-config"
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/llvm-runtimes') %]
 -cp -r /var/tmp/dist/llvm-runtimes/* /var/tmp/dist/clang/
 -export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/clang/bin:$PATH"
 +export LLVM_CONFIG="/var/tmp/dist/clang-linux/bin/llvm-config"
 +export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/clang-linux/bin:$PATH"
  [% IF c("var/rlbox") -%]
    tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %]
 -  # XXX: We need the libclang_rt.builtins-wasm32.a in our clang lib directory.
 -  # Copy it over.
 -  # https://searchfox.org/mozilla-central/source/build/build-clang/build-clang.py#890,
 -  # include it directly in our clang
 -  rtdir="/var/tmp/dist/clang/lib/clang/[% pc('llvm-project', 'var/llvm_major') %]/lib/wasi"
 -  mkdir -p "$rtdir"
 -  cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a "$rtdir/"
 -  export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
 +  export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/
  [% END -%]
  cd /var/tmp/build/[% project %]-[% c("version") %]

@@ -7,6 +7,7 @@ mkdir -p "$distdir"
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %]
  tar -C $distdir -xf [% c('input_files_by_name/clang') %]
 +tar -C $distdir -xf $rootdir/[% c('input_files_by_name/wasi-compiler-rt') %]
  tar -C $distdir -xf [% c('input_files_by_name/cctools') %]
  tar -C $builddir -xf [% c('input_files_by_name/llvm-project') %]

@@ -32,6 +32,9 @@ input_files:
      sha512sum: a4e05d2a051027620c86f72694da126a2ceed59b8740270426c4ca3adb38e16fe981b536a373a7f0cdcc84bbe9d9149608ccd03967dbd94262548ccadbfa0f5d
    - name: llvm-project
      project: llvm-project
 +  - project: wasi-sysroot
 +    name: wasi-compiler-rt
 +    pkg_type: build_compiler_rt
    - filename: macpkg.py
    - filename: unpack-sdk.py
    - filename: tools.diff

@@ -6,6 +6,7 @@ mkdir -p $distdir $builddir
  cd /var/tmp/dist
  tar -xf $rootdir/[% c('input_files_by_name/clang') %]
 +tar -xf $rootdir/[% c('input_files_by_name/wasi-compiler-rt') %]
  mv clang/* mingw-w64-clang/
  tar -xf $rootdir/[% c('input_files_by_name/cmake') %]

@@ -30,13 +30,14 @@ We stick to the same version used by Firefox, that can be found in the usual
  # Mozilla's build script
 -Our build script is an adaptation of Firefox's
 -`taskcluster/scripts/misc/build-sysroot-wasi.sh`.
 -After ESR updates, we should check if that script was updated.
+-
 -The main difference is that Mozilla builds `libclang_rt.builtins-wasm32.a` with
 -Clang, whereas we build it here and inject it only in the `firefox`/`geckoview`
 -projects.
 +Similar to Firefox's build system, this project has two different build targets.
 +The default `build` target builds the wasi-sysroot, while the `build_compiler_rt`
 +target builds the wasm32 compiler-rt which gets injected into clang by the `clang-linux`,
 +`mingw-w64-clang` and `macosx-toolchain` projects.
++
 +These build scripts are adaptations of Firefox's `taskcluster/scripts/misc/build-sysroot-wasi.sh`
 +and `taskcluster/scripts/misc/build-compiler-rt-wasi.sh`. **After ESR updates, we should
 +check if that script was updated.**
  # Different build ids, same outputs

  #!/bin/bash
 -[% c("var/set_default_env") -%]
 -distdir=/var/tmp/dist/[% project %]
 -builddir=/var/tmp/build/[% project %]-[% c("abbrev") %]
 -mkdir -p $distdir
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %]
 -export PATH="/var/tmp/dist/ninja:/var/tmp/dist/cmake/bin:$PATH"
 +[% INCLUDE 'build_common' %]
 -[% IF c("var/linux") -%]
 -  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
 -  [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
 -[% END -%]
 +# We already packed compiler-rt alongside clang.
 +# Just let the build system know it's there.
 +touch build/compiler-rt.BUILT
 -mkdir -p /var/tmp/build
 -tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.[% c('compress_tar') %]
+-
 -cd /var/tmp/build
 -tar -xf $rootdir/[% project %]-[% c('version') %].tar.[% c('compress_tar') %]
 -cd $builddir
+-
 -pushd src
 -rmdir config llvm-project wasi-libc
 -tar -xf $rootdir/[% c('input_files_by_name/wasi-config') %]
 -mv wasi-config-* config
 -tar -xf $rootdir/[% c('input_files_by_name/clang-source') %]
 -mv clang-source llvm-project
 -export LLVM_HOME=$(pwd)/llvm-project
 -tar -xf $rootdir/[% c('input_files_by_name/wasi-libc') %]
 -mv wasi-libc-* wasi-libc
 -popd
+-
 -# What follows has been copied from Firefox's
 -# taskcluster/scripts/misc/build-sysroot-wasi.sh and adapted to
 -# tor-browser-build.
+-
 -patch -p1 < $rootdir/wasi-sdk.patch
+-
 -mkdir -p build/install/wasi
 -# The wasi-sdk build system wants to build clang itself. We trick it into
 -# thinking it did, and put our own clang where it would have built its own.
 -tar -C build -xf $rootdir/[% c('input_files_by_name/clang') %]
 -mv build/clang build/llvm
 -touch build/llvm.BUILT
+-
 -# The wasi-sdk build system wants a clang and an ar binary in
 -# build/install/$PREFIX/bin
 -ln -s $(pwd)/build/llvm/bin build/install/wasi/bin
 -ln -s llvm-ar build/install/wasi/bin/ar
+-
 -# tor-browser-build: we have no .git in our tarball but the script version.sh
 -# used by the Makefile depends on that. We replace the script.
 -cat > 'version.sh' << EOF
 -#!/bin/sh
 -echo '[% c("abbrev") %]'
 -EOF
+-
 -# Build wasi-libc, and re-pack it, to make sure we get a deterministic output.
 -make PREFIX=/wasi build/wasi-libc.BUILT
+-
 -# We need to compile compiler-rt before building libc++ and libc++abi, because
 -# we need to inject it to Clang.
 -make \
 -  LLVM_PROJ_DIR=$LLVM_HOME \
 -  PREFIX=/wasi \
 -  build/compiler-rt.BUILT \
 -  NINJA_FLAGS='-j[% c("num_procs") %]'
+-
 -# Does Mozilla maybe do this when compiling Clang?
 -rtdir="build/llvm/lib/clang/$(ls build/llvm/lib/clang)/lib/wasi"
 -mkdir -p "$rtdir"
 -cp build/compiler-rt/lib/wasi/libclang_rt.builtins-wasm32.a "$rtdir/"
+-
 -# Now we can finally build libc++ and libc++abi.
  make \
    LLVM_PROJ_DIR=$LLVM_HOME \
    PREFIX=/wasi \
    build/libcxx.BUILT \
    NINJA_FLAGS='-j[% c("num_procs") %]'
 -mv build/install/wasi/share $distdir/
 -mv build/install/wasi/lib $distdir/
 +mv build/install/wasi/share/wasi-sysroot/* $distdir/
  cd /var/tmp/dist
  [% c('tar', {

@@ -15,27 +15,67 @@ container:
  var:
    no_crosscompile: 1
 +targets:
 +  linux:
 +    var:
 +      clang-project: clang-linux
 +  android:
 +    var:
 +      clang-project: clang-linux
 +  macos:
 +    var:
 +      clang-project: macosx-toolchain
 +  windows:
 +    var:
 +      clang-project: mingw-w64-clang
++
 +steps:
 +  build_compiler_rt:
 +    filename: 'wasi-compiler-rt-[% c("version") %]-[% c("var/build_id") %].tar.[% c("compress_tar") %]'
 +    build_compiler_rt: '[% INCLUDE build_compiler_rt %]'
 +    var:
 +      is_compiler_rt: 1
++
  input_files:
    - project: container-image
 +    pkg_type: build
    # We do not use GCC, but we need its libsdc++ to run Clang
    - name: '[% c("var/compiler") %]'
      project: '[% c("var/compiler") %]'
      enable: '[% c("var/linux") %]'
 +    pkg_type: build
    - name: clang
      project: clang
 +    enable: '[% c("var/is_compiler_rt") %]'
 +    pkg_type: build
 +  - name: clang
 +    project: clang-linux
 +    enable: '[% !c("var/is_compiler_rt") && (c("var/linux") || c("var/android")) %]'
 +  - name: clang
 +    project: macosx-toolchain
 +    enable: '[% !c("var/is_compiler_rt") && c("var/macos") %]'
 +  - name: clang
 +    project: mingw-w64-clang
 +    enable: '[% !c("var/is_compiler_rt") && c("var/windows") %]'
    - project: llvm-project
      name: clang-source
 +    pkg_type: build
    - name: cmake
      project: cmake
 +    pkg_type: build
    - name: ninja
      project: ninja
 +    pkg_type: build
    - name: python
      project: python
      enable: '[% c("var/linux") %]'
 +    pkg_type: build
    - name: wasi-config
      project: wasi-config
 +    pkg_type: build
    - name: wasi-libc
      project: wasi-libc
 +    pkg_type: build
    # taskcluster/scripts/misc/wasi-sdk.patch from Firefox's source code.
    # Taken from gecko-dev@823106ac4295f822717f5a953047dc89074d632b
    - filename: wasi-sdk.patch