GitLab

at 2023-01-19T15:18:08+01:00

Bug 40353: Re-enable rlbox

Updated the creation of the WASI sysroot, to be more similar to what
Mozilla does in shell scripts included in Firefox source code.
This includes the same hacks, like re-using the LLVM we already build.

We were still using lucetc, which is not needed anymore. Therefore,
this commit removes it.

@@ -274,9 +274,6 @@ cargo_vendor-cbindgen-android: submodule-update
  cargo_vendor-cbindgen: submodule-update
  	$(rbm) build cbindgen --step cargo_vendor --target nightly --target torbrowser-linux-x86_64
 -cargo_vendor-lucetc: submodule-update
 -	$(rbm) build lucetc --step cargo_vendor --target nightly --target torbrowser-linux-x86_64
+-
  cargo_vendor-uniffi-rs: submodule-update
  	$(rbm) build uniffi-rs --step cargo_vendor --target nightly --target torbrowser-linux-x86_64

@@ -24,7 +24,7 @@ cd build
  cmake ../llvm -GNinja \
                -DCMAKE_INSTALL_PREFIX=$distdir \
                -DCMAKE_BUILD_TYPE=Release \
 -              -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64[% IF c("var/rlbox") -%];WebAssembly[% END %]" \
 +              -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64;WebAssembly" \
                -DLLVM_ENABLE_PROJECTS="clang;lld" \
                -DLLVM_ENABLE_ZLIB=ON \
                -DLLVM_INSTALL_UTILS=ON \

@@ -49,10 +49,7 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bi
      rtdir=/var/tmp/dist/clang/lib/clang/[% pc("clang", "version") %]/lib/wasi
    [% END -%]
    mkdir -p $rtdir
 -  cp /var/tmp/dist/wasi-sysroot/lib/clang/11.0.0/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
 -  tar -C /var/tmp/dist -xf [% c('input_files_by_name/lucetc') %]
 -  export PATH="/var/tmp/dist/lucetc/bin:$PATH"
 -  export WASM_SANDBOXED_LIBRARIES=graphite,ogg
 +  cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
    export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
  [% END -%]

@@ -158,9 +158,6 @@ input_files:
    - project: wasi-sysroot
      name: wasi-sysroot
      enable: '[% c("var/rlbox") %]'
 -  - project: lucetc
 -    name: lucetc
 -    enable: '[% c("var/rlbox") %]'
    - project: node
      name: node
    - project: nasm

@@ -213,15 +213,5 @@ needed=$(perl -MYAML::XS -e "$p")
  current='[% pc("wasi-sysroot", "git_hash") %]'
  check_update_needed wasi-sdk "$needed" "$current"
+-
 -# lucetc
 -read -d '' p << 'EOF' || true
 -my $d = YAML::XS::LoadFile('taskcluster/ci/fetch/toolchains.yml');
 -print $d->{'lucetc-source'}{fetch}{revision};
 -EOF
 -needed=$(perl -MYAML::XS -e "$p")
 -current='[% pc("lucetc", "git_hash") %]'
 -check_update_needed lucetc "$needed" "$current"
+-
  # End RLBox
  [% END -%]

Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build

Commits:

17 changed files:

Changes:

@@ -26,6 +26,18 @@ cp -r /var/tmp/dist/compiler-rt/* /var/tmp/dist/clang/
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/binutils') %]
  export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH"
 +[% IF c("var/rlbox") -%]
 +    tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %]
 +    # XXX: We need the libclang_rt.builtins-wasm32.a in our clang lib directory.
 +    # Copy it over.
 +    # https://searchfox.org/mozilla-central/source/build/build-clang/build-clang.py#890,
 +    # include it directly in our clang
 +    rtdir=/var/tmp/dist/clang/lib/clang/[% pc("clang", "version") %]/lib/wasi
 +    mkdir -p $rtdir
 +    cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
 +    export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
 +[% END -%]
++
  tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
  cd /var/tmp/build/[% project %]-[% c("version") %]

@@ -173,6 +173,9 @@ input_files:
      name: clang
    - project: 'compiler-rt'
      name: 'compiler-rt'
 +  - project: wasi-sysroot
 +    name: wasi-sysroot
 +    enable: '[% c("var/rlbox") %]'
    - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
      name: gradle-dependencies
      exec: '[% INCLUDE "fetch-gradle-dependencies" %]'

 -From ce82762aef3f8cdbdd340f39f95d2491f69a7da0 Mon Sep 17 00:00:00 2001
 -From: Nicolas Vigier <boklm@torproject.org>
 -Date: Mon, 29 Mar 2021 13:42:48 +0200
 -Subject: [PATCH] Remove -march=native from COMMON_CFLAGS
+-
 -The -march=native flag is causing an error on some hardware. It can also
 -make the build non-reproducible when not building on the same hardware.
 ----
 - lucet-builtins/Makefile | 2 +-
 - 1 file changed, 1 insertion(+), 1 deletion(-)
+-
 -diff --git a/lucet-builtins/Makefile b/lucet-builtins/Makefile
 -index 6ddbcfd..de7e540 100644
 ---- a/lucet-builtins/Makefile
 -+++ b/lucet-builtins/Makefile
 -@@ -7,7 +7,7 @@ LIBBUILTINS_OBJS:=$(addprefix build/, \
 - 	strcmp.o \
 - 	strlen.o )
+-
 --COMMON_CFLAGS:= --std=gnu99 -Ofast -Wall -Werror -march=native -fPIC \
 -+COMMON_CFLAGS:= --std=gnu99 -Ofast -Wall -Werror -fPIC \
 - 	-I../lucet-runtime/include
+-
 - default: build/libbuiltins.so

 -#!/bin/bash
 -[% c("var/set_default_env") -%]
 -# XXX: Do we need the GCC part here if we set CC to clang anyway?
 -[% IF c("var/linux") %]
 -  [% pc('gcc', 'var/setup', { compiler_tarfile => c('input_files_by_name/gcc'),
 -                              hardened_gcc => 0 }) %]
 -  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
 -  export PATH="/var/tmp/dist/binutils/bin:$PATH"
 -[% END -%]
 -distdir=/var/tmp/dist
 -builddir=/var/tmp/build/[% project %]
+-
 -mkdir -p $distdir
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %]
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %]
 -tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %]
 -export PATH="/var/tmp/dist/cmake/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/rust/bin:$PATH"
 -export CC=$distdir/clang/bin/clang
 -export CFLAGS="-L$distdir/clang/lib"
 -export CXX=$distdir/clang/bin/clang++
 -export CXXFLAGS=$CFLAGS
 -export AR=$distdir/clang/bin/llvm-ar
 -export RUSTFLAGS="-C linker=$CXX -C link-arg=$CXXFLAGS"
 -export LUCET_PREFIX=$distdir/[% project %]
 -# XXX: Why do we need that when Mozilla doesn't? See: taskcluster/scripts/misc/build-lucetc.sh
 -export WASI_SYSROOT=$distdir/wasi-sysroot
+-
 -mkdir -p /var/tmp/build
 -tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+-
 -# Now prepare the offline build
 -# Move the directory for hardcoding the path in .cargo/config
 -mv /var/tmp/build/[% project %]-[% c('version') %] $builddir
 -tar -C $builddir -xf [% c('input_files_by_name/lucetc-vendor') %]
 -cd $builddir
 -# .cargo exists already, let's just add to it
 -cat >> .cargo/config << 'EOF'
+-
 -[source.crates-io]
 -replace-with = "vendored-sources"
+-
 -[source.vendored-sources]
 -directory = "/var/tmp/build/lucetc/vendor"
 -EOF
+-
 -patch -p1 < $rootdir/Remove-march-native-from-COMMON_CFLAGS.patch
+-
 -make -j[% c("num_procs") %] build
 -# XXX: Should not be needed, just the lucetc binary, see: taskcluster/scripts/misc/build-lucetc.sh
 -make install
+-
 -cd $distdir
 -[% c('tar', {
 -        tar_src => [ project ],
 -        tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
 -    }) %]

 -# vim: filetype=yaml sw=2
 -filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
 -version: '[% c("abbrev") %]'
 -git_hash: d582d76bf0d17984501f3933688bf7a1eb5cf208
 -git_url: https://github.com/PLSysSec/lucet_sandbox_compiler/
 -git_submodule: 1
+-
 -container:
 -  use_container: 1
+-
 -input_files:
 -  - project: container-image
 -  - name: binutils
 -    project: binutils
 -    enable: '[% c("var/linux") %]'
 -  # Our clang still needs GCC parts (libstdc++ for instance) so we include that
 -  # one here even though compilation gets done solely by clang.
 -  - project: '[% c("var/compiler") %]'
 -    name: '[% c("var/compiler") %]'
 -    enable: '[% c("var/linux") %]'
 -  - name: cmake
 -    project: cmake
 -  - name: clang
 -    project: clang
 -  - name: rust
 -    project: rust
 -  - name: wasi-sysroot
 -    project: wasi-sysroot
 -  # Use `make cargo_vendor-lucetc` to re-generate the vendor tarball
 -  - name: lucetc-vendor
 -    URL: 'https://people.torproject.org/~boklm/mirrors/sources/lucetc-vendor-[% c("version") %].tar.bz2'
 -    sha256sum: 6b044ed76bc8870040174c8fbfa49c3a791b3aba7d32dd00cad541025eb16077
 -  - filename: Remove-march-native-from-COMMON_CFLAGS.patch
+-
 -steps:
 -  cargo_vendor:
 -    filename: '[% project %]-vendor-[% c("version") %].tar.bz2'
 -    input_files:
 -      - project: container-image
 -        pkg_type: build
 -      - project: rust
 -        name: rust
 -        pkg_type: build

 +# vim: filetype=yaml sw=2
 +version: '[% c("abbrev") %]'
 +git_url: https://git.savannah.gnu.org/git/config.git
 +git_hash: c179db1b6f2ae484bfca1e9f8bae273e3319fa7d
 +filename: '[% project %]-[% c("version") %].tar.gz'

 +# vim: filetype=yaml sw=2
 +version: '[% c("abbrev") %]'
 +git_url: https://github.com/WebAssembly/wasi-libc.git
 +git_hash: 9886d3d6200fcc3726329966860fc058707406cd
 +filename: '[% c("project") %]-[% c("version") %].tar.gz'

  #!/bin/bash
  [% c("var/set_default_env") -%]
 -[% IF c("var/linux") %]
 -  [% pc('gcc', 'var/setup', { compiler_tarfile => c('input_files_by_name/gcc'),
 -                              hardened_gcc => 0 }) %]
 -  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
 -  export PATH="/var/tmp/dist/binutils/bin:$PATH"
 -[% END -%]
 -distdir=/var/tmp/dist
 -builddir=/var/tmp/build/[% project %]
 +distdir=/var/tmp/dist/[% project %]
 +builddir=/var/tmp/build/[% project %]-[% c("abbrev") %]
  mkdir -p $distdir
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
  tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %]
 -export PATH=/var/tmp/dist/binutils/bin:/var/tmp/dist/cmake/bin:/var/tmp/dist/ninja:$PATH
 +export PATH="/var/tmp/dist/ninja:/var/tmp/dist/cmake/bin:$PATH"
++
 +[% IF c("var/linux") -%]
 +  [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
 +  [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
 +[% END -%]
  mkdir -p /var/tmp/build
  tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
 -mv /var/tmp/build/[% project %]-[% c('version') %] $builddir
 +cd /var/tmp/build
 +tar -xf $rootdir/[% project %]-[% c('version') %].tar.gz
  cd $builddir
 -# We have no .git in our tar ball but the script version.sh used by
 -# the Makefile depends on that. We replace the script.
++
 +pushd src
 +rmdir config llvm-project wasi-libc
 +tar -xf $rootdir/[% c('input_files_by_name/wasi-config') %]
 +mv wasi-config-* config
 +tar -xf $rootdir/[% c('input_files_by_name/clang-source') %]
 +mv clang-source llvm-project
 +export LLVM_HOME=$(pwd)/llvm-project
 +tar -xf $rootdir/[% c('input_files_by_name/wasi-libc') %]
 +mv wasi-libc-* wasi-libc
 +popd
++
 +# What follows has been copied from Firefox's
 +# taskcluster/scripts/misc/build-sysroot-wasi.sh and adapted to
 +# tor-browser-build.
++
 +mkdir -p build/install/wasi
 +# The wasi-sdk build system wants to build clang itself. We trick it into
 +# thinking it did, and put our own clang where it would have built its own.
 +tar -C build -xf $rootdir/[% c('input_files_by_name/clang') %]
 +mv build/clang build/llvm
 +touch build/llvm.BUILT
++
 +# The wasi-sdk build system wants a clang and an ar binary in
 +# build/install/$PREFIX/bin
 +ln -s $(pwd)/build/llvm/bin build/install/wasi/bin
 +ln -s llvm-ar build/install/wasi/bin/ar
++
 +# tor-browser-build: we have no .git in our tarball but the script version.sh
 +# used by the Makefile depends on that. We replace the script.
  cat > 'version.sh' << EOF
  #!/bin/sh
  echo '[% c("abbrev") %]'
  EOF
 -# XXX: We don't want to use the clang 9.0 wasi-sdk would use, see:
 -# https://bugzilla.mozilla.org/show_bug.cgi?id=1605708
 -make -j[% c("num_procs") %] PREFIX=$distdir/[% project %] DESTDIR=/
 +# Build wasi-libc, libc++ and libc++abi.
 +make \
 +  LLVM_PROJ_DIR=$LLVM_HOME \
 +  PREFIX=/wasi \
 +  build/wasi-libc.BUILT \
 +  build/compiler-rt.BUILT \
 +  NINJA_FLAGS='-j[% c("num_procs") %]'
++
 +# Does Mozilla maybe do this when compiling Clang?
 +rtdir="build/llvm/lib/clang/$(ls build/llvm/lib/clang)/lib/wasi"
 +mkdir -p "$rtdir"
 +cp build/compiler-rt/lib/wasi/libclang_rt.builtins-wasm32.a "$rtdir/"
++
 +make \
 +  LLVM_PROJ_DIR=$LLVM_HOME \
 +  PREFIX=/wasi \
 +  build/libcxx.BUILT \
 +  NINJA_FLAGS='-j[% c("num_procs") %]'
++
 +mv build/install/wasi/share $distdir/
 +mv build/install/wasi/lib $distdir/
 -cd $distdir
 +cd /var/tmp/dist
  [% c('tar', {
          tar_src => [ project ],
          tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),

@@ -168,6 +168,8 @@ var:
      - zh-rCN
      - zh-rTW
 +  rlbox: 1
++
    sign_build: '[% ENV.RBM_SIGN_BUILD %]'
    sign_build_gpg_opts: '[% ENV.RBM_GPG_OPTS %]'
@@ -401,8 +403,6 @@ targets:
        osname: linux-x86_64
        linux-cross: 0
        arch_debian: amd64
 -      # We only support RLBox on the nightly channel and x86_64 for now
 -      rlbox: 0
    linux-i686:
      arch: i686
      var:
@@ -569,7 +569,6 @@ targets:
        FLAGS: "-target [% c('var/build_target') %] -B $cctoolsdir -isysroot $sysrootdir [% IF c('var/macos-aarch64') %]-mcpu=apple-m1[% END %]"
        LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie"
        locale_ja: ja-JP-mac
 -      rlbox: 0
        deps:
          - build-essential
          - faketime