Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build

Commits: d940f9db by Pier Angelo Vendrame at 2023-01-19T15:18:08+01:00 Bug 40353: Re-enable rlbox

Updated the creation of the WASI sysroot, to be more similar to what Mozilla does in shell scripts included in Firefox source code. This includes the same hacks, like re-using the LLVM we already build.

We were still using lucetc, which is not needed anymore. Therefore, this commit removes it.

- - - - -

17 changed files:

- Makefile - projects/clang/build - projects/firefox/build - projects/firefox/config - projects/firefox/list_toolchain_updates_checks - projects/geckoview/build - projects/geckoview/config - − projects/lucetc/Remove-march-native-from-COMMON_CFLAGS.patch - − projects/lucetc/build - − projects/lucetc/config - + projects/wasi-config/build - + projects/wasi-config/config - + projects/wasi-libc/build - + projects/wasi-libc/config - projects/wasi-sysroot/build - projects/wasi-sysroot/config - rbm.conf

Changes:

===================================== Makefile ===================================== @@ -274,9 +274,6 @@ cargo_vendor-cbindgen-android: submodule-update cargo_vendor-cbindgen: submodule-update $(rbm) build cbindgen --step cargo_vendor --target nightly --target torbrowser-linux-x86_64

-cargo_vendor-lucetc: submodule-update - $(rbm) build lucetc --step cargo_vendor --target nightly --target torbrowser-linux-x86_64 - cargo_vendor-uniffi-rs: submodule-update $(rbm) build uniffi-rs --step cargo_vendor --target nightly --target torbrowser-linux-x86_64

===================================== projects/clang/build ===================================== @@ -24,7 +24,7 @@ cd build cmake ../llvm -GNinja \ -DCMAKE_INSTALL_PREFIX=$distdir \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64[% IF c("var/rlbox") -%];WebAssembly[% END %]" \ + -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64;WebAssembly" \ -DLLVM_ENABLE_PROJECTS="clang;lld" \ -DLLVM_ENABLE_ZLIB=ON \ -DLLVM_INSTALL_UTILS=ON \

===================================== projects/firefox/build ===================================== @@ -49,10 +49,7 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bi rtdir=/var/tmp/dist/clang/lib/clang/[% pc("clang", "version") %]/lib/wasi [% END -%] mkdir -p $rtdir - cp /var/tmp/dist/wasi-sysroot/lib/clang/11.0.0/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir - tar -C /var/tmp/dist -xf [% c('input_files_by_name/lucetc') %] - export PATH="/var/tmp/dist/lucetc/bin:$PATH" - export WASM_SANDBOXED_LIBRARIES=graphite,ogg + cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot [% END -%]

===================================== projects/firefox/config ===================================== @@ -158,9 +158,6 @@ input_files: - project: wasi-sysroot name: wasi-sysroot enable: '[% c("var/rlbox") %]' - - project: lucetc - name: lucetc - enable: '[% c("var/rlbox") %]' - project: node name: node - project: nasm

===================================== projects/firefox/list_toolchain_updates_checks ===================================== @@ -213,15 +213,5 @@ needed=$(perl -MYAML::XS -e "$p") current='[% pc("wasi-sysroot", "git_hash") %]' check_update_needed wasi-sdk "$needed" "$current"

- -# lucetc -read -d '' p << 'EOF' || true -my $d = YAML::XS::LoadFile('taskcluster/ci/fetch/toolchains.yml'); -print $d->{'lucetc-source'}{fetch}{revision}; -EOF -needed=$(perl -MYAML::XS -e "$p") -current='[% pc("lucetc", "git_hash") %]' -check_update_needed lucetc "$needed" "$current" - # End RLBox [% END -%]

===================================== projects/geckoview/build ===================================== @@ -26,6 +26,18 @@ cp -r /var/tmp/dist/compiler-rt/* /var/tmp/dist/clang/ tar -C /var/tmp/dist -xf [% c('input_files_by_name/binutils') %] export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH"

+[% IF c("var/rlbox") -%] + tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %] + # XXX: We need the libclang_rt.builtins-wasm32.a in our clang lib directory. + # Copy it over. + # https://searchfox.org/mozilla-central/source/build/build-clang/build-clang.p..., + # include it directly in our clang + rtdir=/var/tmp/dist/clang/lib/clang/[% pc("clang", "version") %]/lib/wasi + mkdir -p $rtdir + cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir + export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot +[% END -%] + tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz

cd /var/tmp/build/[% project %]-[% c("version") %]

===================================== projects/geckoview/config ===================================== @@ -173,6 +173,9 @@ input_files: name: clang - project: 'compiler-rt' name: 'compiler-rt' + - project: wasi-sysroot + name: wasi-sysroot + enable: '[% c("var/rlbox") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies exec: '[% INCLUDE "fetch-gradle-dependencies" %]'

===================================== projects/lucetc/Remove-march-native-from-COMMON_CFLAGS.patch deleted ===================================== @@ -1,24 +0,0 @@ -From ce82762aef3f8cdbdd340f39f95d2491f69a7da0 Mon Sep 17 00:00:00 2001 -From: Nicolas Vigier boklm@torproject.org -Date: Mon, 29 Mar 2021 13:42:48 +0200 -Subject: [PATCH] Remove -march=native from COMMON_CFLAGS - -The -march=native flag is causing an error on some hardware. It can also -make the build non-reproducible when not building on the same hardware. ---- - lucet-builtins/Makefile | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lucet-builtins/Makefile b/lucet-builtins/Makefile -index 6ddbcfd..de7e540 100644 ---- a/lucet-builtins/Makefile -+++ b/lucet-builtins/Makefile -@@ -7,7 +7,7 @@ LIBBUILTINS_OBJS:=$(addprefix build/, \ - strcmp.o \ - strlen.o ) - --COMMON_CFLAGS:= --std=gnu99 -Ofast -Wall -Werror -march=native -fPIC \ -+COMMON_CFLAGS:= --std=gnu99 -Ofast -Wall -Werror -fPIC \ - -I../lucet-runtime/include - - default: build/libbuiltins.so

===================================== projects/lucetc/build deleted ===================================== @@ -1,57 +0,0 @@ -#!/bin/bash -[% c("var/set_default_env") -%] -# XXX: Do we need the GCC part here if we set CC to clang anyway? -[% IF c("var/linux") %] - [% pc('gcc', 'var/setup', { compiler_tarfile => c('input_files_by_name/gcc'), - hardened_gcc => 0 }) %] - tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %] - export PATH="/var/tmp/dist/binutils/bin:$PATH" -[% END -%] -distdir=/var/tmp/dist -builddir=/var/tmp/build/[% project %] - -mkdir -p $distdir -tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %] -tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %] -tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %] -tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %] -export PATH="/var/tmp/dist/cmake/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/rust/bin:$PATH" -export CC=$distdir/clang/bin/clang -export CFLAGS="-L$distdir/clang/lib" -export CXX=$distdir/clang/bin/clang++ -export CXXFLAGS=$CFLAGS -export AR=$distdir/clang/bin/llvm-ar -export RUSTFLAGS="-C linker=$CXX -C link-arg=$CXXFLAGS" -export LUCET_PREFIX=$distdir/[% project %] -# XXX: Why do we need that when Mozilla doesn't? See: taskcluster/scripts/misc/build-lucetc.sh -export WASI_SYSROOT=$distdir/wasi-sysroot - -mkdir -p /var/tmp/build -tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz - -# Now prepare the offline build -# Move the directory for hardcoding the path in .cargo/config -mv /var/tmp/build/[% project %]-[% c('version') %] $builddir -tar -C $builddir -xf [% c('input_files_by_name/lucetc-vendor') %] -cd $builddir -# .cargo exists already, let's just add to it -cat >> .cargo/config << 'EOF' - -[source.crates-io] -replace-with = "vendored-sources" - -[source.vendored-sources] -directory = "/var/tmp/build/lucetc/vendor" -EOF - -patch -p1 < $rootdir/Remove-march-native-from-COMMON_CFLAGS.patch - -make -j[% c("num_procs") %] build -# XXX: Should not be needed, just the lucetc binary, see: taskcluster/scripts/misc/build-lucetc.sh -make install - -cd $distdir -[% c('tar', { - tar_src => [ project ], - tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), - }) %]

===================================== projects/lucetc/config deleted ===================================== @@ -1,43 +0,0 @@ -# vim: filetype=yaml sw=2 -filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -version: '[% c("abbrev") %]' -git_hash: d582d76bf0d17984501f3933688bf7a1eb5cf208 -git_url: https://github.com/PLSysSec/lucet_sandbox_compiler/ -git_submodule: 1 - -container: - use_container: 1 - -input_files: - - project: container-image - - name: binutils - project: binutils - enable: '[% c("var/linux") %]' - # Our clang still needs GCC parts (libstdc++ for instance) so we include that - # one here even though compilation gets done solely by clang. - - project: '[% c("var/compiler") %]' - name: '[% c("var/compiler") %]' - enable: '[% c("var/linux") %]' - - name: cmake - project: cmake - - name: clang - project: clang - - name: rust - project: rust - - name: wasi-sysroot - project: wasi-sysroot - # Use `make cargo_vendor-lucetc` to re-generate the vendor tarball - - name: lucetc-vendor - URL: 'https://people.torproject.org/~boklm/mirrors/sources/lucetc-vendor-%5B% c("version") %].tar.bz2' - sha256sum: 6b044ed76bc8870040174c8fbfa49c3a791b3aba7d32dd00cad541025eb16077 - - filename: Remove-march-native-from-COMMON_CFLAGS.patch - -steps: - cargo_vendor: - filename: '[% project %]-vendor-[% c("version") %].tar.bz2' - input_files: - - project: container-image - pkg_type: build - - project: rust - name: rust - pkg_type: build

===================================== projects/wasi-config/build ===================================== @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +mkdir -p /var/tmp/dist +mv $rootdir/[% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c("filename") %]

===================================== projects/wasi-config/config ===================================== @@ -0,0 +1,5 @@ +# vim: filetype=yaml sw=2 +version: '[% c("abbrev") %]' +git_url: https://git.savannah.gnu.org/git/config.git +git_hash: c179db1b6f2ae484bfca1e9f8bae273e3319fa7d +filename: '[% project %]-[% c("version") %].tar.gz'

===================================== projects/wasi-libc/build ===================================== @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +mkdir -p /var/tmp/dist +mv $rootdir/[% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c("filename") %]

===================================== projects/wasi-libc/config ===================================== @@ -0,0 +1,5 @@ +# vim: filetype=yaml sw=2 +version: '[% c("abbrev") %]' +git_url: https://github.com/WebAssembly/wasi-libc.git +git_hash: 9886d3d6200fcc3726329966860fc058707406cd +filename: '[% c("project") %]-[% c("version") %].tar.gz'

===================================== projects/wasi-sysroot/build ===================================== @@ -1,35 +1,81 @@ #!/bin/bash [% c("var/set_default_env") -%] -[% IF c("var/linux") %] - [% pc('gcc', 'var/setup', { compiler_tarfile => c('input_files_by_name/gcc'), - hardened_gcc => 0 }) %] - tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %] - export PATH="/var/tmp/dist/binutils/bin:$PATH" -[% END -%] -distdir=/var/tmp/dist -builddir=/var/tmp/build/[% project %] +distdir=/var/tmp/dist/[% project %] +builddir=/var/tmp/build/[% project %]-[% c("abbrev") %] mkdir -p $distdir tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %] tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %] -export PATH=/var/tmp/dist/binutils/bin:/var/tmp/dist/cmake/bin:/var/tmp/dist/ninja:$PATH +export PATH="/var/tmp/dist/ninja:/var/tmp/dist/cmake/bin:$PATH" + +[% IF c("var/linux") -%] + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] + [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] +[% END -%]

mkdir -p /var/tmp/build tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz

-mv /var/tmp/build/[% project %]-[% c('version') %] $builddir +cd /var/tmp/build +tar -xf $rootdir/[% project %]-[% c('version') %].tar.gz cd $builddir -# We have no .git in our tar ball but the script version.sh used by -# the Makefile depends on that. We replace the script. + +pushd src +rmdir config llvm-project wasi-libc +tar -xf $rootdir/[% c('input_files_by_name/wasi-config') %] +mv wasi-config-* config +tar -xf $rootdir/[% c('input_files_by_name/clang-source') %] +mv clang-source llvm-project +export LLVM_HOME=$(pwd)/llvm-project +tar -xf $rootdir/[% c('input_files_by_name/wasi-libc') %] +mv wasi-libc-* wasi-libc +popd + +# What follows has been copied from Firefox's +# taskcluster/scripts/misc/build-sysroot-wasi.sh and adapted to +# tor-browser-build. + +mkdir -p build/install/wasi +# The wasi-sdk build system wants to build clang itself. We trick it into +# thinking it did, and put our own clang where it would have built its own. +tar -C build -xf $rootdir/[% c('input_files_by_name/clang') %] +mv build/clang build/llvm +touch build/llvm.BUILT + +# The wasi-sdk build system wants a clang and an ar binary in +# build/install/$PREFIX/bin +ln -s $(pwd)/build/llvm/bin build/install/wasi/bin +ln -s llvm-ar build/install/wasi/bin/ar + +# tor-browser-build: we have no .git in our tarball but the script version.sh +# used by the Makefile depends on that. We replace the script. cat > 'version.sh' << EOF #!/bin/sh echo '[% c("abbrev") %]' EOF

-# XXX: We don't want to use the clang 9.0 wasi-sdk would use, see: -# https://bugzilla.mozilla.org/show_bug.cgi?id=1605708 -make -j[% c("num_procs") %] PREFIX=$distdir/[% project %] DESTDIR=/ +# Build wasi-libc, libc++ and libc++abi. +make \ + LLVM_PROJ_DIR=$LLVM_HOME \ + PREFIX=/wasi \ + build/wasi-libc.BUILT \ + build/compiler-rt.BUILT \ + NINJA_FLAGS='-j[% c("num_procs") %]' + +# Does Mozilla maybe do this when compiling Clang? +rtdir="build/llvm/lib/clang/$(ls build/llvm/lib/clang)/lib/wasi" +mkdir -p "$rtdir" +cp build/compiler-rt/lib/wasi/libclang_rt.builtins-wasm32.a "$rtdir/" + +make \ + LLVM_PROJ_DIR=$LLVM_HOME \ + PREFIX=/wasi \ + build/libcxx.BUILT \ + NINJA_FLAGS='-j[% c("num_procs") %]' + +mv build/install/wasi/share $distdir/ +mv build/install/wasi/lib $distdir/

-cd $distdir +cd /var/tmp/dist [% c('tar', { tar_src => [ project ], tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),

===================================== projects/wasi-sysroot/config ===================================== @@ -1,22 +1,35 @@ # vim: filetype=yaml sw=2 -filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' +filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' version: '[% c("abbrev") %]' -git_hash: b36c433738f0c29160a5ac1c1cee1b1b884bf4a0 -git_url: https://github.com/CraneStation/wasi-sdk -git_submodule: 1 +# Notice: we are not enabling submodules because we use the same hacks as +# Mozilla and re-use the LLVM we already build. +# So, be sure to update also wasi-libc and wasi-config when you update this +# git hash! +git_hash: 37ae6af88201884c8b98a65adcd6d8cd1284f95e +git_url: https://github.com/WebAssembly/wasi-sdk.git +git_submodule: 0

container: use_container: 1

input_files: - project: container-image - - name: binutils - project: binutils - enable: '[% c("var/linux") %]' - - project: '[% c("var/compiler") %]' - name: '[% c("var/compiler") %]' + # We do not use GCC, but we need its libsdc++ to run Clang + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' enable: '[% c("var/linux") %]' + - name: clang + project: clang + - project: llvm-project + name: clang-source - name: cmake project: cmake - name: ninja project: ninja + - name: python + project: python + enable: '[% c("var/linux") %]' + - name: wasi-config + project: wasi-config + - name: wasi-libc + project: wasi-libc

===================================== rbm.conf ===================================== @@ -168,6 +168,8 @@ var: - zh-rCN - zh-rTW

+ rlbox: 1 + sign_build: '[% ENV.RBM_SIGN_BUILD %]' sign_build_gpg_opts: '[% ENV.RBM_GPG_OPTS %]'

@@ -401,8 +403,6 @@ targets: osname: linux-x86_64 linux-cross: 0 arch_debian: amd64 - # We only support RLBox on the nightly channel and x86_64 for now - rlbox: 0 linux-i686: arch: i686 var: @@ -569,7 +569,6 @@ targets: FLAGS: "-target [% c('var/build_target') %] -B $cctoolsdir -isysroot $sysrootdir [% IF c('var/macos-aarch64') %]-mcpu=apple-m1[% END %]" LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie" locale_ja: ja-JP-mac - rlbox: 0 deps: - build-essential - faketime

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/d9...