[tor-commits] [Git][tpo/applications/tor-browser-build][main] Bug 40353: Re-enable rlbox
Pier Angelo Vendrame (@pierov)
git at gitlab.torproject.org
Thu Jan 19 14:18:53 UTC 2023
Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build
Commits:
d940f9db by Pier Angelo Vendrame at 2023-01-19T15:18:08+01:00
Bug 40353: Re-enable rlbox
Updated the creation of the WASI sysroot, to be more similar to what
Mozilla does in shell scripts included in Firefox source code.
This includes the same hacks, like re-using the LLVM we already build.
We were still using lucetc, which is not needed anymore. Therefore,
this commit removes it.
- - - - -
17 changed files:
- Makefile
- projects/clang/build
- projects/firefox/build
- projects/firefox/config
- projects/firefox/list_toolchain_updates_checks
- projects/geckoview/build
- projects/geckoview/config
- − projects/lucetc/Remove-march-native-from-COMMON_CFLAGS.patch
- − projects/lucetc/build
- − projects/lucetc/config
- + projects/wasi-config/build
- + projects/wasi-config/config
- + projects/wasi-libc/build
- + projects/wasi-libc/config
- projects/wasi-sysroot/build
- projects/wasi-sysroot/config
- rbm.conf
Changes:
=====================================
Makefile
=====================================
@@ -274,9 +274,6 @@ cargo_vendor-cbindgen-android: submodule-update
cargo_vendor-cbindgen: submodule-update
$(rbm) build cbindgen --step cargo_vendor --target nightly --target torbrowser-linux-x86_64
-cargo_vendor-lucetc: submodule-update
- $(rbm) build lucetc --step cargo_vendor --target nightly --target torbrowser-linux-x86_64
-
cargo_vendor-uniffi-rs: submodule-update
$(rbm) build uniffi-rs --step cargo_vendor --target nightly --target torbrowser-linux-x86_64
=====================================
projects/clang/build
=====================================
@@ -24,7 +24,7 @@ cd build
cmake ../llvm -GNinja \
-DCMAKE_INSTALL_PREFIX=$distdir \
-DCMAKE_BUILD_TYPE=Release \
- -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64[% IF c("var/rlbox") -%];WebAssembly[% END %]" \
+ -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64;WebAssembly" \
-DLLVM_ENABLE_PROJECTS="clang;lld" \
-DLLVM_ENABLE_ZLIB=ON \
-DLLVM_INSTALL_UTILS=ON \
=====================================
projects/firefox/build
=====================================
@@ -49,10 +49,7 @@ export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bi
rtdir=/var/tmp/dist/clang/lib/clang/[% pc("clang", "version") %]/lib/wasi
[% END -%]
mkdir -p $rtdir
- cp /var/tmp/dist/wasi-sysroot/lib/clang/11.0.0/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
- tar -C /var/tmp/dist -xf [% c('input_files_by_name/lucetc') %]
- export PATH="/var/tmp/dist/lucetc/bin:$PATH"
- export WASM_SANDBOXED_LIBRARIES=graphite,ogg
+ cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
[% END -%]
=====================================
projects/firefox/config
=====================================
@@ -158,9 +158,6 @@ input_files:
- project: wasi-sysroot
name: wasi-sysroot
enable: '[% c("var/rlbox") %]'
- - project: lucetc
- name: lucetc
- enable: '[% c("var/rlbox") %]'
- project: node
name: node
- project: nasm
=====================================
projects/firefox/list_toolchain_updates_checks
=====================================
@@ -213,15 +213,5 @@ needed=$(perl -MYAML::XS -e "$p")
current='[% pc("wasi-sysroot", "git_hash") %]'
check_update_needed wasi-sdk "$needed" "$current"
-
-# lucetc
-read -d '' p << 'EOF' || true
-my $d = YAML::XS::LoadFile('taskcluster/ci/fetch/toolchains.yml');
-print $d->{'lucetc-source'}{fetch}{revision};
-EOF
-needed=$(perl -MYAML::XS -e "$p")
-current='[% pc("lucetc", "git_hash") %]'
-check_update_needed lucetc "$needed" "$current"
-
# End RLBox
[% END -%]
=====================================
projects/geckoview/build
=====================================
@@ -26,6 +26,18 @@ cp -r /var/tmp/dist/compiler-rt/* /var/tmp/dist/clang/
tar -C /var/tmp/dist -xf [% c('input_files_by_name/binutils') %]
export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH"
+[% IF c("var/rlbox") -%]
+ tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %]
+ # XXX: We need the libclang_rt.builtins-wasm32.a in our clang lib directory.
+ # Copy it over.
+ # https://searchfox.org/mozilla-central/source/build/build-clang/build-clang.py#890,
+ # include it directly in our clang
+ rtdir=/var/tmp/dist/clang/lib/clang/[% pc("clang", "version") %]/lib/wasi
+ mkdir -p $rtdir
+ cp /var/tmp/dist/wasi-sysroot/lib/clang/*/lib/wasi/libclang_rt.builtins-wasm32.a $rtdir
+ export WASI_SYSROOT=/var/tmp/dist/wasi-sysroot/share/wasi-sysroot
+[% END -%]
+
tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
cd /var/tmp/build/[% project %]-[% c("version") %]
=====================================
projects/geckoview/config
=====================================
@@ -173,6 +173,9 @@ input_files:
name: clang
- project: 'compiler-rt'
name: 'compiler-rt'
+ - project: wasi-sysroot
+ name: wasi-sysroot
+ enable: '[% c("var/rlbox") %]'
- filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
name: gradle-dependencies
exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
=====================================
projects/lucetc/Remove-march-native-from-COMMON_CFLAGS.patch deleted
=====================================
@@ -1,24 +0,0 @@
-From ce82762aef3f8cdbdd340f39f95d2491f69a7da0 Mon Sep 17 00:00:00 2001
-From: Nicolas Vigier <boklm at torproject.org>
-Date: Mon, 29 Mar 2021 13:42:48 +0200
-Subject: [PATCH] Remove -march=native from COMMON_CFLAGS
-
-The -march=native flag is causing an error on some hardware. It can also
-make the build non-reproducible when not building on the same hardware.
----
- lucet-builtins/Makefile | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/lucet-builtins/Makefile b/lucet-builtins/Makefile
-index 6ddbcfd..de7e540 100644
---- a/lucet-builtins/Makefile
-+++ b/lucet-builtins/Makefile
-@@ -7,7 +7,7 @@ LIBBUILTINS_OBJS:=$(addprefix build/, \
- strcmp.o \
- strlen.o )
-
--COMMON_CFLAGS:= --std=gnu99 -Ofast -Wall -Werror -march=native -fPIC \
-+COMMON_CFLAGS:= --std=gnu99 -Ofast -Wall -Werror -fPIC \
- -I../lucet-runtime/include
-
- default: build/libbuiltins.so
=====================================
projects/lucetc/build deleted
=====================================
@@ -1,57 +0,0 @@
-#!/bin/bash
-[% c("var/set_default_env") -%]
-# XXX: Do we need the GCC part here if we set CC to clang anyway?
-[% IF c("var/linux") %]
- [% pc('gcc', 'var/setup', { compiler_tarfile => c('input_files_by_name/gcc'),
- hardened_gcc => 0 }) %]
- tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
- export PATH="/var/tmp/dist/binutils/bin:$PATH"
-[% END -%]
-distdir=/var/tmp/dist
-builddir=/var/tmp/build/[% project %]
-
-mkdir -p $distdir
-tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
-tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %]
-tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %]
-tar -C /var/tmp/dist -xf [% c('input_files_by_name/wasi-sysroot') %]
-export PATH="/var/tmp/dist/cmake/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/rust/bin:$PATH"
-export CC=$distdir/clang/bin/clang
-export CFLAGS="-L$distdir/clang/lib"
-export CXX=$distdir/clang/bin/clang++
-export CXXFLAGS=$CFLAGS
-export AR=$distdir/clang/bin/llvm-ar
-export RUSTFLAGS="-C linker=$CXX -C link-arg=$CXXFLAGS"
-export LUCET_PREFIX=$distdir/[% project %]
-# XXX: Why do we need that when Mozilla doesn't? See: taskcluster/scripts/misc/build-lucetc.sh
-export WASI_SYSROOT=$distdir/wasi-sysroot
-
-mkdir -p /var/tmp/build
-tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
-
-# Now prepare the offline build
-# Move the directory for hardcoding the path in .cargo/config
-mv /var/tmp/build/[% project %]-[% c('version') %] $builddir
-tar -C $builddir -xf [% c('input_files_by_name/lucetc-vendor') %]
-cd $builddir
-# .cargo exists already, let's just add to it
-cat >> .cargo/config << 'EOF'
-
-[source.crates-io]
-replace-with = "vendored-sources"
-
-[source.vendored-sources]
-directory = "/var/tmp/build/lucetc/vendor"
-EOF
-
-patch -p1 < $rootdir/Remove-march-native-from-COMMON_CFLAGS.patch
-
-make -j[% c("num_procs") %] build
-# XXX: Should not be needed, just the lucetc binary, see: taskcluster/scripts/misc/build-lucetc.sh
-make install
-
-cd $distdir
-[% c('tar', {
- tar_src => [ project ],
- tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
- }) %]
=====================================
projects/lucetc/config deleted
=====================================
@@ -1,43 +0,0 @@
-# vim: filetype=yaml sw=2
-filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
-version: '[% c("abbrev") %]'
-git_hash: d582d76bf0d17984501f3933688bf7a1eb5cf208
-git_url: https://github.com/PLSysSec/lucet_sandbox_compiler/
-git_submodule: 1
-
-container:
- use_container: 1
-
-input_files:
- - project: container-image
- - name: binutils
- project: binutils
- enable: '[% c("var/linux") %]'
- # Our clang still needs GCC parts (libstdc++ for instance) so we include that
- # one here even though compilation gets done solely by clang.
- - project: '[% c("var/compiler") %]'
- name: '[% c("var/compiler") %]'
- enable: '[% c("var/linux") %]'
- - name: cmake
- project: cmake
- - name: clang
- project: clang
- - name: rust
- project: rust
- - name: wasi-sysroot
- project: wasi-sysroot
- # Use `make cargo_vendor-lucetc` to re-generate the vendor tarball
- - name: lucetc-vendor
- URL: 'https://people.torproject.org/~boklm/mirrors/sources/lucetc-vendor-[% c("version") %].tar.bz2'
- sha256sum: 6b044ed76bc8870040174c8fbfa49c3a791b3aba7d32dd00cad541025eb16077
- - filename: Remove-march-native-from-COMMON_CFLAGS.patch
-
-steps:
- cargo_vendor:
- filename: '[% project %]-vendor-[% c("version") %].tar.bz2'
- input_files:
- - project: container-image
- pkg_type: build
- - project: rust
- name: rust
- pkg_type: build
=====================================
projects/wasi-config/build
=====================================
@@ -0,0 +1,4 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+mkdir -p /var/tmp/dist
+mv $rootdir/[% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c("filename") %]
=====================================
projects/wasi-config/config
=====================================
@@ -0,0 +1,5 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://git.savannah.gnu.org/git/config.git
+git_hash: c179db1b6f2ae484bfca1e9f8bae273e3319fa7d
+filename: '[% project %]-[% c("version") %].tar.gz'
=====================================
projects/wasi-libc/build
=====================================
@@ -0,0 +1,4 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+mkdir -p /var/tmp/dist
+mv $rootdir/[% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c("filename") %]
=====================================
projects/wasi-libc/config
=====================================
@@ -0,0 +1,5 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://github.com/WebAssembly/wasi-libc.git
+git_hash: 9886d3d6200fcc3726329966860fc058707406cd
+filename: '[% c("project") %]-[% c("version") %].tar.gz'
=====================================
projects/wasi-sysroot/build
=====================================
@@ -1,35 +1,81 @@
#!/bin/bash
[% c("var/set_default_env") -%]
-[% IF c("var/linux") %]
- [% pc('gcc', 'var/setup', { compiler_tarfile => c('input_files_by_name/gcc'),
- hardened_gcc => 0 }) %]
- tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
- export PATH="/var/tmp/dist/binutils/bin:$PATH"
-[% END -%]
-distdir=/var/tmp/dist
-builddir=/var/tmp/build/[% project %]
+distdir=/var/tmp/dist/[% project %]
+builddir=/var/tmp/build/[% project %]-[% c("abbrev") %]
mkdir -p $distdir
tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
tar -C /var/tmp/dist -xf [% c('input_files_by_name/ninja') %]
-export PATH=/var/tmp/dist/binutils/bin:/var/tmp/dist/cmake/bin:/var/tmp/dist/ninja:$PATH
+export PATH="/var/tmp/dist/ninja:/var/tmp/dist/cmake/bin:$PATH"
+
+[% IF c("var/linux") -%]
+ [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+ [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
+[% END -%]
mkdir -p /var/tmp/build
tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
-mv /var/tmp/build/[% project %]-[% c('version') %] $builddir
+cd /var/tmp/build
+tar -xf $rootdir/[% project %]-[% c('version') %].tar.gz
cd $builddir
-# We have no .git in our tar ball but the script version.sh used by
-# the Makefile depends on that. We replace the script.
+
+pushd src
+rmdir config llvm-project wasi-libc
+tar -xf $rootdir/[% c('input_files_by_name/wasi-config') %]
+mv wasi-config-* config
+tar -xf $rootdir/[% c('input_files_by_name/clang-source') %]
+mv clang-source llvm-project
+export LLVM_HOME=$(pwd)/llvm-project
+tar -xf $rootdir/[% c('input_files_by_name/wasi-libc') %]
+mv wasi-libc-* wasi-libc
+popd
+
+# What follows has been copied from Firefox's
+# taskcluster/scripts/misc/build-sysroot-wasi.sh and adapted to
+# tor-browser-build.
+
+mkdir -p build/install/wasi
+# The wasi-sdk build system wants to build clang itself. We trick it into
+# thinking it did, and put our own clang where it would have built its own.
+tar -C build -xf $rootdir/[% c('input_files_by_name/clang') %]
+mv build/clang build/llvm
+touch build/llvm.BUILT
+
+# The wasi-sdk build system wants a clang and an ar binary in
+# build/install/$PREFIX/bin
+ln -s $(pwd)/build/llvm/bin build/install/wasi/bin
+ln -s llvm-ar build/install/wasi/bin/ar
+
+# tor-browser-build: we have no .git in our tarball but the script version.sh
+# used by the Makefile depends on that. We replace the script.
cat > 'version.sh' << EOF
#!/bin/sh
echo '[% c("abbrev") %]'
EOF
-# XXX: We don't want to use the clang 9.0 wasi-sdk would use, see:
-# https://bugzilla.mozilla.org/show_bug.cgi?id=1605708
-make -j[% c("num_procs") %] PREFIX=$distdir/[% project %] DESTDIR=/
+# Build wasi-libc, libc++ and libc++abi.
+make \
+ LLVM_PROJ_DIR=$LLVM_HOME \
+ PREFIX=/wasi \
+ build/wasi-libc.BUILT \
+ build/compiler-rt.BUILT \
+ NINJA_FLAGS='-j[% c("num_procs") %]'
+
+# Does Mozilla maybe do this when compiling Clang?
+rtdir="build/llvm/lib/clang/$(ls build/llvm/lib/clang)/lib/wasi"
+mkdir -p "$rtdir"
+cp build/compiler-rt/lib/wasi/libclang_rt.builtins-wasm32.a "$rtdir/"
+
+make \
+ LLVM_PROJ_DIR=$LLVM_HOME \
+ PREFIX=/wasi \
+ build/libcxx.BUILT \
+ NINJA_FLAGS='-j[% c("num_procs") %]'
+
+mv build/install/wasi/share $distdir/
+mv build/install/wasi/lib $distdir/
-cd $distdir
+cd /var/tmp/dist
[% c('tar', {
tar_src => [ project ],
tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
=====================================
projects/wasi-sysroot/config
=====================================
@@ -1,22 +1,35 @@
# vim: filetype=yaml sw=2
-filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz'
version: '[% c("abbrev") %]'
-git_hash: b36c433738f0c29160a5ac1c1cee1b1b884bf4a0
-git_url: https://github.com/CraneStation/wasi-sdk
-git_submodule: 1
+# Notice: we are not enabling submodules because we use the same hacks as
+# Mozilla and re-use the LLVM we already build.
+# So, be sure to update also wasi-libc and wasi-config when you update this
+# git hash!
+git_hash: 37ae6af88201884c8b98a65adcd6d8cd1284f95e
+git_url: https://github.com/WebAssembly/wasi-sdk.git
+git_submodule: 0
container:
use_container: 1
input_files:
- project: container-image
- - name: binutils
- project: binutils
- enable: '[% c("var/linux") %]'
- - project: '[% c("var/compiler") %]'
- name: '[% c("var/compiler") %]'
+ # We do not use GCC, but we need its libsdc++ to run Clang
+ - name: '[% c("var/compiler") %]'
+ project: '[% c("var/compiler") %]'
enable: '[% c("var/linux") %]'
+ - name: clang
+ project: clang
+ - project: llvm-project
+ name: clang-source
- name: cmake
project: cmake
- name: ninja
project: ninja
+ - name: python
+ project: python
+ enable: '[% c("var/linux") %]'
+ - name: wasi-config
+ project: wasi-config
+ - name: wasi-libc
+ project: wasi-libc
=====================================
rbm.conf
=====================================
@@ -168,6 +168,8 @@ var:
- zh-rCN
- zh-rTW
+ rlbox: 1
+
sign_build: '[% ENV.RBM_SIGN_BUILD %]'
sign_build_gpg_opts: '[% ENV.RBM_GPG_OPTS %]'
@@ -401,8 +403,6 @@ targets:
osname: linux-x86_64
linux-cross: 0
arch_debian: amd64
- # We only support RLBox on the nightly channel and x86_64 for now
- rlbox: 0
linux-i686:
arch: i686
var:
@@ -569,7 +569,6 @@ targets:
FLAGS: "-target [% c('var/build_target') %] -B $cctoolsdir -isysroot $sysrootdir [% IF c('var/macos-aarch64') %]-mcpu=apple-m1[% END %]"
LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie"
locale_ja: ja-JP-mac
- rlbox: 0
deps:
- build-essential
- faketime
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/d940f9dbe8891cfd203301f0f8b7419aa74e823d
--
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/d940f9dbe8891cfd203301f0f8b7419aa74e823d
You're receiving this email because of your account on gitlab.torproject.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.torproject.org/pipermail/tor-commits/attachments/20230119/ca7cf908/attachment-0001.htm>
More information about the tor-commits
mailing list