commit 9696b8e15ce1c766311c43b821997654e074af08 Author: Nicolas Vigier boklm@torproject.org Date: Mon Apr 5 13:15:17 2021 +0200
Bug 34108: Add scripts to check for needed toolchain updates
Checking toolchain updates can be done with the following commands:
make list_toolchain_updates-geckoview make list_toolchain_updates-application-services make list_toolchain_updates-android-components make list_toolchain_updates-fenix --- Makefile | 12 ++ projects/android-components/config | 20 ++ .../list_toolchain_updates_checks | 85 +++++++++ projects/android-toolchain/config | 3 +- projects/application-services/config | 7 + .../list_toolchain_updates_checks | 97 ++++++++++ projects/common/list_toolchain_updates | 38 ++++ projects/fenix/config | 7 + projects/fenix/list_toolchain_updates_checks | 73 ++++++++ projects/geckoview/config | 5 + projects/geckoview/list_toolchain_updates_checks | 202 +++++++++++++++++++++ projects/glean/config | 9 + projects/llvm-project/config | 2 +- rbm.conf | 3 + 14 files changed, 561 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile index 7e2e104..fa613e1 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,18 @@ dmg2mar-alpha: submodule-update tools/update-responses/download_missing_versions alpha CHECK_CODESIGNATURE_EXISTS=1 MAR_SKIP_EXISTING=1 tools/update-responses/gen_incrementals alpha
+list_toolchain_updates-fenix: submodule-update + $(rbm) build fenix --step list_toolchain_updates --target nightly --target torbrowser-android-armv7 + +list_toolchain_updates-android-components: submodule-update + $(rbm) build android-components --step list_toolchain_updates --target nightly --target torbrowser-android-armv7 + +list_toolchain_updates-application-services: submodule-update + $(rbm) build application-services --step list_toolchain_updates --target nightly --target torbrowser-android-armv7 + +list_toolchain_updates-geckoview: submodule-update + $(rbm) build geckoview --step list_toolchain_updates --target nightly --target torbrowser-android-armv7 + submodule-update: git submodule update --init
diff --git a/projects/android-components/config b/projects/android-components/config index eb1b7de..c9c5d55 100644 --- a/projects/android-components/config +++ b/projects/android-components/config @@ -50,3 +50,23 @@ input_files: - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-%5B% c('var/glean_parser') %].tar.bz2 sha256sum: 19dbdd4958022a1a638e0217489ab722fe7d4f588f1978a4ae162f93e75694c2 enable: '[% !c("var/fetch_gradle_dependencies") %]' + +steps: + list_toolchain_updates: + var: + container: + use_container: 0 + get_glean_version: | + #!/bin/bash + read -d '' p << 'EOF' || true + if (m/const\sval\smozilla_glean\s=\s"([^"]+)"/) { + print $1; + exit; + } + EOF + perl -ne "$p" < buildSrc/src/main/java/Dependencies.kt + glean_version: '[% exec(c("var/get_glean_version")) %]' + input_files: + - name: glean + project: glean + pkg_type: src diff --git a/projects/android-components/list_toolchain_updates_checks b/projects/android-components/list_toolchain_updates_checks new file mode 100644 index 0000000..a46afb4 --- /dev/null +++ b/projects/android-components/list_toolchain_updates_checks @@ -0,0 +1,85 @@ +#!/bin/bash + +# compileSdkVersion +read -d '' p << 'EOF' || true +if (m/compileSdkVersion:\s+(.*)$/) { + print $1; + exit; +} +EOF +needed=$(cat .config.yml | perl -ne "$p") +current=30 +check_update_needed compileSdkVersion "$needed" "$current" + + +# targetSdkVersion +read -d '' p << 'EOF' || true +if (m/targetSdkVersion:\s+(.*)$/) { + print $1; + exit; +} +EOF +needed=$(cat .config.yml | perl -ne "$p") +current=30 +check_update_needed targetSdkVersion "$needed" "$current" + + +# minSdkVersion +read -d '' p << 'EOF' || true +if (m/minSdkVersion:\s+(.*)$/) { + print $1; + exit; +} +EOF +needed=$(cat .config.yml | perl -ne "$p") +current=21 +check_update_needed minSdkVersion "$needed" "$current" + + +# application-services +read -d '' p << 'EOF' || true +if (m/const\sval\smozilla_appservices\s=\s"([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/Dependencies.kt | perl -ne "$p") +current='[% pc("application-services", "version") %]' +check_update_needed application-services "$needed" "$current" + + +# glean_parser +read -d '' p << 'EOF' || true +if (m/^\s*"glean_parser==([^"]+)",/) { + print $1; + exit; +} +EOF +tar xf $rootdir/[% c('input_files_by_name/glean') %] +needed=$(cat glean-[% c("var/glean_version") %]/glean-core/python/setup.py | perl -ne "$p") +current='[% c("var/glean_parser") %]' +check_update_needed glean_parser "$needed" "$current" + + +# gradle +read -d '' p << 'EOF' || true +if (m|distributionUrl=https\\://services.gradle.org/distributions/gradle-(.*)-all.zip|) { + print $1; + exit; +} +EOF +needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p") +current='[% c("var/gradle_version") %]' +check_update_needed gradle "$needed" "$current" + + +# android-gradle-plugin +read -d '' p << 'EOF' || true +if (m/const\s+val\s+android_gradle_plugin\s+=\s+"([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/Dependencies.kt | perl -ne "$p") +current='4.0.1' +check_update_needed android-gradle-plugin "$needed" "$current" diff --git a/projects/android-toolchain/config b/projects/android-toolchain/config index dc30b4d..6d78b37 100644 --- a/projects/android-toolchain/config +++ b/projects/android-toolchain/config @@ -31,6 +31,7 @@ var: android_platform_revision_new: 03 android_ndk_version: 21 android_ndk_revision: d + sdk_tools_version: 4333796 input_files: - project: container-image - URL: '[% c("var/google_repo") %]/build-tools_r[% c("version") %]-linux.zip' @@ -54,7 +55,7 @@ input_files: - URL: '[% c("var/google_repo") %]/platform-tools_r[% c("version") %]-linux.zip' name: platform_tools sha256sum: 633b6dfa245f5dc58d15da9ead655bcd14de5784196ec0f0dc7e37a5acb61be4 - - URL: '[% c("var/google_repo") %]/sdk-tools-linux-4333796.zip' + - URL: '[% c("var/google_repo") %]/sdk-tools-linux-[% c("var/sdk_tools_version") %].zip' name: android_sdk_tools sha256sum: 92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9 - URL: '[% c("var/google_repo") %]/android-ndk-r[% c("var/android_ndk_version") %][% c("var/android_ndk_revision") %]-linux-x86_64.zip' diff --git a/projects/application-services/config b/projects/application-services/config index 951d5b0..6a80ffc 100644 --- a/projects/application-services/config +++ b/projects/application-services/config @@ -81,3 +81,10 @@ input_files: - filename: viaduct-workaround.patch - filename: viaduct-workaround2.patch - filename: update-cargo-lock.patch + +steps: + list_toolchain_updates: + input_files: [] + var: + container: + use_container: 0 diff --git a/projects/application-services/list_toolchain_updates_checks b/projects/application-services/list_toolchain_updates_checks new file mode 100644 index 0000000..69c232c --- /dev/null +++ b/projects/application-services/list_toolchain_updates_checks @@ -0,0 +1,97 @@ +#!/bin/bash + +# ndkVersion +read -d '' p << 'EOF' || true +if (m/^\s*ndkVersion:\s"([^"]*)",/) { + print $1; + exit; +} +EOF +needed=$(cat build.gradle | perl -ne "$p") +current='21.3.6528147' +check_update_needed ndkVersion "$needed" "$current" + + +# compileSdkVersion +read -d '' p << 'EOF' || true +if (m/^\s*compileSdkVersion:\s([^"]*),/) { + print $1; + exit; +} +EOF +needed=$(cat build.gradle | perl -ne "$p") +current=29 +check_update_needed compileSdkVersion "$needed" "$current" + + +# minSdkVersion +read -d '' p << 'EOF' || true +if (m/^\s*minSdkVersion:\s([^"]*),/) { + print $1; + exit; +} +EOF +needed=$(cat build.gradle | perl -ne "$p") +current=21 +check_update_needed minSdkVersion "$needed" "$current" + + +# gradle +read -d '' p << 'EOF' || true +if (m|distributionUrl=https\\://services.gradle.org/distributions/gradle-(.*)-.*.zip|) { + print $1; + exit; +} +EOF +needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p") +current='[% c("var/gradle_version") %]' +check_update_needed gradle "$needed" "$current" + + +# nss-nspr +read -d '' p << 'EOF' || true +if (m/NSS_ARCHIVE="nss-(.*-with-nspr-.*)\.tar\.gz"/) { + print $1; + exit; +} +EOF +needed=$(cat libs/build-all.sh | perl -ne "$p") +current='[% pc("nss", "version") %]-with-nspr-[% pc("nss", "nspr_version") %]' +check_update_needed nss-nspr "$needed" "$current" + + +# sqlcipher +read -d '' p << 'EOF' || true +if (m/SQLCIPHER_VERSION="([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat libs/build-all.sh | perl -ne "$p") +current='[% pc("sqlcipher", "version") %]' +check_update_needed sqlcipher "$needed" "$current" + + +# android-gradle-plugin +read -d '' p << 'EOF' || true +if (m/^\s*ext\.android_gradle_plugin_version\s=\s'([^']*)'/) { + print $1; + exit; +} +EOF +needed=$(cat build.gradle | perl -ne "$p") +current='4.0.1' +check_update_needed android-gradle-plugin "$needed" "$current" + + +# android-components +read -d '' p << 'EOF' || true +if (m/^\s*ext\.android_components_version\s=\s'([^']*)'/) { + print $1; + exit; +} +EOF +needed=$(cat build.gradle | perl -ne "$p") +current=71.0.0 +check_update_needed android-components "$needed" "$current" + diff --git a/projects/common/list_toolchain_updates b/projects/common/list_toolchain_updates new file mode 100644 index 0000000..4cfd829 --- /dev/null +++ b/projects/common/list_toolchain_updates @@ -0,0 +1,38 @@ +#!/usr/bin/bash + +set -e + +rootdir="$(pwd)" +no_updates="$rootdir/no_updates.txt" +updates="$rootdir/updates.txt" + +tar -xf [% project %]-[% c('version') %].tar.gz +cd [% project %]-[% c('version') %] + +function check_update_needed() { + name="$1" + v1="$2" + v2="$3" + if test "$v1" = "$v2" + then + echo "* $name ($v1)" >> "$no_updates" + else + echo "* $name needs to be updated to $v1 (currently at $v2)" >> "$updates" + fi +} + +[% INCLUDE list_toolchain_updates_checks %] + +echo '### Component: [% project %] ([% c("git_hash") %])' + +if test -f "$updates" +then + echo "The following components need to be updated:" + cat "$updates" + echo +fi +if test -f "$no_updates" +then + echo "The following components don't need to be updated:" + cat "$no_updates" +fi diff --git a/projects/fenix/config b/projects/fenix/config index cabdb55..a72fff4 100644 --- a/projects/fenix/config +++ b/projects/fenix/config @@ -61,3 +61,10 @@ input_files: name: tor-android-service - project: tor-onion-proxy-library name: topl + +steps: + list_toolchain_updates: + input_files: [] + var: + container: + use_container: 0 diff --git a/projects/fenix/list_toolchain_updates_checks b/projects/fenix/list_toolchain_updates_checks new file mode 100644 index 0000000..43dd1b4 --- /dev/null +++ b/projects/fenix/list_toolchain_updates_checks @@ -0,0 +1,73 @@ +#!/bin/bash + +# android-components +read -d '' p << 'EOF' || true +if (m/const\s+val\s+VERSION\s+=\s+"([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/AndroidComponents.kt | perl -ne "$p") +current='[% pc("android-components", "var/android_components_version") %]' +check_update_needed android-components "$needed" "$current" + + +# targetSdkVersion +read -d '' p << 'EOF' || true +if (m/const\s+val\s+targetSdkVersion\s+=\s+([^"]+)/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/Config.kt | perl -ne "$p") +current=29 +check_update_needed targetSdkVersion "$needed" "$current" + + +# compileSdkVersion +read -d '' p << 'EOF' || true +if (m/const\s+val\s+compileSdkVersion\s+=\s+([^"]+)/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/Config.kt | perl -ne "$p") +current=29 +check_update_needed compileSdkVersion "$needed" "$current" + + +# minSdkVersion +read -d '' p << 'EOF' || true +if (m/const\s+val\s+minSdkVersion\s+=\s+([^"]+)/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/Config.kt | perl -ne "$p") +current=21 +check_update_needed compileSdkVersion "$needed" "$current" + + +# gradle +read -d '' p << 'EOF' || true +if (m|distributionUrl=https\\://services.gradle.org/distributions/gradle-(.*)-all.zip|) { + print $1; + exit; +} +EOF +needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p") +current='[% c("var/gradle_version") %]' +check_update_needed gradle "$needed" "$current" + + +# android-gradle-plugin +read -d '' p << 'EOF' || true +if (m/const\s+val\s+android_gradle_plugin\s+=\s+"([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat buildSrc/src/main/java/Dependencies.kt | perl -ne "$p") +current='4.0.1' +check_update_needed android-gradle-plugin "$needed" "$current" + diff --git a/projects/geckoview/config b/projects/geckoview/config index a893d31..d186c8b 100644 --- a/projects/geckoview/config +++ b/projects/geckoview/config @@ -137,6 +137,11 @@ steps: target_prepend: - torbrowser-android-x86_64
+ list_toolchain_updates: + input_files: [] + var: + container: + use_container: 0
targets: nightly: diff --git a/projects/geckoview/list_toolchain_updates_checks b/projects/geckoview/list_toolchain_updates_checks new file mode 100644 index 0000000..ffc5e23 --- /dev/null +++ b/projects/geckoview/list_toolchain_updates_checks @@ -0,0 +1,202 @@ +#!/bin/bash + +# ndk version +read -d '' p << 'EOF' || true +if (m/^\s*NDK_VERSION\s*=\s*"(.+)"/) { + print $1; + exit; +} +EOF +needed=$(cat python/mozboot/mozboot/android.py | perl -ne "$p") +current='r[% pc("android-toolchain", "var/android_ndk_version") %][% pc("android-toolchain", "var/android_ndk_revision") %]' +check_update_needed ndk_version "$needed" "$current" + + +# rust +read -d '' p << 'EOF' || true +my $d = YAML::XS::LoadFile('taskcluster/ci/toolchain/rust.yml'); +foreach my $t (keys %$d) { + if ($d->{$t}{run}{'toolchain-alias'} eq 'linux64-rust-android') { + my $channel; + foreach my $arg (@{$d->{$t}{run}{arguments}}) { + if ($arg eq '--channel') { + $channel = 1; + next; + } + if ($channel) { + print $arg; + exit; + } + } + } +} +EOF +needed=$(perl -MYAML::XS -e "$p") +current='[% pc("rust", "version") %]' +check_update_needed rust "$needed" "$current" + + +# build_tools +read -d '' p << 'EOF' || true +if (m/build_tools_version\s*=\s*"([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat build/moz.configure/android-sdk.configure | perl -ne "$p") +current='[% pc("android-toolchain", "version") %]' +check_update_needed build_tools "$needed" "$current" + + +# target_sdk +read -d '' p << 'EOF' || true +if (m/target_sdk_version\s*=\s*"(.+)"/) { + print $1; + exit; +} +EOF +needed=$(cat build/moz.configure/android-sdk.configure | perl -ne "$p") +current='[% pc("android-toolchain", "var/android_api_level") %]' +check_update_needed target_sdk "$needed" "$current" + + +# sdk-tools +read -d '' p << 'EOF' || true +if (m|https://dl%5C%5C.google%5C%5C.com/android/repository/sdk-tools-.+-(%5B%5E-%5...) { + print $1; + exit; +} +EOF +needed=$(cat python/mozboot/mozboot/android.py | perl -ne "$p") +current='[% pc("android-toolchain", "var/sdk_tools_version") %]' +check_update_needed sdk-tools "$needed" "$current" + + +# min-android +read -d '' p << 'EOF' || true +use Path::Tiny; +use Digest::SHA qw(sha256_hex); +my $f; +my $min_indent; +foreach (path('build/moz.configure/android-ndk.configure')->lines_utf8) { + if ($_ eq "def min_android_version(target):\n") { + $f = $_; + next; + } else { + next unless $f; + } + m/^(\s*)/; + my $indent = length $1; + $min_indent = $indent unless $min_indent; + last if $indent < $min_indent; + $f .= $_; +} +print substr(sha256_hex($f), 0, 10); +EOF +needed=$(perl -e "$p") +# We can't easily parse the min_android_version function. +# Instead we get a checksum of the function, and manually check it when +# it was updated. +# Current value of min_android_version is: +# 21 on aarch64, x86_64 +# 16 on other archs +current=303de6de36 +check_update_needed min-android "$needed" "$current" + + +# min_sdk +read -d '' p << 'EOF' || true +if (m/^\s*MOZ_ANDROID_MIN_SDK_VERSION\s*=\s*([^\s]+)/) { + print $1; + exit; +} +EOF +needed=$(cat mobile/android/confvars.sh | perl -ne "$p") +current=16 +check_update_needed min_sdk "$needed" "$current" + + +# gradle +read -d '' p << 'EOF' || true +if (m|distributionUrl=https\\://services.gradle.org/distributions/gradle-(.*)-all.zip|) { + print $1; + exit; +} +EOF +needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p") +current='[% c("var/gradle_version") %]' +check_update_needed gradle "$needed" "$current" + + +# cbindgen +read -d '' p << 'EOF' || true +if (m/^\s*cbindgen_min_version\s*=\s*Version\("([^"]+)"\)/) { + print $1; + exit; +} +EOF +needed=$(cat build/moz.configure/bindgen.configure | perl -ne "$p") +current='[% pc("cbindgen", "version") %]' +check_update_needed cbindgen "$needed" "$current" + + +# nasm +read -d '' p << 'EOF' || true +if (m/^\s*MODERN_NASM_VERSION\s*=\s*LooseVersion\("([^"]+)"\)/) { + print $1; + exit; +} +EOF +needed=$(cat python/mozboot/mozboot/base.py | perl -ne "$p") +current='2.14' +check_update_needed nasm "$needed" "$current" + + +# clang +read -d '' p << 'EOF' || true +my $d = YAML::XS::LoadFile('taskcluster/ci/toolchain/clang.yml'); +my $clang_toolchain; +foreach my $t (keys %$d) { + if ($d->{$t}{run}{'toolchain-alias'} eq 'linux64-clang-android-cross') { + foreach my $fetch (@{$d->{$t}{fetches}{fetch}}) { + $clang_toolchain = $fetch if $fetch =~ m/^clang-.*/; + } + last; + } +} + +if (!$clang_toolchain) { + print STDERR "Error: could not find clang toolchain"; + exit 1; +} + +my $fetch = YAML::XS::LoadFile('taskcluster/ci/fetch/toolchains.yml'); +print $fetch->{$clang_toolchain}{fetch}{revision}; +EOF +needed=$(perl -MYAML::XS -e "$p") +current='[% pc("llvm-project", "git_hash") %]' +check_update_needed clang "$needed" "$current" + + +# node +read -d '' p << 'EOF' || true +if (m/^\s*NODE_MIN_VERSION\s*=\s*StrictVersion\("([^"]+)"\)/) { + print $1; + exit; +} +EOF +needed=$(cat python/mozbuild/mozbuild/nodeutil.py | perl -ne "$p") +current='[% pc("node", "version") %]' +check_update_needed node "$needed" "$current" + + +# python +read -d '' p << 'EOF' || true +if (m/find_python3_executable\(min_version\s*=\s*"([^"]+)"/) { + print $1; + exit; +} +EOF +needed=$(cat build/moz.configure/init.configure | perl -ne "$p") +current=3.6.0 +check_update_needed python "$needed" "$current" diff --git a/projects/glean/config b/projects/glean/config new file mode 100644 index 0000000..45bc43b --- /dev/null +++ b/projects/glean/config @@ -0,0 +1,9 @@ +# vim: filetype=yaml sw=2 +git_hash: 'v[% c("version") %]' +git_url: https://github.com/mozilla/glean.git +filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' +version: '[% pc(c("origin_project"), "var/glean_version", { step => "list_toolchain_updates" }) %]' + +src: | + #!/bin/bash + mv -f [% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c('filename') %] diff --git a/projects/llvm-project/config b/projects/llvm-project/config index 822f08d..c6fa2d8 100644 --- a/projects/llvm-project/config +++ b/projects/llvm-project/config @@ -1,5 +1,5 @@ # vim: filetype=yaml sw=2 version: 11.0.1 -git_hash: 43ff75f2c3feef64f9d73328230d34dac8832a9 +git_hash: 43ff75f2c3feef64f9d73328230d34dac8832a91 git_url: https://github.com/llvm/llvm-project filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' diff --git a/rbm.conf b/rbm.conf index cbc705a..646f353 100644 --- a/rbm.conf +++ b/rbm.conf @@ -15,6 +15,9 @@ steps: set -e mkdir -p '[% dest_dir %]' mv -vf '[% project %]-[% c("version") %].tar.xz' '[% dest_dir %]/[% c("filename") %]' + list_toolchain_updates: + build_log: '-' + list_toolchain_updates: '[% INCLUDE list_toolchain_updates %]'
# buildconf contains build options that the user can change in rbm.local.conf # When adding a new option to buildconf, a default value should be defined