commit e414bcd0a51859acc3fbdc48366b612b20425be3 Author: Georg Koppen gk@torproject.org Date: Thu Sep 3 09:46:55 2020 +0000

Bug 40055: Integrate building Glean in offline mode

Using Glean 32.1.1 (via android-components 56.0.0 or later) did not land yet in the application-services tag which Fenix and android-components depend on. Thus, we patch application-services to include the earliest version of android-components shipping Glean 32.1.1. --- projects/android-components/build | 11 ++++++-- projects/android-components/config | 6 ++++ projects/application-services/1651662.patch | 22 +++++++++++++++ projects/application-services/build | 8 ++++++ projects/application-services/config | 9 +++++- .../gradle-dependencies-list.txt | 32 +++++++++++----------- projects/fenix/build | 7 +++++ projects/fenix/config | 6 ++++ rbm.conf | 2 ++ 9 files changed, 84 insertions(+), 19 deletions(-)

diff --git a/projects/android-components/build b/projects/android-components/build index a024861..3cac44a 100644 --- a/projects/android-components/build +++ b/projects/android-components/build @@ -3,10 +3,11 @@ [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] distdir=/var/tmp/dist builddir=/var/tmp/build/[% project %] -mkdir -p $distdir/[% project %] -mkdir -p /var/tmp/build +mkdir $distdir/[% project %] +mkdir /var/tmp/build

[% IF !c("var/fetch_gradle_dependencies") %] + [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] # XXX: Make this more generic and not only for geckoview-beta. geckoview_version=$(find . -type f -name geckoview-beta-*.aar | cut -d - -f 6 | cut -d . -f 1-3) gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %] @@ -29,6 +30,12 @@ patch -p1 < $rootdir/git.patch # `assemble` instead for now. gradle --debug -no-daemon assemble [% ELSE %] + # Prepare Glean dependencies for offline build + tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2 + # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding + # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/ + export LC_ALL=C.UTF-8 + export LANG=C.UTF-8 patch -p1 < $rootdir/mavenLocal.patch

# Make sure our GeckoView dependency is used. XXX: Make this more generic and diff --git a/projects/android-components/config b/projects/android-components/config index 3bdca82..56380e6 100644 --- a/projects/android-components/config +++ b/projects/android-components/config @@ -23,6 +23,9 @@ input_files: - name: application-services project: application-services enable: '[% !c("var/fetch_gradle_dependencies") %]' + - name: python + project: python + enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies exec: '[% INCLUDE "fetch-gradle-dependencies" %]' @@ -30,3 +33,6 @@ input_files: - filename: git.patch - filename: mavenLocal.patch enable: '[% !c("var/fetch_gradle_dependencies") %]' + - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-%5B% c('var/glean_parser') %].tar.bz2 + sha256sum: 69234aa1ae38fd5dda4de71bd5e536795073703c3ed5bb994b9b8485bd7a55dc + enable: '[% !c("var/fetch_gradle_dependencies") %]' diff --git a/projects/application-services/1651662.patch b/projects/application-services/1651662.patch new file mode 100644 index 0000000..0275da2 --- /dev/null +++ b/projects/application-services/1651662.patch @@ -0,0 +1,22 @@ +From 1818764e69145839ab33a2f885139ea320f2283f Mon Sep 17 00:00:00 2001 +From: Georg Koppen gk@torproject.org +Date: Wed, 26 Aug 2020 15:56:49 +0000 +Subject: [PATCH] 1651662 + + +diff --git a/build.gradle b/build.gradle +index 58a4ddae..c6e0be82 100644 +--- a/build.gradle ++++ b/build.gradle +@@ -4,7 +4,7 @@ buildscript { + ext.kotlin_version = '1.3.71' + ext.jna_version = '5.2.0' + ext.android_gradle_plugin_version = '3.6.0' +- ext.android_components_version = '47.0.0' ++ ext.android_components_version = '56.0.0' + + ext.build = [ + ndkVersion: "21.3.6528147", // Keep it in sync in TC Dockerfile. +-- +2.28.0 + diff --git a/projects/application-services/build b/projects/application-services/build index 051163f..5c6b5e0 100644 --- a/projects/application-services/build +++ b/projects/application-services/build @@ -29,6 +29,7 @@ export ANDROID_NDK_API_VERSION=[% pc("android-toolchain", "var/android_ndk_versi export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64

[% IF !c('var/fetch_gradle_dependencies') %] + [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %] cp -r $gradle_repo/dl/android/maven2/* $gradle_repo cp -r $gradle_repo/m2/* $gradle_repo @@ -82,11 +83,18 @@ patch -p1 < $rootdir/no-git.patch # complexity. That's needed for running unit tests later on, though. patch -p1 < $rootdir/target.patch patch -p1 < $rootdir/1651660.patch +patch -p1 < $rootdir/1651662.patch [% IF c('var/fetch_gradle_dependencies') %] # XXX: `assemble` is still not enough to see all fetched dependencies via # Gradle's --debug. See: tor-browser-build#40056. $GRADLE_HOME/gradle-6.3/bin/gradle --debug --no-daemon assemble [% ELSE %] + # Prepare Glean dependencies for offline build + tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2 + # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding + # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/ + export LC_ALL=C.UTF-8 + export LANG=C.UTF-8 patch -p1 < $rootdir/mavenLocal.patch gradle_flags="--offline --no-daemon -Dmaven.repo.local=$gradle_repo" $GRADLE_HOME/gradle-6.3/bin/gradle $gradle_flags assembleRelease diff --git a/projects/application-services/config b/projects/application-services/config index 50986bb..1d9fd79 100644 --- a/projects/application-services/config +++ b/projects/application-services/config @@ -8,7 +8,7 @@ var: container: use_container: 1 # This should be updated when the list of gradle dependencies is changed. - gradle_dependencies_version: 1 + gradle_dependencies_version: 2 # This should be updated when the list of rust dependencies is changed. rust_vendor_version: 1 # For armv7 we need a different target than the default one we use for @@ -71,10 +71,16 @@ input_files: name: sqlcipher-x86_64 target_prepend: - torbrowser-android-x86_64 + - name: python + project: python + enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies exec: '[% INCLUDE "fetch-gradle-dependencies" %]' enable: '[% !c("var/fetch_gradle_dependencies") %]' + - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-%5B% c('var/glean_parser') %].tar.bz2 + sha256sum: 69234aa1ae38fd5dda4de71bd5e536795073703c3ed5bb994b9b8485bd7a55dc + enable: '[% !c("var/fetch_gradle_dependencies") %]' # `cargo vendor vendor` in the `application-services` directory has vendored # the sources. - URL: https://people.torproject.org/~gk/mirrors/sources/application-services-vendo... c('var/rust_vendor_version') %].tar.bz2 @@ -84,3 +90,4 @@ input_files: enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: target.patch - filename: 1651660.patch + - filename: 1651662.patch diff --git a/projects/application-services/gradle-dependencies-list.txt b/projects/application-services/gradle-dependencies-list.txt index ffad4f0..3978277 100644 --- a/projects/application-services/gradle-dependencies-list.txt +++ b/projects/application-services/gradle-dependencies-list.txt @@ -403,22 +403,22 @@ b51f8867c92b6a722499557fc3a1fdea77bdf9ef574722fe90ce436a29559454 | https://jcent fb40265f982548212ff82e362e59732b2187ec6f0d80182885c14ef1f982827a | https://jcenter.bintray.com/org/sonatype/oss/oss-parent/9/oss-parent-9.pom 13009fb5ede3cf2be5a8d0f1602155aeaa0ce5ef5f9366892bd258d8d3d4d2b1 | https://jcenter.bintray.com/org/yaml/snakeyaml/1.23/snakeyaml-1.23.jar 1e1beb22ca906209700122b5cacf6f2719324538c5b1e3c27bf91564c8d31dbd | https://jcenter.bintray.com/org/yaml/snakeyaml/1.23/snakeyaml-1.23.pom -4bca00bc684d935cf3675d3437113d1284bb94412b6b9aea03194f946c719a9c | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/47.0.0... -c2ec8bc73e7e1a86b53f1c8ac8e2f09eee5e31b6bdc291f31495e3080ab66261 | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/47.0.0... -5df22d5e7864c593c0b78624304d3046514177cd819bada0ce5927ed22e38bdf | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/47.0.0... -c062918be5f021628bf9705898a9e7598ed340f5ec971033ab0ea239ee6efaeb | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/47.0.0... -c959adf0ae40a073b3db4e21c1f6a9a961bbb7d6cc33c569492e2268b8a473a1 | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/47.0.0/... -0f47f76b38975b5407b212f1aa4dd1daabde311989ff934360afa529c1c77d92 | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/47.0.0/... -2813838b7bb4d2973b3663f2aac08b36c3db9a372d266eb228fba98bfcb09017 | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/47.0.0/s... -0d72828ec6af2fe0b7021c2cada48204a47ccd6e47eeea35dd2b51c24df3b8ff | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/47.0.0/s... -bd295b27f098eed0aadf2ebbf3daf2356c6ebe73c2cde49597bdf33414936699 | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/47.0.0... -10543160bfa1790c2d98ba8e0de1230e0feea1926fd4859751c931e3bd5e1c34 | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/47.0.0... -e24f78d36c64695407ace7b499472306161b1abf47b65c99ec790f321a6663e1 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle... -6320746e6d914aa45d9f8e0de23bdbd81c8097999b3dd64796363ed878633600 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle... -68f97a68faf84fa03cf3e1898042ffefe17b37c4c02df9e754fd2a47e6746196 | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/31.1.1/glean-31... -9d6e64de942f997293b5ed35337da9350798192b22d8928030f191174b9f77fa | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/31.1.1/glean-31... -3c2868a395d588eef2a370e4cd889a3a71351cff8eb4dd557dee764bc9f6e90a | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/3... -7517604c7643984718d421c85df395ae9f69f74c78ebd825930b4bceccf8196c | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/3... +e14c614b65ae4fe6c6d26ce369d523cf1ff837ce697e5e3f47e8cdfa355357f9 | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/56.0.0... +e7bd7b2e2013a080bcb3cb9db974185eb28dd87c0594394f951a51ab5d6fcf5d | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/56.0.0... +e79f67b5180f06a4680ffea310ceb2c5b0af205a88b3fb1cc76ce96e5b4a7ea7 | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/56.0.0... +32acc51221752051a03765ac0bbf26a1da663ba2f7b3fe69ae422d5aac228f81 | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/56.0.0... +ae35db5670efe458d3b9646cab337f2f7667becacdb023b599cc62bfa95b59ec | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/56.0.0/... +d87cd2eea092430c4e5a95ff04c3213be89f54aa61c2c7f80fb6548e6dc38bab | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/56.0.0/... +96aeede35017fd79576c5faa1cb41efc8137616e20231b41934999ca711f4407 | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/56.0.0/s... +dbd706151fe820ed233792a6773b70efcceb42987c209e27575ef91d4552b8c4 | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/56.0.0/s... +22934a560b0bc34f01304efbdccc988d2e0fca039ae577894fcec52a7fdfd88e | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/56.0.0... +a2c1e39e5e58f585c2908af2266953b6b43e51f2fabe45e7723bc177c9c49bc0 | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/56.0.0... +0165b5554aa3427a2e4026e9ca78c1ea5021f1578d36839b37350dbf0f192d29 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle... +878570d1c4e5b06d2434b7133a14ce97fb516b7871c7637c44a4b087db291a49 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle... +5b078e136164082be6d49afa0d87f849c7b9f65e21125fa57056cd630756279b | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/32.1.1/glean-32... +64371d71a4fb11d63d2a5ff60149c94c1d0e0b909de09ffe199f6872663976b2 | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/32.1.1/glean-32... +ab514359d7686282b7a7c8463e81237ff98460a23df534a6c9663b463e65423f | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/3... +f724de3fc1c37d8ecb4bbb7957dd67a489d0321da71eebb6b8d605ec4a57755a | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/3... 470564ff0a452da1c96be4ca3a9cdeceaa2912537ee2a8def486f3b383d43f15 | https://plugins.gradle.org/m2/com/jetbrains/python/envs/com.jetbrains.python... 6438812f927b505a4c4ef2703b45433a1b99e45334526bfed25e71e1a79abb14 | https://plugins.gradle.org/m2/gradle/plugin/com/jetbrains/python/gradle-pyth... 317aa4c467a37d5fcab6657c326a4069ea8e8c6dda951ec2091ffb95c596a2a3 | https://plugins.gradle.org/m2/gradle/plugin/com/jetbrains/python/gradle-pyth... diff --git a/projects/fenix/build b/projects/fenix/build index 5e483c9..0cde15d 100644 --- a/projects/fenix/build +++ b/projects/fenix/build @@ -7,6 +7,7 @@ mkdir -p $distdir/[% project %] mkdir -p /var/tmp/build

[% IF !c("var/fetch_gradle_dependencies") %] + [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] # XXX: Make this more generic and not only for geckoview-beta. geckoview_version=$(find . -type f -name geckoview-beta-*.aar | cut -d - -f 6 | cut -d . -f 1-3) gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %] @@ -32,6 +33,12 @@ cp $rootdir/[% c('input_files_by_name/tor-android-service') %]/* app/ [% IF c("var/fetch_gradle_dependencies") %] gradle --debug --no-daemon app:assemble[% c('variant') %] -x lint [% ELSE %] + # Prepare Glean dependencies for offline build + tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2 + # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding + # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/ + export LC_ALL=C.UTF-8 + export LANG=C.UTF-8 patch -p1 < $rootdir/mavenLocal.patch

# Make sure our GeckoView dependency is used. XXX: Make this more generic and diff --git a/projects/fenix/config b/projects/fenix/config index 01d3a25..33d4e44 100644 --- a/projects/fenix/config +++ b/projects/fenix/config @@ -39,10 +39,16 @@ input_files: project: geckoview pkg_type: merge_aars enable: '[% !c("var/fetch_gradle_dependencies") %]' + - name: python + project: python + enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies enable: '[% !c("var/fetch_gradle_dependencies") %]' exec: '[% INCLUDE "fetch-gradle-dependencies" %]' + - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-%5B% c('var/glean_parser') %].tar.bz2 + sha256sum: 69234aa1ae38fd5dda4de71bd5e536795073703c3ed5bb994b9b8485bd7a55dc + enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: mavenLocal.patch enable: '[% !c("var/fetch_gradle_dependencies") %]' - project: tor-android-service diff --git a/rbm.conf b/rbm.conf index 411e120..b5dff53 100644 --- a/rbm.conf +++ b/rbm.conf @@ -261,6 +261,8 @@ targets: # API 21 is the minimum we currently support for 64 bit on Android android_min_api_x86_64: 21 android_min_api_aarch64: 21 + # This is needed to get the offline build part for Glean right. + glean_parser: 1.28.1 # We only build snowflake on the alpha and nightly # channels for now. snowflake: '[% c("var/alpha") || c("var/nightly") %]'