[tor-commits] [tor-browser-build/master] Bug 40055: Integrate building Glean in offline mode
sysrqb at torproject.org
sysrqb at torproject.org
Fri Sep 4 15:33:45 UTC 2020
commit e414bcd0a51859acc3fbdc48366b612b20425be3
Author: Georg Koppen <gk at torproject.org>
Date: Thu Sep 3 09:46:55 2020 +0000
Bug 40055: Integrate building Glean in offline mode
Using Glean 32.1.1 (via android-components 56.0.0 or later) did not land
yet in the application-services tag which Fenix and android-components
depend on. Thus, we patch application-services to include the earliest
version of android-components shipping Glean 32.1.1.
---
projects/android-components/build | 11 ++++++--
projects/android-components/config | 6 ++++
projects/application-services/1651662.patch | 22 +++++++++++++++
projects/application-services/build | 8 ++++++
projects/application-services/config | 9 +++++-
.../gradle-dependencies-list.txt | 32 +++++++++++-----------
projects/fenix/build | 7 +++++
projects/fenix/config | 6 ++++
rbm.conf | 2 ++
9 files changed, 84 insertions(+), 19 deletions(-)
diff --git a/projects/android-components/build b/projects/android-components/build
index a024861..3cac44a 100644
--- a/projects/android-components/build
+++ b/projects/android-components/build
@@ -3,10 +3,11 @@
[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
distdir=/var/tmp/dist
builddir=/var/tmp/build/[% project %]
-mkdir -p $distdir/[% project %]
-mkdir -p /var/tmp/build
+mkdir $distdir/[% project %]
+mkdir /var/tmp/build
[% IF !c("var/fetch_gradle_dependencies") %]
+ [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
# XXX: Make this more generic and not only for geckoview-beta.
geckoview_version=$(find . -type f -name geckoview-beta-*.aar | cut -d \- -f 6 | cut -d \. -f 1-3)
gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %]
@@ -29,6 +30,12 @@ patch -p1 < $rootdir/git.patch
# `assemble` instead for now.
gradle --debug -no-daemon assemble
[% ELSE %]
+ # Prepare Glean dependencies for offline build
+ tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2
+ # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding
+ # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/
+ export LC_ALL=C.UTF-8
+ export LANG=C.UTF-8
patch -p1 < $rootdir/mavenLocal.patch
# Make sure our GeckoView dependency is used. XXX: Make this more generic and
diff --git a/projects/android-components/config b/projects/android-components/config
index 3bdca82..56380e6 100644
--- a/projects/android-components/config
+++ b/projects/android-components/config
@@ -23,6 +23,9 @@ input_files:
- name: application-services
project: application-services
enable: '[% !c("var/fetch_gradle_dependencies") %]'
+ - name: python
+ project: python
+ enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
name: gradle-dependencies
exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
@@ -30,3 +33,6 @@ input_files:
- filename: git.patch
- filename: mavenLocal.patch
enable: '[% !c("var/fetch_gradle_dependencies") %]'
+ - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-[% c('var/glean_parser') %].tar.bz2
+ sha256sum: 69234aa1ae38fd5dda4de71bd5e536795073703c3ed5bb994b9b8485bd7a55dc
+ enable: '[% !c("var/fetch_gradle_dependencies") %]'
diff --git a/projects/application-services/1651662.patch b/projects/application-services/1651662.patch
new file mode 100644
index 0000000..0275da2
--- /dev/null
+++ b/projects/application-services/1651662.patch
@@ -0,0 +1,22 @@
+From 1818764e69145839ab33a2f885139ea320f2283f Mon Sep 17 00:00:00 2001
+From: Georg Koppen <gk at torproject.org>
+Date: Wed, 26 Aug 2020 15:56:49 +0000
+Subject: [PATCH] 1651662
+
+
+diff --git a/build.gradle b/build.gradle
+index 58a4ddae..c6e0be82 100644
+--- a/build.gradle
++++ b/build.gradle
+@@ -4,7 +4,7 @@ buildscript {
+ ext.kotlin_version = '1.3.71'
+ ext.jna_version = '5.2.0'
+ ext.android_gradle_plugin_version = '3.6.0'
+- ext.android_components_version = '47.0.0'
++ ext.android_components_version = '56.0.0'
+
+ ext.build = [
+ ndkVersion: "21.3.6528147", // Keep it in sync in TC Dockerfile.
+--
+2.28.0
+
diff --git a/projects/application-services/build b/projects/application-services/build
index 051163f..5c6b5e0 100644
--- a/projects/application-services/build
+++ b/projects/application-services/build
@@ -29,6 +29,7 @@ export ANDROID_NDK_API_VERSION=[% pc("android-toolchain", "var/android_ndk_versi
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
[% IF !c('var/fetch_gradle_dependencies') %]
+ [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %]
cp -r $gradle_repo/dl/android/maven2/* $gradle_repo
cp -r $gradle_repo/m2/* $gradle_repo
@@ -82,11 +83,18 @@ patch -p1 < $rootdir/no-git.patch
# complexity. That's needed for running unit tests later on, though.
patch -p1 < $rootdir/target.patch
patch -p1 < $rootdir/1651660.patch
+patch -p1 < $rootdir/1651662.patch
[% IF c('var/fetch_gradle_dependencies') %]
# XXX: `assemble` is still not enough to see all fetched dependencies via
# Gradle's --debug. See: tor-browser-build#40056.
$GRADLE_HOME/gradle-6.3/bin/gradle --debug --no-daemon assemble
[% ELSE %]
+ # Prepare Glean dependencies for offline build
+ tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2
+ # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding
+ # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/
+ export LC_ALL=C.UTF-8
+ export LANG=C.UTF-8
patch -p1 < $rootdir/mavenLocal.patch
gradle_flags="--offline --no-daemon -Dmaven.repo.local=$gradle_repo"
$GRADLE_HOME/gradle-6.3/bin/gradle $gradle_flags assembleRelease
diff --git a/projects/application-services/config b/projects/application-services/config
index 50986bb..1d9fd79 100644
--- a/projects/application-services/config
+++ b/projects/application-services/config
@@ -8,7 +8,7 @@ var:
container:
use_container: 1
# This should be updated when the list of gradle dependencies is changed.
- gradle_dependencies_version: 1
+ gradle_dependencies_version: 2
# This should be updated when the list of rust dependencies is changed.
rust_vendor_version: 1
# For armv7 we need a different target than the default one we use for
@@ -71,10 +71,16 @@ input_files:
name: sqlcipher-x86_64
target_prepend:
- torbrowser-android-x86_64
+ - name: python
+ project: python
+ enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
name: gradle-dependencies
exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
enable: '[% !c("var/fetch_gradle_dependencies") %]'
+ - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-[% c('var/glean_parser') %].tar.bz2
+ sha256sum: 69234aa1ae38fd5dda4de71bd5e536795073703c3ed5bb994b9b8485bd7a55dc
+ enable: '[% !c("var/fetch_gradle_dependencies") %]'
# `cargo vendor vendor` in the `application-services` directory has vendored
# the sources.
- URL: https://people.torproject.org/~gk/mirrors/sources/application-services-vendor-[% c('var/rust_vendor_version') %].tar.bz2
@@ -84,3 +90,4 @@ input_files:
enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: target.patch
- filename: 1651660.patch
+ - filename: 1651662.patch
diff --git a/projects/application-services/gradle-dependencies-list.txt b/projects/application-services/gradle-dependencies-list.txt
index ffad4f0..3978277 100644
--- a/projects/application-services/gradle-dependencies-list.txt
+++ b/projects/application-services/gradle-dependencies-list.txt
@@ -403,22 +403,22 @@ b51f8867c92b6a722499557fc3a1fdea77bdf9ef574722fe90ce436a29559454 | https://jcent
fb40265f982548212ff82e362e59732b2187ec6f0d80182885c14ef1f982827a | https://jcenter.bintray.com/org/sonatype/oss/oss-parent/9/oss-parent-9.pom
13009fb5ede3cf2be5a8d0f1602155aeaa0ce5ef5f9366892bd258d8d3d4d2b1 | https://jcenter.bintray.com/org/yaml/snakeyaml/1.23/snakeyaml-1.23.jar
1e1beb22ca906209700122b5cacf6f2719324538c5b1e3c27bf91564c8d31dbd | https://jcenter.bintray.com/org/yaml/snakeyaml/1.23/snakeyaml-1.23.pom
-4bca00bc684d935cf3675d3437113d1284bb94412b6b9aea03194f946c719a9c | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/47.0.0/concept-fetch-47.0.0.aar
-c2ec8bc73e7e1a86b53f1c8ac8e2f09eee5e31b6bdc291f31495e3080ab66261 | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/47.0.0/concept-fetch-47.0.0.pom
-5df22d5e7864c593c0b78624304d3046514177cd819bada0ce5927ed22e38bdf | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/47.0.0/service-glean-47.0.0.aar
-c062918be5f021628bf9705898a9e7598ed340f5ec971033ab0ea239ee6efaeb | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/47.0.0/service-glean-47.0.0.pom
-c959adf0ae40a073b3db4e21c1f6a9a961bbb7d6cc33c569492e2268b8a473a1 | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/47.0.0/support-base-47.0.0.aar
-0f47f76b38975b5407b212f1aa4dd1daabde311989ff934360afa529c1c77d92 | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/47.0.0/support-base-47.0.0.pom
-2813838b7bb4d2973b3663f2aac08b36c3db9a372d266eb228fba98bfcb09017 | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/47.0.0/support-ktx-47.0.0.aar
-0d72828ec6af2fe0b7021c2cada48204a47ccd6e47eeea35dd2b51c24df3b8ff | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/47.0.0/support-ktx-47.0.0.pom
-bd295b27f098eed0aadf2ebbf3daf2356c6ebe73c2cde49597bdf33414936699 | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/47.0.0/support-utils-47.0.0.aar
-10543160bfa1790c2d98ba8e0de1230e0feea1926fd4859751c931e3bd5e1c34 | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/47.0.0/support-utils-47.0.0.pom
-e24f78d36c64695407ace7b499472306161b1abf47b65c99ec790f321a6663e1 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle/47.0.0/tooling-glean-gradle-47.0.0.jar
-6320746e6d914aa45d9f8e0de23bdbd81c8097999b3dd64796363ed878633600 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle/47.0.0/tooling-glean-gradle-47.0.0.pom
-68f97a68faf84fa03cf3e1898042ffefe17b37c4c02df9e754fd2a47e6746196 | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/31.1.1/glean-31.1.1.aar
-9d6e64de942f997293b5ed35337da9350798192b22d8928030f191174b9f77fa | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/31.1.1/glean-31.1.1.pom
-3c2868a395d588eef2a370e4cd889a3a71351cff8eb4dd557dee764bc9f6e90a | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/31.1.1/glean-gradle-plugin-31.1.1.jar
-7517604c7643984718d421c85df395ae9f69f74c78ebd825930b4bceccf8196c | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/31.1.1/glean-gradle-plugin-31.1.1.pom
+e14c614b65ae4fe6c6d26ce369d523cf1ff837ce697e5e3f47e8cdfa355357f9 | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/56.0.0/concept-fetch-56.0.0.aar
+e7bd7b2e2013a080bcb3cb9db974185eb28dd87c0594394f951a51ab5d6fcf5d | https://maven.mozilla.org/maven2/org/mozilla/components/concept-fetch/56.0.0/concept-fetch-56.0.0.pom
+e79f67b5180f06a4680ffea310ceb2c5b0af205a88b3fb1cc76ce96e5b4a7ea7 | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/56.0.0/service-glean-56.0.0.aar
+32acc51221752051a03765ac0bbf26a1da663ba2f7b3fe69ae422d5aac228f81 | https://maven.mozilla.org/maven2/org/mozilla/components/service-glean/56.0.0/service-glean-56.0.0.pom
+ae35db5670efe458d3b9646cab337f2f7667becacdb023b599cc62bfa95b59ec | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/56.0.0/support-base-56.0.0.aar
+d87cd2eea092430c4e5a95ff04c3213be89f54aa61c2c7f80fb6548e6dc38bab | https://maven.mozilla.org/maven2/org/mozilla/components/support-base/56.0.0/support-base-56.0.0.pom
+96aeede35017fd79576c5faa1cb41efc8137616e20231b41934999ca711f4407 | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/56.0.0/support-ktx-56.0.0.aar
+dbd706151fe820ed233792a6773b70efcceb42987c209e27575ef91d4552b8c4 | https://maven.mozilla.org/maven2/org/mozilla/components/support-ktx/56.0.0/support-ktx-56.0.0.pom
+22934a560b0bc34f01304efbdccc988d2e0fca039ae577894fcec52a7fdfd88e | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/56.0.0/support-utils-56.0.0.aar
+a2c1e39e5e58f585c2908af2266953b6b43e51f2fabe45e7723bc177c9c49bc0 | https://maven.mozilla.org/maven2/org/mozilla/components/support-utils/56.0.0/support-utils-56.0.0.pom
+0165b5554aa3427a2e4026e9ca78c1ea5021f1578d36839b37350dbf0f192d29 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle/56.0.0/tooling-glean-gradle-56.0.0.jar
+878570d1c4e5b06d2434b7133a14ce97fb516b7871c7637c44a4b087db291a49 | https://maven.mozilla.org/maven2/org/mozilla/components/tooling-glean-gradle/56.0.0/tooling-glean-gradle-56.0.0.pom
+5b078e136164082be6d49afa0d87f849c7b9f65e21125fa57056cd630756279b | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/32.1.1/glean-32.1.1.aar
+64371d71a4fb11d63d2a5ff60149c94c1d0e0b909de09ffe199f6872663976b2 | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean/32.1.1/glean-32.1.1.pom
+ab514359d7686282b7a7c8463e81237ff98460a23df534a6c9663b463e65423f | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/32.1.1/glean-gradle-plugin-32.1.1.jar
+f724de3fc1c37d8ecb4bbb7957dd67a489d0321da71eebb6b8d605ec4a57755a | https://maven.mozilla.org/maven2/org/mozilla/telemetry/glean-gradle-plugin/32.1.1/glean-gradle-plugin-32.1.1.pom
470564ff0a452da1c96be4ca3a9cdeceaa2912537ee2a8def486f3b383d43f15 | https://plugins.gradle.org/m2/com/jetbrains/python/envs/com.jetbrains.python.envs.gradle.plugin/0.0.26/com.jetbrains.python.envs.gradle.plugin-0.0.26.pom
6438812f927b505a4c4ef2703b45433a1b99e45334526bfed25e71e1a79abb14 | https://plugins.gradle.org/m2/gradle/plugin/com/jetbrains/python/gradle-python-envs/0.0.26/gradle-python-envs-0.0.26.jar
317aa4c467a37d5fcab6657c326a4069ea8e8c6dda951ec2091ffb95c596a2a3 | https://plugins.gradle.org/m2/gradle/plugin/com/jetbrains/python/gradle-python-envs/0.0.26/gradle-python-envs-0.0.26.pom
diff --git a/projects/fenix/build b/projects/fenix/build
index 5e483c9..0cde15d 100644
--- a/projects/fenix/build
+++ b/projects/fenix/build
@@ -7,6 +7,7 @@ mkdir -p $distdir/[% project %]
mkdir -p /var/tmp/build
[% IF !c("var/fetch_gradle_dependencies") %]
+ [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
# XXX: Make this more generic and not only for geckoview-beta.
geckoview_version=$(find . -type f -name geckoview-beta-*.aar | cut -d \- -f 6 | cut -d \. -f 1-3)
gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %]
@@ -32,6 +33,12 @@ cp $rootdir/[% c('input_files_by_name/tor-android-service') %]/* app/
[% IF c("var/fetch_gradle_dependencies") %]
gradle --debug --no-daemon app:assemble[% c('variant') %] -x lint
[% ELSE %]
+ # Prepare Glean dependencies for offline build
+ tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2
+ # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding
+ # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/
+ export LC_ALL=C.UTF-8
+ export LANG=C.UTF-8
patch -p1 < $rootdir/mavenLocal.patch
# Make sure our GeckoView dependency is used. XXX: Make this more generic and
diff --git a/projects/fenix/config b/projects/fenix/config
index 01d3a25..33d4e44 100644
--- a/projects/fenix/config
+++ b/projects/fenix/config
@@ -39,10 +39,16 @@ input_files:
project: geckoview
pkg_type: merge_aars
enable: '[% !c("var/fetch_gradle_dependencies") %]'
+ - name: python
+ project: python
+ enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
name: gradle-dependencies
enable: '[% !c("var/fetch_gradle_dependencies") %]'
exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
+ - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-[% c('var/glean_parser') %].tar.bz2
+ sha256sum: 69234aa1ae38fd5dda4de71bd5e536795073703c3ed5bb994b9b8485bd7a55dc
+ enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: mavenLocal.patch
enable: '[% !c("var/fetch_gradle_dependencies") %]'
- project: tor-android-service
diff --git a/rbm.conf b/rbm.conf
index 411e120..b5dff53 100644
--- a/rbm.conf
+++ b/rbm.conf
@@ -261,6 +261,8 @@ targets:
# API 21 is the minimum we currently support for 64 bit on Android
android_min_api_x86_64: 21
android_min_api_aarch64: 21
+ # This is needed to get the offline build part for Glean right.
+ glean_parser: 1.28.1
# We only build snowflake on the alpha and nightly
# channels for now.
snowflake: '[% c("var/alpha") || c("var/nightly") %]'
More information about the tor-commits
mailing list