[tor-commits] [tor-browser-build/maint-10.0-android] Bug 40270: Improve process to generate gradle-dependencies-list.txt
sysrqb at torproject.org
sysrqb at torproject.org
Thu May 6 03:59:31 UTC 2021
commit 0ef5812083c3cdd2d4aedf15144fc75074645aa8
Author: Nicolas Vigier <boklm at torproject.org>
Date: Tue Apr 13 17:59:12 2021 +0200
Bug 40270: Improve process to generate gradle-dependencies-list.txt
---
Makefile | 9 +++++
projects/android-components/build | 4 ++-
projects/android-components/config | 14 ++++++--
projects/application-services/build | 4 ++-
projects/application-services/config | 23 +++++++++++--
{tools => projects/common}/gen_gradle_deps_file.sh | 0
.../how-to-create-gradle-dependencies-list.txt | 40 ++++++++++++++++------
projects/fenix/build | 7 +++-
projects/fenix/config | 33 ++++++++++++------
9 files changed, 105 insertions(+), 29 deletions(-)
diff --git a/Makefile b/Makefile
index d4562ac..c30c70b 100644
--- a/Makefile
+++ b/Makefile
@@ -219,6 +219,15 @@ create_glean_deps_tarball: submodule-update
create_glean_deps_tarball-with_torsocks: submodule-update
$(rbm) build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7 --target with_torsocks
+get_gradle_dependencies_list-fenix: submodule-update
+ $(rbm) build fenix --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7
+
+get_gradle_dependencies_list-application-services: submodule-update
+ $(rbm) build application-services --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7
+
+get_gradle_dependencies_list-android-components: submodule-update
+ $(rbm) build android-components --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7
+
submodule-update:
git submodule update --init
diff --git a/projects/android-components/build b/projects/android-components/build
index 896ba3a..e14b803 100644
--- a/projects/android-components/build
+++ b/projects/android-components/build
@@ -41,7 +41,9 @@ patch -p1 < $rootdir/git.patch
# XXX: fetching deps for `assembleGeckoBeta -x lint` by using that same target
# results in some missing dependencies for yet unknown reasons. Thus, we use
# `assemble` instead for now.
- gradle --debug --no-daemon assemble
+ gradle --debug --no-daemon assemble > gradle.log 2>&1 || ( cat gradle.log; exit 1 )
+ $rootdir/gen_gradle_deps_file.sh gradle.log
+ mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
[% ELSE %]
# Prepare Glean dependencies for offline build
tar -xf $rootdir/[% c('input_files_by_name/glean') %]
diff --git a/projects/android-components/config b/projects/android-components/config
index 531b278..33bcdaf 100644
--- a/projects/android-components/config
+++ b/projects/android-components/config
@@ -14,9 +14,6 @@ var:
use_container: 1
# This should be updated when the list of gradle dependencies is changed.
gradle_dependencies_version: 22
- # Switch to make it easier to grab all dependencies during a dry-run.
- # Note: Use the commit before support for new GeckoView interfaces gets added.
- fetch_gradle_dependencies: 0
gradle_version: 6.6.1
glean_parser: 2.2.0
@@ -27,10 +24,13 @@ targets:
input_files:
- project: container-image
+ pkg_type: build
- name: '[% c("var/compiler") %]'
project: '[% c("var/compiler") %]'
+ pkg_type: build
- project: gradle
name: gradle
+ pkg_type: build
- name: geckoview
project: geckoview
pkg_type: merge_aars
@@ -52,6 +52,8 @@ input_files:
name: glean
sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6
enable: '[% !c("var/fetch_gradle_dependencies") %]'
+ - filename: gen_gradle_deps_file.sh
+ enable: '[% c("var/fetch_gradle_dependencies") %]'
steps:
list_toolchain_updates:
@@ -75,3 +77,9 @@ steps:
- name: glean
project: glean
pkg_type: src
+
+ get_gradle_dependencies_list:
+ filename: 'gradle-dependencies-list-[% c("var/android_components_version") %].txt'
+ get_gradle_dependencies_list: '[% INCLUDE build %]'
+ var:
+ fetch_gradle_dependencies: 1
diff --git a/projects/application-services/build b/projects/application-services/build
index b7461bf..40001cd 100644
--- a/projects/application-services/build
+++ b/projects/application-services/build
@@ -83,7 +83,9 @@ patch -p1 < $rootdir/target.patch
[% IF c('var/fetch_gradle_dependencies') %]
# XXX: `assemble` is still not enough to see all fetched dependencies via
# Gradle's --debug. See: tor-browser-build#40056.
- gradle --debug --no-daemon assemble
+ gradle --debug --no-daemon assemble > gradle.log 2>&1 || ( cat gradle.log; exit 1 )
+ $rootdir/gen_gradle_deps_file.sh gradle.log
+ mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
[% ELSE %]
# Prepare Glean dependencies for offline build
tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2
diff --git a/projects/application-services/config b/projects/application-services/config
index 588d573..456ce87 100644
--- a/projects/application-services/config
+++ b/projects/application-services/config
@@ -12,50 +12,61 @@ var:
gradle_dependencies_version: 5
# This should be updated when the list of rust dependencies is changed.
rust_vendor_version: 5
- # Switch to make it easier to grab all dependencies during a dry-run.
- fetch_gradle_dependencies: 0
gradle_version: 6.5
input_files:
- project: container-image
+ pkg_type: build
- project: '[% c("var/compiler") %]'
name: '[% c("var/compiler") %]'
+ pkg_type: build
- project: gradle
name: gradle
+ pkg_type: build
- project: rust
name: rust
+ pkg_type: build
- project: uniffi-rs
name: uniffi-rs
+ pkg_type: build
- project: nss
name: nss-armv7
+ pkg_type: build
target_prepend:
- android-armv7
- project: nss
name: nss-aarch64
+ pkg_type: build
target_prepend:
- android-aarch64
- project: nss
name: nss-x86
+ pkg_type: build
target_prepend:
- android-x86
- project: nss
name: nss-x86_64
+ pkg_type: build
target_prepend:
- android-x86_64
- project: sqlcipher
name: sqlcipher-armv7
+ pkg_type: build
target_prepend:
- android-armv7
- project: sqlcipher
name: sqlcipher-aarch64
+ pkg_type: build
target_prepend:
- android-aarch64
- project: sqlcipher
name: sqlcipher-x86
+ pkg_type: build
target_prepend:
- android-x86
- project: sqlcipher
name: sqlcipher-x86_64
+ pkg_type: build
target_prepend:
- android-x86_64
- name: python
@@ -79,6 +90,8 @@ input_files:
- filename: target.patch
- filename: viaduct-workaround.patch
- filename: update-cargo-lock.patch
+ - filename: gen_gradle_deps_file.sh
+ enable: '[% c("var/fetch_gradle_dependencies") %]'
steps:
list_toolchain_updates:
@@ -87,3 +100,9 @@ steps:
var:
container:
use_container: 0
+
+ get_gradle_dependencies_list:
+ filename: 'gradle-dependencies-list-[% c("version") %].txt'
+ get_gradle_dependencies_list: '[% INCLUDE build %]'
+ var:
+ fetch_gradle_dependencies: 1
diff --git a/tools/gen_gradle_deps_file.sh b/projects/common/gen_gradle_deps_file.sh
similarity index 100%
rename from tools/gen_gradle_deps_file.sh
rename to projects/common/gen_gradle_deps_file.sh
diff --git a/projects/common/how-to-create-gradle-dependencies-list.txt b/projects/common/how-to-create-gradle-dependencies-list.txt
index 940f810..b233eec 100644
--- a/projects/common/how-to-create-gradle-dependencies-list.txt
+++ b/projects/common/how-to-create-gradle-dependencies-list.txt
@@ -3,9 +3,17 @@ If additional Android dependencies are required by the project's build, then
the Gradle build will fail due to missing dependencies. To find out what the
missing dependencies are take the following steps.
-If the dependencies for `application-services`, `android-components`, or `fenix`
-are needed, set the `fetch_gradle_dependencies` flag in their `config` files to
-`1`.
+1) Updating gradle dependencies for `application-services`, `android-components`
+ or `fenix`
+
+The following makefile rules can be used:
+
+ make get_gradle_dependencies_list-application-services
+ make get_gradle_dependencies_list-android-components
+ make get_gradle_dependencies_list-fenix
+
+which should create the gradle-dependencies-list.txt files in the
+corresponding out/$project directory, using nightly's branch.
Note: `android-components` and `fenix` require modified `geckoview` and
`android-components` artifacts to compile successfully. In order to generate the
@@ -18,7 +26,16 @@ breaks). For `fenix` dependencies there is still a manual processing required
afterwards right now as e.g. our tor-android-service related artifacts are not
picked up.
-For the firefox project, comment out the following line in the project's build file:
+
+2) Updating gradle dependencies for `geckoview`, `tor-android-service`
+ or `tor-onion-proxy-library`
+
+The `geckoview`, `tor-android-service` and `tor-onion-proxy-library`
+projects don't have (yet) a makefile rule to generate their
+gradle-dependencies-list.txt file, so a few more steps are needed.
+
+For the geckoview project, comment out the following line in the project's
+build file:
export GRADLE_MAVEN_REPOSITORIES="file://$gradle_repo"
@@ -34,11 +51,14 @@ Finally, allow network access during the build by setting
Dependent artifacts will show up as downloads in the logs. You can pull out
these dependencies into a list by passing the log file to the gradle dependency
-list script in the tools directory:
+list script in the projects/common directory:
+
+`projects/common/gen_gradle_deps_file.sh /path/to/log/file`
+
-`./gen_gradle_deps_file.sh /path/to/log/file`
+3) Copying the resulting `gradle-dependencies-list.txt`
-Copy the resulting `gradle-dependencies-list.txt` over the one in the respective
-project. Then, in the project's config file, increment the
-`var/gradle_dependencies_version` and make sure to restore the project's build
-file back to original or set `fetch_gradle_dependencies` to `0` again.
+Copy the resulting `gradle-dependencies-list.txt` (from step 1. or 2.)
+over the one in the respective project. Then, in the project's config
+file, increment the `var/gradle_dependencies_version` and make sure to
+restore the project's build file back to original if you modified it.
diff --git a/projects/fenix/build b/projects/fenix/build
index b118cdc..6de75cf 100644
--- a/projects/fenix/build
+++ b/projects/fenix/build
@@ -42,7 +42,12 @@ cp $rootdir/[% c('input_files_by_name/tor-android-service') %]/* app/
# XXX We need the build variant in lower case. Do something smarter here.
v=[% c("variant") %]
[% IF c("var/fetch_gradle_dependencies") %]
- $GRADLE_HOME/gradle/bin/gradle --debug --no-daemon app:assemble[% c('variant') %] app:assembleAndroidTest -x lint -PtestBuildType=${v,} -PdisableOptimization
+ $GRADLE_HOME/gradle/bin/gradle --debug --no-daemon app:assemble[% c('variant') %] \
+ app:assembleAndroidTest -x lint -PtestBuildType=${v,} -PdisableOptimization \
+ > gradle.log 2>&1 \
+ || ( cat gradle.log; exit 1 )
+ $rootdir/gen_gradle_deps_file.sh gradle.log
+ mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
[% ELSE %]
# Add our localized strings
[% FOREACH lang = c('var/locales_mobile');
diff --git a/projects/fenix/config b/projects/fenix/config
index af2772e..b9371d9 100644
--- a/projects/fenix/config
+++ b/projects/fenix/config
@@ -15,9 +15,6 @@ var:
use_container: 1
# This should be updated when the list of gradle dependencies is changed.
gradle_dependencies_version: 24
- # Switch to make it easier to grab all dependencies during a dry-run.
- # Note: Use the commit that integrates Tor.
- fetch_gradle_dependencies: 0
gradle_version: 6.5.1
glean_parser: 2.2.0
@@ -35,30 +32,22 @@ input_files:
name: gradle
- name: android-components
project: android-components
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- name: application-services
project: application-services
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- name: geckoview
project: geckoview
pkg_type: merge_aars
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- name: python
project: python
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- name: tba-translation
project: tba-translation
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
name: gradle-dependencies
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
- URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %].tar.xz
name: glean
sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- filename: mavenLocal.patch
- enable: '[% !c("var/fetch_gradle_dependencies") %]'
- project: tor-android-service
name: tor-android-service
- project: tor-onion-proxy-library
@@ -73,3 +62,25 @@ steps:
var:
container:
use_container: 0
+
+ get_gradle_dependencies_list:
+ filename: 'gradle-dependencies-list-[% c("var/fenix_version") %].txt'
+ get_gradle_dependencies_list: '[% INCLUDE build %]'
+ var:
+ fetch_gradle_dependencies: 1
+ input_files:
+ - project: container-image
+ pkg_type: build
+ - filename: gen_gradle_deps_file.sh
+ - name: '[% c("var/compiler") %]'
+ project: '[% c("var/compiler") %]'
+ pkg_type: build
+ - project: gradle
+ name: gradle
+ pkg_type: build
+ - project: tor-android-service
+ name: tor-android-service
+ pkg_type: build
+ - project: tor-onion-proxy-library
+ name: topl
+ pkg_type: build
More information about the tor-commits
mailing list