commit 8ba33c11bb66c6ef775155542bba3f24706a71d1 Author: Nicolas Vigier boklm@torproject.org Date: Tue Apr 13 17:59:12 2021 +0200

Bug 40270: Improve process to generate gradle-dependencies-list.txt --- Makefile | 9 +++++ projects/android-components/build | 4 ++- projects/android-components/config | 14 ++++++-- projects/application-services/build | 4 ++- projects/application-services/config | 23 +++++++++++-- {tools => projects/common}/gen_gradle_deps_file.sh | 0 .../how-to-create-gradle-dependencies-list.txt | 40 ++++++++++++++++------ projects/fenix/build | 7 +++- projects/fenix/config | 33 ++++++++++++------ 9 files changed, 105 insertions(+), 29 deletions(-)

diff --git a/Makefile b/Makefile index 35e8834..1d16fa4 100644 --- a/Makefile +++ b/Makefile @@ -219,6 +219,15 @@ create_glean_deps_tarball: submodule-update create_glean_deps_tarball-with_torsocks: submodule-update $(rbm) build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7 --target with_torsocks

+get_gradle_dependencies_list-fenix: submodule-update + $(rbm) build fenix --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7 + +get_gradle_dependencies_list-application-services: submodule-update + $(rbm) build application-services --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7 + +get_gradle_dependencies_list-android-components: submodule-update + $(rbm) build android-components --step get_gradle_dependencies_list --target nightly --target torbrowser-android-armv7 + submodule-update: git submodule update --init

diff --git a/projects/android-components/build b/projects/android-components/build index 896ba3a..e14b803 100644 --- a/projects/android-components/build +++ b/projects/android-components/build @@ -41,7 +41,9 @@ patch -p1 < $rootdir/git.patch # XXX: fetching deps for `assembleGeckoBeta -x lint` by using that same target # results in some missing dependencies for yet unknown reasons. Thus, we use # `assemble` instead for now. - gradle --debug --no-daemon assemble + gradle --debug --no-daemon assemble > gradle.log 2>&1 || ( cat gradle.log; exit 1 ) + $rootdir/gen_gradle_deps_file.sh gradle.log + mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]' [% ELSE %] # Prepare Glean dependencies for offline build tar -xf $rootdir/[% c('input_files_by_name/glean') %] diff --git a/projects/android-components/config b/projects/android-components/config index e3068b6..5284ce7 100644 --- a/projects/android-components/config +++ b/projects/android-components/config @@ -14,9 +14,6 @@ var: use_container: 1 # This should be updated when the list of gradle dependencies is changed. gradle_dependencies_version: 22 - # Switch to make it easier to grab all dependencies during a dry-run. - # Note: Use the commit before support for new GeckoView interfaces gets added. - fetch_gradle_dependencies: 0 gradle_version: 6.6.1 glean_parser: 2.2.0

@@ -27,10 +24,13 @@ targets:

input_files: - project: container-image + pkg_type: build - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' + pkg_type: build - project: gradle name: gradle + pkg_type: build - name: geckoview project: geckoview pkg_type: merge_aars @@ -52,6 +52,8 @@ input_files: name: glean sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6 enable: '[% !c("var/fetch_gradle_dependencies") %]' + - filename: gen_gradle_deps_file.sh + enable: '[% c("var/fetch_gradle_dependencies") %]'

steps: list_toolchain_updates: @@ -75,3 +77,9 @@ steps: - name: glean project: glean pkg_type: src + + get_gradle_dependencies_list: + filename: 'gradle-dependencies-list-[% c("var/android_components_version") %].txt' + get_gradle_dependencies_list: '[% INCLUDE build %]' + var: + fetch_gradle_dependencies: 1 diff --git a/projects/application-services/build b/projects/application-services/build index b7461bf..40001cd 100644 --- a/projects/application-services/build +++ b/projects/application-services/build @@ -83,7 +83,9 @@ patch -p1 < $rootdir/target.patch [% IF c('var/fetch_gradle_dependencies') %] # XXX: `assemble` is still not enough to see all fetched dependencies via # Gradle's --debug. See: tor-browser-build#40056. - gradle --debug --no-daemon assemble + gradle --debug --no-daemon assemble > gradle.log 2>&1 || ( cat gradle.log; exit 1 ) + $rootdir/gen_gradle_deps_file.sh gradle.log + mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]' [% ELSE %] # Prepare Glean dependencies for offline build tar -xjf $rootdir/glean-parser-[% c('var/glean_parser') %].tar.bz2 diff --git a/projects/application-services/config b/projects/application-services/config index 588d573..456ce87 100644 --- a/projects/application-services/config +++ b/projects/application-services/config @@ -12,50 +12,61 @@ var: gradle_dependencies_version: 5 # This should be updated when the list of rust dependencies is changed. rust_vendor_version: 5 - # Switch to make it easier to grab all dependencies during a dry-run. - fetch_gradle_dependencies: 0 gradle_version: 6.5

input_files: - project: container-image + pkg_type: build - project: '[% c("var/compiler") %]' name: '[% c("var/compiler") %]' + pkg_type: build - project: gradle name: gradle + pkg_type: build - project: rust name: rust + pkg_type: build - project: uniffi-rs name: uniffi-rs + pkg_type: build - project: nss name: nss-armv7 + pkg_type: build target_prepend: - android-armv7 - project: nss name: nss-aarch64 + pkg_type: build target_prepend: - android-aarch64 - project: nss name: nss-x86 + pkg_type: build target_prepend: - android-x86 - project: nss name: nss-x86_64 + pkg_type: build target_prepend: - android-x86_64 - project: sqlcipher name: sqlcipher-armv7 + pkg_type: build target_prepend: - android-armv7 - project: sqlcipher name: sqlcipher-aarch64 + pkg_type: build target_prepend: - android-aarch64 - project: sqlcipher name: sqlcipher-x86 + pkg_type: build target_prepend: - android-x86 - project: sqlcipher name: sqlcipher-x86_64 + pkg_type: build target_prepend: - android-x86_64 - name: python @@ -79,6 +90,8 @@ input_files: - filename: target.patch - filename: viaduct-workaround.patch - filename: update-cargo-lock.patch + - filename: gen_gradle_deps_file.sh + enable: '[% c("var/fetch_gradle_dependencies") %]'

steps: list_toolchain_updates: @@ -87,3 +100,9 @@ steps: var: container: use_container: 0 + + get_gradle_dependencies_list: + filename: 'gradle-dependencies-list-[% c("version") %].txt' + get_gradle_dependencies_list: '[% INCLUDE build %]' + var: + fetch_gradle_dependencies: 1 diff --git a/tools/gen_gradle_deps_file.sh b/projects/common/gen_gradle_deps_file.sh similarity index 100% rename from tools/gen_gradle_deps_file.sh rename to projects/common/gen_gradle_deps_file.sh diff --git a/projects/common/how-to-create-gradle-dependencies-list.txt b/projects/common/how-to-create-gradle-dependencies-list.txt index 940f810..b233eec 100644 --- a/projects/common/how-to-create-gradle-dependencies-list.txt +++ b/projects/common/how-to-create-gradle-dependencies-list.txt @@ -3,9 +3,17 @@ If additional Android dependencies are required by the project's build, then the Gradle build will fail due to missing dependencies. To find out what the missing dependencies are take the following steps.

-If the dependencies for `application-services`, `android-components`, or `fenix` -are needed, set the `fetch_gradle_dependencies` flag in their `config` files to -`1`. +1) Updating gradle dependencies for `application-services`, `android-components` + or `fenix` + +The following makefile rules can be used: + + make get_gradle_dependencies_list-application-services + make get_gradle_dependencies_list-android-components + make get_gradle_dependencies_list-fenix + +which should create the gradle-dependencies-list.txt files in the +corresponding out/$project directory, using nightly's branch.

Note: `android-components` and `fenix` require modified `geckoview` and `android-components` artifacts to compile successfully. In order to generate the @@ -18,7 +26,16 @@ breaks). For `fenix` dependencies there is still a manual processing required afterwards right now as e.g. our tor-android-service related artifacts are not picked up.

-For the firefox project, comment out the following line in the project's build file: + +2) Updating gradle dependencies for `geckoview`, `tor-android-service` + or `tor-onion-proxy-library` + +The `geckoview`, `tor-android-service` and `tor-onion-proxy-library` +projects don't have (yet) a makefile rule to generate their +gradle-dependencies-list.txt file, so a few more steps are needed. + +For the geckoview project, comment out the following line in the project's +build file:

export GRADLE_MAVEN_REPOSITORIES="file://$gradle_repo"

@@ -34,11 +51,14 @@ Finally, allow network access during the build by setting

Dependent artifacts will show up as downloads in the logs. You can pull out these dependencies into a list by passing the log file to the gradle dependency -list script in the tools directory: +list script in the projects/common directory: + +`projects/common/gen_gradle_deps_file.sh /path/to/log/file` +

-`./gen_gradle_deps_file.sh /path/to/log/file` +3) Copying the resulting `gradle-dependencies-list.txt`

-Copy the resulting `gradle-dependencies-list.txt` over the one in the respective -project. Then, in the project's config file, increment the -`var/gradle_dependencies_version` and make sure to restore the project's build -file back to original or set `fetch_gradle_dependencies` to `0` again. +Copy the resulting `gradle-dependencies-list.txt` (from step 1. or 2.) +over the one in the respective project. Then, in the project's config +file, increment the `var/gradle_dependencies_version` and make sure to +restore the project's build file back to original if you modified it. diff --git a/projects/fenix/build b/projects/fenix/build index b118cdc..6de75cf 100644 --- a/projects/fenix/build +++ b/projects/fenix/build @@ -42,7 +42,12 @@ cp $rootdir/[% c('input_files_by_name/tor-android-service') %]/* app/ # XXX We need the build variant in lower case. Do something smarter here. v=[% c("variant") %] [% IF c("var/fetch_gradle_dependencies") %] - $GRADLE_HOME/gradle/bin/gradle --debug --no-daemon app:assemble[% c('variant') %] app:assembleAndroidTest -x lint -PtestBuildType=${v,} -PdisableOptimization + $GRADLE_HOME/gradle/bin/gradle --debug --no-daemon app:assemble[% c('variant') %] \ + app:assembleAndroidTest -x lint -PtestBuildType=${v,} -PdisableOptimization \ + > gradle.log 2>&1 \ + || ( cat gradle.log; exit 1 ) + $rootdir/gen_gradle_deps_file.sh gradle.log + mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]' [% ELSE %] # Add our localized strings [% FOREACH lang = c('var/locales_mobile'); diff --git a/projects/fenix/config b/projects/fenix/config index 46cbc1e..ff87d4b 100644 --- a/projects/fenix/config +++ b/projects/fenix/config @@ -15,9 +15,6 @@ var: use_container: 1 # This should be updated when the list of gradle dependencies is changed. gradle_dependencies_version: 24 - # Switch to make it easier to grab all dependencies during a dry-run. - # Note: Use the commit that integrates Tor. - fetch_gradle_dependencies: 0 gradle_version: 6.5.1 glean_parser: 2.2.0

@@ -35,30 +32,22 @@ input_files: name: gradle - name: android-components project: android-components - enable: '[% !c("var/fetch_gradle_dependencies") %]' - name: application-services project: application-services - enable: '[% !c("var/fetch_gradle_dependencies") %]' - name: geckoview project: geckoview pkg_type: merge_aars - enable: '[% !c("var/fetch_gradle_dependencies") %]' - name: python project: python - enable: '[% !c("var/fetch_gradle_dependencies") %]' - name: tba-translation project: tba-translation - enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies - enable: '[% !c("var/fetch_gradle_dependencies") %]' exec: '[% INCLUDE "fetch-gradle-dependencies" %]' - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %].tar.xz name: glean sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6 - enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: mavenLocal.patch - enable: '[% !c("var/fetch_gradle_dependencies") %]' - project: tor-android-service name: tor-android-service - project: tor-onion-proxy-library @@ -73,3 +62,25 @@ steps: var: container: use_container: 0 + + get_gradle_dependencies_list: + filename: 'gradle-dependencies-list-[% c("var/fenix_version") %].txt' + get_gradle_dependencies_list: '[% INCLUDE build %]' + var: + fetch_gradle_dependencies: 1 + input_files: + - project: container-image + pkg_type: build + - filename: gen_gradle_deps_file.sh + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + pkg_type: build + - project: gradle + name: gradle + pkg_type: build + - project: tor-android-service + name: tor-android-service + pkg_type: build + - project: tor-onion-proxy-library + name: topl + pkg_type: build