commit 95e526842b59d63d438e6b554a428897e50bd70d Author: Nicolas Vigier boklm@torproject.org Date: Thu Sep 30 11:45:17 2021 +0200

Bug 40368: Use system's python3 for android builds --- projects/android-components/build | 1 - projects/android-components/config | 9 ++++----- projects/application-services/build | 4 ++-- projects/application-services/config | 10 +++++----- projects/fenix/build | 1 - projects/fenix/config | 8 ++++---- projects/geckoview/build | 3 +-- projects/geckoview/config | 14 +++----------- projects/glean/config | 27 ++++++++++++++------------- 9 files changed, 33 insertions(+), 44 deletions(-)

diff --git a/projects/android-components/build b/projects/android-components/build index 35e3421..0500fbd 100644 --- a/projects/android-components/build +++ b/projects/android-components/build @@ -13,7 +13,6 @@ cat > get-moz-build-date << "EOF" EOF

[% IF !c("var/fetch_gradle_dependencies") %] - [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] tar -C $distdir -xf [% c('input_files_by_name/geckoview') %] gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %] # This overwrites the release, beta, and nightly geckoview .aar files in our diff --git a/projects/android-components/config b/projects/android-components/config index 539bd8d..0047529 100644 --- a/projects/android-components/config +++ b/projects/android-components/config @@ -17,6 +17,8 @@ var: gradle_version: 6.6.1 glean_parser: 3.4.0 git_branch: '[% project %]-[% c("var/android_components_version") %]-[% c("var/torbrowser_branch") %]-1' + arch_deps: + - python3-venv

targets: nightly: @@ -39,9 +41,6 @@ input_files: - name: application-services project: application-services enable: '[% !c("var/fetch_gradle_dependencies") %]' - - name: python - project: python - enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies exec: '[% INCLUDE "fetch-gradle-dependencies" %]' @@ -49,9 +48,9 @@ input_files: - filename: git.patch - filename: mavenLocal.patch enable: '[% !c("var/fetch_gradle_dependencies") %]' - - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %].tar.xz + - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %]-build2.tar.xz name: glean - sha256sum: b74c9184dd2ee4c84a6e9766cd754cd07fc9be7233a1cdaa35bba6bd169beb75 + sha256sum: 6bbf3a9ffa939ca87e00b486f2020ab5cf9dea772f1d37b048f666fed38f98c1 enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: gen_gradle_deps_file.sh enable: '[% c("var/fetch_gradle_dependencies") %]' diff --git a/projects/application-services/build b/projects/application-services/build index c6e52ad..82b3350 100644 --- a/projects/application-services/build +++ b/projects/application-services/build @@ -22,7 +22,6 @@ cd $rootdir export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64

[% IF !c('var/fetch_gradle_dependencies') %] - [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %] cp -r $gradle_repo/dl/android/maven2/* $gradle_repo cp -r $gradle_repo/m2/* $gradle_repo @@ -75,6 +74,7 @@ patch -p1 < $rootdir/no-git.patch # XXX: We don't build the code for Linux for now as this involves additional # complexity. That's needed for running unit tests later on, though. patch -p1 < $rootdir/target.patch +export RUST_ANDROID_GRADLE_PYTHON_COMMAND=python3 [% IF c('var/fetch_gradle_dependencies') %] # XXX: `assemble` is still not enough to see all fetched dependencies via # Gradle's --debug. See: tor-browser-build#40056. @@ -83,7 +83,7 @@ patch -p1 < $rootdir/target.patch mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]' [% ELSE %] # Prepare Glean dependencies for offline build - tar -xf $rootdir/glean-wheels-[% c('var/glean_parser') %].tar.xz + tar -xf $rootdir/[% c('input_files_by_name/glean-wheels') %] # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/ export LC_ALL=C.UTF-8 diff --git a/projects/application-services/config b/projects/application-services/config index 22bfa06..c41a812 100644 --- a/projects/application-services/config +++ b/projects/application-services/config @@ -12,6 +12,8 @@ var: gradle_dependencies_version: 6 gradle_version: 6.5 glean_parser: 2.2.0 + arch_deps: + - python3-venv

input_files: - project: container-image @@ -68,15 +70,13 @@ input_files: pkg_type: build target_prepend: - android-x86_64 - - name: python - project: python - enable: '[% !c("var/fetch_gradle_dependencies") %]' - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies exec: '[% INCLUDE "fetch-gradle-dependencies" %]' enable: '[% !c("var/fetch_gradle_dependencies") %]' - - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %].tar.xz - sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6 + - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %]-build2.tar.xz + name: glean-wheels + sha256sum: 75107e7f84152806fde2070d1830174919a7a2ef54c774a1925a92dd4e20d0f7 enable: '[% !c("var/fetch_gradle_dependencies") %]' # Use `make cargo_vendor-application-services` to re-generate the vendor tarball - URL: https://people.torproject.org/~boklm/mirrors/sources/application-services-ve... c('version') %].tar.bz2 diff --git a/projects/fenix/build b/projects/fenix/build index 6de75cf..094b0ee 100644 --- a/projects/fenix/build +++ b/projects/fenix/build @@ -13,7 +13,6 @@ cat > get-moz-build-date << "EOF" EOF

[% IF !c("var/fetch_gradle_dependencies") %] - [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] tar -C $distdir -xf [% c('input_files_by_name/geckoview') %] gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %] # This overwrites the release, beta, and nightly geckoview .aar files in our diff --git a/projects/fenix/config b/projects/fenix/config index e5e0b50..c38ff99 100644 --- a/projects/fenix/config +++ b/projects/fenix/config @@ -18,6 +18,8 @@ var: gradle_dependencies_version: 32 gradle_version: 6.5.1 glean_parser: 3.4.0 + arch_deps: + - python3-venv

targets: release: @@ -40,16 +42,14 @@ input_files: - name: geckoview project: geckoview pkg_type: merge_aars - - name: python - project: python - name: tba-translation project: tba-translation - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' name: gradle-dependencies exec: '[% INCLUDE "fetch-gradle-dependencies" %]' - - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %].tar.xz + - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-%5B% c('var/glean_parser') %]-build2.tar.xz name: glean - sha256sum: b74c9184dd2ee4c84a6e9766cd754cd07fc9be7233a1cdaa35bba6bd169beb75 + sha256sum: 6bbf3a9ffa939ca87e00b486f2020ab5cf9dea772f1d37b048f666fed38f98c1 - filename: mavenLocal.patch - project: tor-android-service name: tor-android-service diff --git a/projects/geckoview/build b/projects/geckoview/build index ab80b81..1d24213 100644 --- a/projects/geckoview/build +++ b/projects/geckoview/build @@ -17,12 +17,11 @@ EOF tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %] tar -C /var/tmp/dist -xf [% c('input_files_by_name/cbindgen') %] tar -C /var/tmp/dist -xf [% c('input_files_by_name/nasm') %] -tar -C /var/tmp/dist -xf [% c('input_files_by_name/python') %] tar -C /var/tmp/dist -xf [% c('input_files_by_name/node') %] tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %] export LLVM_CONFIG="/var/tmp/dist/clang/bin/llvm-config" tar -C /var/tmp/dist -xf [% c('input_files_by_name/binutils') %] -export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/python/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH" +export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH"

tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz

diff --git a/projects/geckoview/config b/projects/geckoview/config index fba0d9b..08d906f 100644 --- a/projects/geckoview/config +++ b/projects/geckoview/config @@ -17,10 +17,8 @@ var: - zip - autoconf2.13 - yasm - # We are building our own version of Python 3.6, which is required - # for the build. However mach still requires Python 2.7, so we - # install this version using the package. - - python + - python3 + - python3-distutils - pkg-config container: use_container: 1 @@ -48,8 +46,7 @@ steps: EOF

tar -C $distdir -xf [% c('input_files_by_name/node') %] - tar -C $distdir -xf [% c('input_files_by_name/python') %] - export PATH="/var/tmp/dist/node/bin:/var/tmp/dist/python/bin:$PATH" + export PATH="/var/tmp/dist/node/bin:$PATH"

tar -C $builddir -xf [% c('input_files_by_name/geckoview_armv7') %] tar -C $builddir -xf [% c('input_files_by_name/geckoview_aarch64') %] @@ -108,9 +105,6 @@ steps: - project: node name: node pkg_type: build - - project: python - name: python - pkg_type: build - filename: mozconfig-android-all name: mozconfig - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' @@ -170,8 +164,6 @@ input_files: name: node - project: nasm name: nasm - - project: python - name: python - project: clang name: clang - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]' diff --git a/projects/glean/config b/projects/glean/config index a1f08e4..66eb272 100644 --- a/projects/glean/config +++ b/projects/glean/config @@ -8,11 +8,22 @@ src: | #!/bin/bash mv -f [% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c('filename') %]

+var: + # The `container-image` project looks at the deps list in origin_project + # in the same step (build), so we cannot put this inside the + # `create_glean_deps_tarball` step. + # rbm#40020 should allow us to fix that. + deps: + - python3 + - python3-pip + - torsocks + - xz-utils + steps: create_glean_deps_tarball: git_url: '' version: 3.4.0 - filename: 'glean-wheels-[% c("version") %].tar.xz' + filename: 'glean-wheels-[% c("version") %]-build2.tar.xz' build_log: '-' var: container: @@ -20,10 +31,11 @@ steps: create_glean_deps_tarball: | #!/bin/bash [% c("var/set_default_env") -%] - [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] mkdir glean-wheels cd glean-wheels [% IF c("var/use_torsocks") %]torsocks [% END%]python3 -m pip download glean_parser==[% c("version") %] + # Get pyyaml, needed for building application-services + [% IF c("var/use_torsocks") %]torsocks [% END%]python3 -m pip download pyyaml==5.3.1 cd .. [% c('tar', { tar_src => [ 'glean-wheels' ], @@ -36,18 +48,7 @@ steps: input_files: - project: container-image pkg_type: build - - project: python - name: python - pkg_type: build

targets: with_torsocks: - var: - # The `container-image` project looks at the deps list in origin_project - # in the same step (build), so we cannot put this inside the - # `create_glean_deps_tarball` step. - # rbm#40020 should allow us to fix that. - deps: - - torsocks - - xz-utils use_torsocks: 1