[tor-commits] [tor-browser-build/master] Bug 40368: Use system's python3 for android builds

sysrqb at torproject.org sysrqb at torproject.org
Fri Dec 17 00:53:53 UTC 2021


commit 95e526842b59d63d438e6b554a428897e50bd70d
Author: Nicolas Vigier <boklm at torproject.org>
Date:   Thu Sep 30 11:45:17 2021 +0200

    Bug 40368: Use system's python3 for android builds
---
 projects/android-components/build    |  1 -
 projects/android-components/config   |  9 ++++-----
 projects/application-services/build  |  4 ++--
 projects/application-services/config | 10 +++++-----
 projects/fenix/build                 |  1 -
 projects/fenix/config                |  8 ++++----
 projects/geckoview/build             |  3 +--
 projects/geckoview/config            | 14 +++-----------
 projects/glean/config                | 27 ++++++++++++++-------------
 9 files changed, 33 insertions(+), 44 deletions(-)

diff --git a/projects/android-components/build b/projects/android-components/build
index 35e3421..0500fbd 100644
--- a/projects/android-components/build
+++ b/projects/android-components/build
@@ -13,7 +13,6 @@ cat > get-moz-build-date << "EOF"
 EOF
 
 [% IF !c("var/fetch_gradle_dependencies") %]
-  [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
   tar -C $distdir -xf [% c('input_files_by_name/geckoview') %]
   gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %]
   # This overwrites the release, beta, and nightly geckoview .aar files in our
diff --git a/projects/android-components/config b/projects/android-components/config
index 539bd8d..0047529 100644
--- a/projects/android-components/config
+++ b/projects/android-components/config
@@ -17,6 +17,8 @@ var:
   gradle_version: 6.6.1
   glean_parser: 3.4.0
   git_branch: '[% project %]-[% c("var/android_components_version") %]-[% c("var/torbrowser_branch") %]-1'
+  arch_deps:
+    - python3-venv
 
 targets:
   nightly:
@@ -39,9 +41,6 @@ input_files:
   - name: application-services
     project: application-services
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
-  - name: python
-    project: python
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
     name: gradle-dependencies
     exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
@@ -49,9 +48,9 @@ input_files:
   - filename: git.patch
   - filename: mavenLocal.patch
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
-  - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %].tar.xz
+  - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %]-build2.tar.xz
     name: glean
-    sha256sum: b74c9184dd2ee4c84a6e9766cd754cd07fc9be7233a1cdaa35bba6bd169beb75
+    sha256sum: 6bbf3a9ffa939ca87e00b486f2020ab5cf9dea772f1d37b048f666fed38f98c1
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - filename: gen_gradle_deps_file.sh
     enable: '[% c("var/fetch_gradle_dependencies") %]'
diff --git a/projects/application-services/build b/projects/application-services/build
index c6e52ad..82b3350 100644
--- a/projects/application-services/build
+++ b/projects/application-services/build
@@ -22,7 +22,6 @@ cd $rootdir
 export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
 
 [% IF !c('var/fetch_gradle_dependencies') %]
-  [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
   gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %]
   cp -r $gradle_repo/dl/android/maven2/* $gradle_repo
   cp -r $gradle_repo/m2/* $gradle_repo
@@ -75,6 +74,7 @@ patch -p1 < $rootdir/no-git.patch
 # XXX: We don't build the code for Linux for now as this involves additional
 # complexity. That's needed for running unit tests later on, though.
 patch -p1 < $rootdir/target.patch
+export RUST_ANDROID_GRADLE_PYTHON_COMMAND=python3
 [% IF c('var/fetch_gradle_dependencies') %]
   # XXX: `assemble` is still not enough to see all fetched dependencies via
   # Gradle's --debug. See: tor-browser-build#40056.
@@ -83,7 +83,7 @@ patch -p1 < $rootdir/target.patch
   mv gradle-dependencies-list.txt '[% dest_dir _ "/" _ c("filename") %]'
 [% ELSE %]
   # Prepare Glean dependencies for offline build
-  tar -xf $rootdir/glean-wheels-[% c('var/glean_parser') %].tar.xz
+  tar -xf $rootdir/[% c('input_files_by_name/glean-wheels') %]
   # We need to set `LC_ALL` and `LANG` to something that is not ASCII as encoding
   # otherwise `click` barfs. See: https://click.palletsprojects.com/python3/
   export LC_ALL=C.UTF-8
diff --git a/projects/application-services/config b/projects/application-services/config
index 22bfa06..c41a812 100644
--- a/projects/application-services/config
+++ b/projects/application-services/config
@@ -12,6 +12,8 @@ var:
   gradle_dependencies_version: 6
   gradle_version: 6.5
   glean_parser: 2.2.0
+  arch_deps:
+    - python3-venv
 
 input_files:
   - project: container-image
@@ -68,15 +70,13 @@ input_files:
     pkg_type: build
     target_prepend:
       - android-x86_64
-  - name: python
-    project: python
-    enable: '[% !c("var/fetch_gradle_dependencies") %]'
   - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
     name: gradle-dependencies
     exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
-  - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %].tar.xz
-    sha256sum: 24ceaaadaf155445e8ad135173d894e05c0745b41ab5fee150f9548550acf2a6
+  - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %]-build2.tar.xz
+    name: glean-wheels
+    sha256sum: 75107e7f84152806fde2070d1830174919a7a2ef54c774a1925a92dd4e20d0f7
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
   # Use `make cargo_vendor-application-services` to re-generate the vendor tarball
   - URL: https://people.torproject.org/~boklm/mirrors/sources/application-services-vendor-[% c('version') %].tar.bz2
diff --git a/projects/fenix/build b/projects/fenix/build
index 6de75cf..094b0ee 100644
--- a/projects/fenix/build
+++ b/projects/fenix/build
@@ -13,7 +13,6 @@ cat > get-moz-build-date << "EOF"
 EOF
 
 [% IF !c("var/fetch_gradle_dependencies") %]
-  [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
   tar -C $distdir -xf [% c('input_files_by_name/geckoview') %]
   gradle_repo=$rootdir/[% c('input_files_by_name/gradle-dependencies') %]
   # This overwrites the release, beta, and nightly geckoview .aar files in our
diff --git a/projects/fenix/config b/projects/fenix/config
index e5e0b50..c38ff99 100644
--- a/projects/fenix/config
+++ b/projects/fenix/config
@@ -18,6 +18,8 @@ var:
   gradle_dependencies_version: 32
   gradle_version: 6.5.1
   glean_parser: 3.4.0
+  arch_deps:
+    - python3-venv
 
 targets:
   release:
@@ -40,16 +42,14 @@ input_files:
   - name: geckoview
     project: geckoview
     pkg_type: merge_aars
-  - name: python
-    project: python
   - name: tba-translation
     project: tba-translation
   - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
     name: gradle-dependencies
     exec: '[% INCLUDE "fetch-gradle-dependencies" %]'
-  - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %].tar.xz
+  - URL: https://people.torproject.org/~boklm/mirrors/sources/glean-wheels-[% c('var/glean_parser') %]-build2.tar.xz
     name: glean
-    sha256sum: b74c9184dd2ee4c84a6e9766cd754cd07fc9be7233a1cdaa35bba6bd169beb75
+    sha256sum: 6bbf3a9ffa939ca87e00b486f2020ab5cf9dea772f1d37b048f666fed38f98c1
   - filename: mavenLocal.patch
   - project: tor-android-service
     name: tor-android-service
diff --git a/projects/geckoview/build b/projects/geckoview/build
index ab80b81..1d24213 100644
--- a/projects/geckoview/build
+++ b/projects/geckoview/build
@@ -17,12 +17,11 @@ EOF
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/rust') %]
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/cbindgen') %]
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/nasm') %]
-tar -C /var/tmp/dist -xf [% c('input_files_by_name/python') %]
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/node') %]
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/clang') %]
 export LLVM_CONFIG="/var/tmp/dist/clang/bin/llvm-config"
 tar -C /var/tmp/dist -xf [% c('input_files_by_name/binutils') %]
-export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/python/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH"
+export PATH="/var/tmp/dist/rust/bin:/var/tmp/dist/cbindgen:/var/tmp/dist/nasm/bin:/var/tmp/dist/node/bin:/var/tmp/dist/clang/bin:/var/tmp/dist/binutils/bin:$PATH"
 
 tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
 
diff --git a/projects/geckoview/config b/projects/geckoview/config
index fba0d9b..08d906f 100644
--- a/projects/geckoview/config
+++ b/projects/geckoview/config
@@ -17,10 +17,8 @@ var:
     - zip
     - autoconf2.13
     - yasm
-    # We are building our own version of Python 3.6, which is required
-    # for the build. However mach still requires Python 2.7, so we
-    # install this version using the package.
-    - python
+    - python3
+    - python3-distutils
     - pkg-config
   container:
     use_container: 1
@@ -48,8 +46,7 @@ steps:
       EOF
 
       tar -C $distdir -xf [% c('input_files_by_name/node') %]
-      tar -C $distdir -xf [% c('input_files_by_name/python') %]
-      export PATH="/var/tmp/dist/node/bin:/var/tmp/dist/python/bin:$PATH"
+      export PATH="/var/tmp/dist/node/bin:$PATH"
 
       tar -C $builddir -xf [% c('input_files_by_name/geckoview_armv7') %]
       tar -C $builddir -xf [% c('input_files_by_name/geckoview_aarch64') %]
@@ -108,9 +105,6 @@ steps:
       - project: node
         name: node
         pkg_type: build
-      - project: python
-        name: python
-        pkg_type: build
       - filename: mozconfig-android-all
         name: mozconfig
       - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
@@ -170,8 +164,6 @@ input_files:
     name: node
   - project: nasm
     name: nasm
-  - project: python
-    name: python
   - project: clang
     name: clang
   - filename: 'gradle-dependencies-[% c("var/gradle_dependencies_version") %]'
diff --git a/projects/glean/config b/projects/glean/config
index a1f08e4..66eb272 100644
--- a/projects/glean/config
+++ b/projects/glean/config
@@ -8,11 +8,22 @@ src: |
   #!/bin/bash
   mv -f [% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c('filename') %]
 
+var:
+  # The `container-image` project looks at the deps list in origin_project
+  # in the same step (build), so we cannot put this inside the
+  # `create_glean_deps_tarball` step.
+  # rbm#40020 should allow us to fix that.
+  deps:
+    - python3
+    - python3-pip
+    - torsocks
+    - xz-utils
+
 steps:
   create_glean_deps_tarball:
     git_url: ''
     version: 3.4.0
-    filename: 'glean-wheels-[% c("version") %].tar.xz'
+    filename: 'glean-wheels-[% c("version") %]-build2.tar.xz'
     build_log: '-'
     var:
       container:
@@ -20,10 +31,11 @@ steps:
     create_glean_deps_tarball: |
       #!/bin/bash
       [% c("var/set_default_env") -%]
-      [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
       mkdir glean-wheels
       cd glean-wheels
       [% IF c("var/use_torsocks") %]torsocks [% END%]python3 -m pip download glean_parser==[% c("version") %]
+      # Get pyyaml, needed for building application-services
+      [% IF c("var/use_torsocks") %]torsocks [% END%]python3 -m pip download pyyaml==5.3.1
       cd ..
       [% c('tar', {
           tar_src => [ 'glean-wheels' ],
@@ -36,18 +48,7 @@ steps:
     input_files:
       - project: container-image
         pkg_type: build
-      - project: python
-        name: python
-        pkg_type: build
 
 targets:
   with_torsocks:
-    var:
-      # The `container-image` project looks at the deps list in origin_project
-      # in the same step (build), so we cannot put this inside the
-      # `create_glean_deps_tarball` step.
-      # rbm#40020 should allow us to fix that.
-      deps:
-        - torsocks
-        - xz-utils
       use_torsocks: 1





More information about the tor-commits mailing list