[tor-commits] [tor-browser-build/master] Bug 40272: Add script to generate glean depencies tarballs

[sysrqb at torproject.org]

commit 4d562a83c40e4579250dc36666ef60d54601bc31
Author: Nicolas Vigier <boklm at torproject.org>
Date:   Fri Apr 9 17:28:55 2021 +0200

    Bug 40272: Add script to generate glean depencies tarballs
---
 Makefile                                       |  6 ++++
 projects/common/how-to-update-glean-parser.txt | 30 +++++++-----------
 projects/glean/config                          | 44 ++++++++++++++++++++++++++
 3 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index 96600e2..35e8834 100644
--- a/Makefile
+++ b/Makefile
@@ -213,6 +213,12 @@ list_toolchain_updates-application-services: submodule-update
 list_toolchain_updates-geckoview: submodule-update
 	$(rbm) build geckoview --step list_toolchain_updates --target nightly --target torbrowser-android-armv7
 
+create_glean_deps_tarball: submodule-update
+	 $(rbm)build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7
+
+create_glean_deps_tarball-with_torsocks: submodule-update
+	$(rbm) build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7 --target with_torsocks
+
 submodule-update:
 	git submodule update --init
 
diff --git a/projects/common/how-to-update-glean-parser.txt b/projects/common/how-to-update-glean-parser.txt
index a125f5f..625049e 100644
--- a/projects/common/how-to-update-glean-parser.txt
+++ b/projects/common/how-to-update-glean-parser.txt
@@ -3,42 +3,36 @@ distribute Glean dependencies beforehand.[1]
 
 There are three steps involved in the process:
   1) Finding out the `glean_parser` version
-  2) Downloading and verifying the dependencies
-  3) Bundling the dependencies up and pushing them to the sources mirror
+  2) Downloading and verifying the dependencies, and creating the tarball
+  3) Pushing it to the sources mirror
 
 1)
 
-The `glean_parser` version is available in glean-core/python/setup.py in the
-Glean code repositoroy.[2] So, make sure which Glean version your
-`android-components` tag is using (`mozilla_glean` in
-buildSrc/src/main/java/Dependencies.kt has it) and then track `glean_parser`
-down in the respective Glean tag.
+Running `make list_toolchain_updates-android-components` should tell
+you which version of `glean_parser` is needed.
 
 2)
 
-Create a directory `glean-wheels` and change into it. Then download packages for
-`glean_parser` and its dependencies using
+Update `steps/create_glean_deps_tarball/version` in `projects/glean/config`
+to the version from step 1 above.
 
-`python3 -m pip download glean_parser==$glean_parser`
+Run this command to generate the tarball:
 
-($glean_parser being the version from step 1 above and Python 3 needs to be 3.6;
-ideally, we take the artifact we built and will use during the build later on as
-well.)
+  make create_glean_deps_tarball
 
 There is no really good way (yet) to verify that `pip` gave us actually what we
 wanted. We can download `glean_parser` and its dependencies using a different
 network path, though, e.g. by using `torsocks`. Comparing the results of both
 downloads should yield the exact same binaries.
 
-3)
+If you want to use torsocks, you can use this command:
 
-Update the `glean_parser` version in `rbm.conf, tar `glean-wheels` up and
-compress it:
+  make create_glean_deps_tarball-with_torsocks
 
-`tar cfj glean-wheels-$glean_parser.tar.bz2 glean-wheels`
+3)
 
 Upload the compressed tarball to the sources mirror and update the SHA-256 sum of
-it in the respective projects.
+it as well as the `glean_parser` version in the respective projects.
 
 [1] See: https://bugzilla.mozilla.org/show_bug.cgi?id=1651662 for details.
 [2] https://github.com/mozilla/glean
diff --git a/projects/glean/config b/projects/glean/config
index 45bc43b..6fd6ce2 100644
--- a/projects/glean/config
+++ b/projects/glean/config
@@ -7,3 +7,47 @@ version: '[% pc(c("origin_project"), "var/glean_version", { step => "list_toolch
 src: |
   #!/bin/bash
   mv -f [% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c('filename') %]
+
+steps:
+  create_glean_deps_tarball:
+    git_url: ''
+    version: 2.2.0
+    filename: 'glean-wheels-[% c("version") %].tar.xz'
+    build_log: '-'
+    var:
+      container:
+        use_container: 1
+    create_glean_deps_tarball: |
+      #!/bin/bash
+      [% c("var/set_default_env") -%]
+      [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %]
+      mkdir glean-wheels
+      cd glean-wheels
+      [% IF c("var/use_torsocks") %]torsocks [% END%]python3 -m pip download glean_parser==[% c("version") %]
+      cd ..
+      [% c('tar', {
+          tar_src => [ 'glean-wheels' ],
+          tar_args => '-cJf ' _ dest_dir _ '/' _ c('filename'),
+          }) %]
+      cd '[% dest_dir %]'
+      sha256sum [% c('filename') %]
+      echo 'The file is ready to be uploaded:'
+      echo '  scp -p out/glean/[% c("filename") %] people.torproject.org:public_html/mirrors/sources/'
+    input_files:
+      - project: container-image
+        pkg_type: build
+      - project: python
+        name: python
+        pkg_type: build
+
+targets:
+  with_torsocks:
+    var:
+      # The `container-image` project looks at the deps list in origin_project
+      # in the same step (build), so we cannot put this inside the
+      # `create_glean_deps_tarball` step.
+      # rbm#40020 should allow us to fix that.
+      deps:
+        - torsocks
+        - xz-utils
+      use_torsocks: 1