commit ccd393a67f8c8aba2944fae23a52b07f951a048a Author: Nicolas Vigier boklm@torproject.org Date: Fri Apr 9 17:28:55 2021 +0200

Bug 40272: Add script to generate glean depencies tarballs --- Makefile | 6 ++++ projects/common/how-to-update-glean-parser.txt | 30 +++++++----------- projects/glean/config | 44 ++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile index fa613e1..d4562ac 100644 --- a/Makefile +++ b/Makefile @@ -213,6 +213,12 @@ list_toolchain_updates-application-services: submodule-update list_toolchain_updates-geckoview: submodule-update $(rbm) build geckoview --step list_toolchain_updates --target nightly --target torbrowser-android-armv7

+create_glean_deps_tarball: submodule-update + $(rbm)build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7 + +create_glean_deps_tarball-with_torsocks: submodule-update + $(rbm) build glean --step create_glean_deps_tarball --target alpha --target torbrowser-android-armv7 --target with_torsocks + submodule-update: git submodule update --init

diff --git a/projects/common/how-to-update-glean-parser.txt b/projects/common/how-to-update-glean-parser.txt index a125f5f..625049e 100644 --- a/projects/common/how-to-update-glean-parser.txt +++ b/projects/common/how-to-update-glean-parser.txt @@ -3,42 +3,36 @@ distribute Glean dependencies beforehand.[1]

There are three steps involved in the process: 1) Finding out the `glean_parser` version - 2) Downloading and verifying the dependencies - 3) Bundling the dependencies up and pushing them to the sources mirror + 2) Downloading and verifying the dependencies, and creating the tarball + 3) Pushing it to the sources mirror

1)

-The `glean_parser` version is available in glean-core/python/setup.py in the -Glean code repositoroy.[2] So, make sure which Glean version your -`android-components` tag is using (`mozilla_glean` in -buildSrc/src/main/java/Dependencies.kt has it) and then track `glean_parser` -down in the respective Glean tag. +Running `make list_toolchain_updates-android-components` should tell +you which version of `glean_parser` is needed.

2)

-Create a directory `glean-wheels` and change into it. Then download packages for -`glean_parser` and its dependencies using +Update `steps/create_glean_deps_tarball/version` in `projects/glean/config` +to the version from step 1 above.

-`python3 -m pip download glean_parser==$glean_parser` +Run this command to generate the tarball:

-($glean_parser being the version from step 1 above and Python 3 needs to be 3.6; -ideally, we take the artifact we built and will use during the build later on as -well.) + make create_glean_deps_tarball

There is no really good way (yet) to verify that `pip` gave us actually what we wanted. We can download `glean_parser` and its dependencies using a different network path, though, e.g. by using `torsocks`. Comparing the results of both downloads should yield the exact same binaries.

-3) +If you want to use torsocks, you can use this command:

-Update the `glean_parser` version in `rbm.conf, tar `glean-wheels` up and -compress it: + make create_glean_deps_tarball-with_torsocks

-`tar cfj glean-wheels-$glean_parser.tar.bz2 glean-wheels` +3)

Upload the compressed tarball to the sources mirror and update the SHA-256 sum of -it in the respective projects. +it as well as the `glean_parser` version in the respective projects.

[1] See: https://bugzilla.mozilla.org/show_bug.cgi?id=1651662 for details. [2] https://github.com/mozilla/glean diff --git a/projects/glean/config b/projects/glean/config index 45bc43b..6fd6ce2 100644 --- a/projects/glean/config +++ b/projects/glean/config @@ -7,3 +7,47 @@ version: '[% pc(c("origin_project"), "var/glean_version", { step => "list_toolch src: | #!/bin/bash mv -f [% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c('filename') %] + +steps: + create_glean_deps_tarball: + git_url: '' + version: 2.2.0 + filename: 'glean-wheels-[% c("version") %].tar.xz' + build_log: '-' + var: + container: + use_container: 1 + create_glean_deps_tarball: | + #!/bin/bash + [% c("var/set_default_env") -%] + [% pc('python', 'var/setup', { python_tarfile => c('input_files_by_name/python') }) %] + mkdir glean-wheels + cd glean-wheels + [% IF c("var/use_torsocks") %]torsocks [% END%]python3 -m pip download glean_parser==[% c("version") %] + cd .. + [% c('tar', { + tar_src => [ 'glean-wheels' ], + tar_args => '-cJf ' _ dest_dir _ '/' _ c('filename'), + }) %] + cd '[% dest_dir %]' + sha256sum [% c('filename') %] + echo 'The file is ready to be uploaded:' + echo ' scp -p out/glean/[% c("filename") %] people.torproject.org:public_html/mirrors/sources/' + input_files: + - project: container-image + pkg_type: build + - project: python + name: python + pkg_type: build + +targets: + with_torsocks: + var: + # The `container-image` project looks at the deps list in origin_project + # in the same step (build), so we cannot put this inside the + # `create_glean_deps_tarball` step. + # rbm#40020 should allow us to fix that. + deps: + - torsocks + - xz-utils + use_torsocks: 1