Pier Angelo Vendrame pushed to branch mullvad-browser-128.1.0esr-14.0-1 at The Tor Project / Applications / Mullvad Browser

Commits: 7c7be722 by Beatriz Rizental at 2024-08-12T16:31:55+02:00 fixup! Add CI for Base Browser

Bug 42997: Reduce the number of git fetches by fetching during a setup job and passing that initial fetch around as a git bundle to other jobs through the Gitlab artifacts. Further reduce the number of fetches by creating a list of changed files in the setup job and passing that around as well as an artifact.

- - - - -

4 changed files:

- .gitlab-ci.yml - .gitlab/ci/lint.yml - .gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py - + .gitlab/ci/setup.yml

Changes:

===================================== .gitlab-ci.yml ===================================== @@ -1,8 +1,10 @@ stages: + - setup - lint

variables: IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest

include: + - local: '.gitlab/ci/setup.yml' - local: '.gitlab/ci/lint.yml'

===================================== .gitlab/ci/lint.yml ===================================== @@ -1,8 +1,15 @@ .base: stage: lint + image: $IMAGE_PATH interruptible: true + needs: + - job: setup-env + artifacts: true + - job: create-bundle + artifacts: true variables: MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild" + GIT_STRATEGY: "none" cache: paths: - node_modules @@ -11,12 +18,26 @@ when: 'always' # Share the cache throughout all pipelines running for a given branch key: $CI_COMMIT_REF_SLUG + before_script: + # DEBUG: Are all artifacts here? + - ls -a + - mkdir app && cd app + # Initialize a fresh git repo + - git init + # Add app.bundle as the remote. All operations that communicate with the remote will be local. + - git remote add origin ../app.bundle + # shallow.txt contains the SHA of the base commit of the bundle. + # The bundle is shallow, therefore it's base commit will not have a parent. + # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need + # to crash when it realizes said base commit doesn't have a parent. + - cp ../shallow.txt .git/shallow + # Finally, unpack the bundle. Time it for debugging purposes. + - time git pull origin $BRANCH_NAME

eslint: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py eslint + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -44,9 +65,8 @@ eslint:

stylelint: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py stylelint + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -63,9 +83,8 @@ stylelint:

py-black: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py black + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -83,9 +102,8 @@ py-black:

py-ruff: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py ruff + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -103,9 +121,8 @@ py-ruff:

yaml: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py yaml + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -119,9 +136,8 @@ yaml:

shellcheck: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py shellcheck + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -134,10 +150,9 @@ shellcheck:

clang-format: extends: .base - image: $IMAGE_PATH script: - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0 - - .gitlab/ci/scripts/run_linters.py clang-format + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -155,9 +170,8 @@ clang-format:

rustfmt: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py rustfmt + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -170,9 +184,8 @@ rustfmt:

fluent-lint: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py fluent-lint + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -186,9 +199,8 @@ fluent-lint:

localization: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py l10n + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -204,9 +216,8 @@ localization:

mingw-capitalization: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py mingw-capitalization + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -222,9 +233,8 @@ mingw-capitalization:

mscom-init: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py mscom-init + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -240,9 +250,8 @@ mscom-init:

file-whitespace: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py file-whitespace + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -270,9 +279,8 @@ file-whitespace:

test-manifest: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -286,9 +294,8 @@ test-manifest:

trojan-source: extends: .base - image: $IMAGE_PATH script: - - .gitlab/ci/scripts/run_linters.py trojan-source + - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes:

===================================== .gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py ===================================== @@ -5,7 +5,6 @@ import os import re import shlex import subprocess -import sys

def git(command): @@ -15,8 +14,8 @@ def git(command): return result.stdout.strip()

-def get_firefox_tag_from_branch_name(branch_name): - """Extracts the Firefox tag associated with a branch name. +def get_firefox_tag(reference): + """Extracts the Firefox tag associated with a branch or tag name.

The "firefox tag" is the tag that marks the end of the Mozilla commits and the start of the Tor Project commits. @@ -25,26 +24,26 @@ def get_firefox_tag_from_branch_name(branch_name): this function may return the incorrect reference number.

Args: - branch_name: The branch name to extract the tag from. + reference: The branch or tag name to extract the Firefox tag from. Expected format is tor-browser-91.2.0esr-11.0-1, where 91.2.0esr is the Firefox version.

Returns: The reference specifier of the matching Firefox tag. - An exception wil be raised if anything goes wrong. + An exception will be raised if anything goes wrong. """

- # Extracts the version number from a branch name. + # Extracts the version number from a branch or tag name. firefox_version = "" - match = re.search(r"(?<=browser-)([^-]+)", branch_name) + match = re.search(r"(?<=browser-)([^-]+)", reference) if match: # TODO: Validate that what we got is actually a valid semver string? firefox_version = match.group(1) else: - raise ValueError(f"Failed to extract version from branch name '{branch_name}'.") + raise ValueError(f"Failed to extract version from reference '{reference}'.")

tag = f"FIREFOX_{firefox_version.replace('.', '_')}_" - remote_tags = git("ls-remote --tags") + remote_tags = git("ls-remote --tags origin")

# Each line looks like: # 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1 @@ -54,7 +53,7 @@ def get_firefox_tag_from_branch_name(branch_name): return match.group(0).split()[0] else: raise ValueError( - f"Failed to find reference specifier for Firefox tag '{tag}' in branch '{branch_name}'." + f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'." )

@@ -74,37 +73,42 @@ def get_list_of_changed_files(): base_reference = ""

if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event": - # For merge requests, the base_reference is the common ancestor between the MR and the target branch. + # For merge requests, the base_reference is the common ancestor between the MR and the target branch base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA") else: # When not in merge requests, the base reference is the Firefox tag - base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH")) + base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))

if not base_reference: raise RuntimeError("No base reference found. There might be more errors above.")

# Fetch the tag reference git(f"fetch origin {base_reference} --depth=1 --filter=blob:none") - # Return the list of changed files - return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n") + # Return but filter the issue_templates files because those file names have spaces which can cause issues + return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")

if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history." + parser = argparse.ArgumentParser(description="") + + parser.add_argument( + "--get-firefox-tag", + help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.", + type=str, ) parser.add_argument( - "linters", metavar="L", type=str, nargs="+", help="A list of linters to run." + "--get-changed-files", + help="Get list of changed files." + "When running from a merge request get sthe list of changed files since the merge-base of the current branch." + "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.", + action="store_true", ) + args = parser.parse_args()

- changed_files = get_list_of_changed_files() - if changed_files: - command = ["./mach", "lint", "-v"] - for linter in args.linters: - command.extend(["-l", linter]) - command.extend(changed_files) - result = subprocess.run(command, text=True) - sys.exit(result.returncode) + if args.get_firefox_tag: + print(get_firefox_tag(args.get_firefox_tag)) + elif args.get_changed_files: + print("\n".join(get_list_of_changed_files())) else: - print("No files changed, skipping linting.") + print("No valid option provided.")

===================================== .gitlab/ci/setup.yml ===================================== @@ -0,0 +1,53 @@ +setup-env: + stage: setup + interruptible: true + variables: + GIT_STRATEGY: "none" + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true' + script: + - | + if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then + echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env + else + echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env + fi + artifacts: + reports: + dotenv: + - build.env + +create-bundle: + stage: setup + # TODO: Find a better suited image, this one just has git. + image: python + needs: + - job: setup-env + artifacts: true + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true' + variables: + GIT_DEPTH: "1" + interruptible: true + script: + # DEBUG: Check repository status. + - git status + # Switch to a named branch. + - git switch -c $BRANCH_NAME + # Create a git bundle -- this will generate the app.bundle file, + # which can be used as a git remote for offline fetching. + - git bundle create app.bundle --all + # Retain the SHA of the base of this shallow repository. + - cat .git/shallow > shallow.txt + # DEBUG: Check sizes. + - du -sh .git + - du -sh app.bundle + # Since this is the only job we have access to the Gitlab remote, + # let's get a list of changed files to use in the next jobs. + - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt + artifacts: + paths: + - app.bundle + - shallow.txt + - changedfiles.txt + expire_in: 1 hour

View it on GitLab: https://gitlab.torproject.org/tpo/applications/mullvad-browser/-/commit/7c7b...