[tor-commits] [Git][tpo/applications/tor-browser][base-browser-128.1.0esr-14.0-1] fixup! Add CI for Base Browser
Pier Angelo Vendrame (@pierov)
git at gitlab.torproject.org
Mon Aug 12 14:32:28 UTC 2024
Pier Angelo Vendrame pushed to branch base-browser-128.1.0esr-14.0-1 at The Tor Project / Applications / Tor Browser
Commits:
8f26cea2 by Beatriz Rizental at 2024-08-12T16:32:16+02:00
fixup! Add CI for Base Browser
Bug 42997: Reduce the number of git fetches by fetching during a setup
job and passing that initial fetch around as a git bundle to other jobs
through the Gitlab artifacts. Further reduce the number of fetches by
creating a list of changed files in the setup job and passing that
around as well as an artifact.
- - - - -
4 changed files:
- .gitlab-ci.yml
- .gitlab/ci/lint.yml
- .gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py
- + .gitlab/ci/setup.yml
Changes:
=====================================
.gitlab-ci.yml
=====================================
@@ -1,8 +1,10 @@
stages:
+ - setup
- lint
variables:
IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
include:
+ - local: '.gitlab/ci/setup.yml'
- local: '.gitlab/ci/lint.yml'
=====================================
.gitlab/ci/lint.yml
=====================================
@@ -1,8 +1,15 @@
.base:
stage: lint
+ image: $IMAGE_PATH
interruptible: true
+ needs:
+ - job: setup-env
+ artifacts: true
+ - job: create-bundle
+ artifacts: true
variables:
MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
+ GIT_STRATEGY: "none"
cache:
paths:
- node_modules
@@ -11,12 +18,26 @@
when: 'always'
# Share the cache throughout all pipelines running for a given branch
key: $CI_COMMIT_REF_SLUG
+ before_script:
+ # DEBUG: Are all artifacts here?
+ - ls -a
+ - mkdir app && cd app
+ # Initialize a fresh git repo
+ - git init
+ # Add app.bundle as the remote. All operations that communicate with the remote will be local.
+ - git remote add origin ../app.bundle
+ # shallow.txt contains the SHA of the base commit of the bundle.
+ # The bundle is shallow, therefore it's base commit will not have a parent.
+ # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need
+ # to crash when it realizes said base commit doesn't have a parent.
+ - cp ../shallow.txt .git/shallow
+ # Finally, unpack the bundle. Time it for debugging purposes.
+ - time git pull origin $BRANCH_NAME
eslint:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py eslint
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -44,9 +65,8 @@ eslint:
stylelint:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py stylelint
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -63,9 +83,8 @@ stylelint:
py-black:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py black
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -83,9 +102,8 @@ py-black:
py-ruff:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py ruff
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -103,9 +121,8 @@ py-ruff:
yaml:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py yaml
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -119,9 +136,8 @@ yaml:
shellcheck:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py shellcheck
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -134,10 +150,9 @@ shellcheck:
clang-format:
extends: .base
- image: $IMAGE_PATH
script:
- ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
- - .gitlab/ci/scripts/run_linters.py clang-format
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -155,9 +170,8 @@ clang-format:
rustfmt:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py rustfmt
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -170,9 +184,8 @@ rustfmt:
fluent-lint:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py fluent-lint
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -186,9 +199,8 @@ fluent-lint:
localization:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py l10n
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -204,9 +216,8 @@ localization:
mingw-capitalization:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py mingw-capitalization
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -222,9 +233,8 @@ mingw-capitalization:
mscom-init:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py mscom-init
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -240,9 +250,8 @@ mscom-init:
file-whitespace:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py file-whitespace
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -270,9 +279,8 @@ file-whitespace:
test-manifest:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
@@ -286,9 +294,8 @@ test-manifest:
trojan-source:
extends: .base
- image: $IMAGE_PATH
script:
- - .gitlab/ci/scripts/run_linters.py trojan-source
+ - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
=====================================
.gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py
=====================================
@@ -5,7 +5,6 @@ import os
import re
import shlex
import subprocess
-import sys
def git(command):
@@ -15,8 +14,8 @@ def git(command):
return result.stdout.strip()
-def get_firefox_tag_from_branch_name(branch_name):
- """Extracts the Firefox tag associated with a branch name.
+def get_firefox_tag(reference):
+ """Extracts the Firefox tag associated with a branch or tag name.
The "firefox tag" is the tag that marks
the end of the Mozilla commits and the start of the Tor Project commits.
@@ -25,26 +24,26 @@ def get_firefox_tag_from_branch_name(branch_name):
this function may return the incorrect reference number.
Args:
- branch_name: The branch name to extract the tag from.
+ reference: The branch or tag name to extract the Firefox tag from.
Expected format is tor-browser-91.2.0esr-11.0-1,
where 91.2.0esr is the Firefox version.
Returns:
The reference specifier of the matching Firefox tag.
- An exception wil be raised if anything goes wrong.
+ An exception will be raised if anything goes wrong.
"""
- # Extracts the version number from a branch name.
+ # Extracts the version number from a branch or tag name.
firefox_version = ""
- match = re.search(r"(?<=browser-)([^-]+)", branch_name)
+ match = re.search(r"(?<=browser-)([^-]+)", reference)
if match:
# TODO: Validate that what we got is actually a valid semver string?
firefox_version = match.group(1)
else:
- raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")
+ raise ValueError(f"Failed to extract version from reference '{reference}'.")
tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"
- remote_tags = git("ls-remote --tags")
+ remote_tags = git("ls-remote --tags origin")
# Each line looks like:
# 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1
@@ -54,7 +53,7 @@ def get_firefox_tag_from_branch_name(branch_name):
return match.group(0).split()[0]
else:
raise ValueError(
- f"Failed to find reference specifier for Firefox tag '{tag}' in branch '{branch_name}'."
+ f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'."
)
@@ -74,37 +73,42 @@ def get_list_of_changed_files():
base_reference = ""
if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
- # For merge requests, the base_reference is the common ancestor between the MR and the target branch.
+ # For merge requests, the base_reference is the common ancestor between the MR and the target branch
base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
else:
# When not in merge requests, the base reference is the Firefox tag
- base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))
+ base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
if not base_reference:
raise RuntimeError("No base reference found. There might be more errors above.")
# Fetch the tag reference
git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
- # Return the list of changed files
- return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")
+ # Return but filter the issue_templates files because those file names have spaces which can cause issues
+ return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
if __name__ == "__main__":
- parser = argparse.ArgumentParser(
- description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."
+ parser = argparse.ArgumentParser(description="")
+
+ parser.add_argument(
+ "--get-firefox-tag",
+ help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
+ type=str,
)
parser.add_argument(
- "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."
+ "--get-changed-files",
+ help="Get list of changed files."
+ "When running from a merge request get sthe list of changed files since the merge-base of the current branch."
+ "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
+ action="store_true",
)
+
args = parser.parse_args()
- changed_files = get_list_of_changed_files()
- if changed_files:
- command = ["./mach", "lint", "-v"]
- for linter in args.linters:
- command.extend(["-l", linter])
- command.extend(changed_files)
- result = subprocess.run(command, text=True)
- sys.exit(result.returncode)
+ if args.get_firefox_tag:
+ print(get_firefox_tag(args.get_firefox_tag))
+ elif args.get_changed_files:
+ print("\n".join(get_list_of_changed_files()))
else:
- print("No files changed, skipping linting.")
+ print("No valid option provided.")
=====================================
.gitlab/ci/setup.yml
=====================================
@@ -0,0 +1,53 @@
+setup-env:
+ stage: setup
+ interruptible: true
+ variables:
+ GIT_STRATEGY: "none"
+ rules:
+ - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
+ script:
+ - |
+ if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
+ echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env
+ else
+ echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env
+ fi
+ artifacts:
+ reports:
+ dotenv:
+ - build.env
+
+create-bundle:
+ stage: setup
+ # TODO: Find a better suited image, this one just has git.
+ image: python
+ needs:
+ - job: setup-env
+ artifacts: true
+ rules:
+ - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
+ variables:
+ GIT_DEPTH: "1"
+ interruptible: true
+ script:
+ # DEBUG: Check repository status.
+ - git status
+ # Switch to a named branch.
+ - git switch -c $BRANCH_NAME
+ # Create a git bundle -- this will generate the app.bundle file,
+ # which can be used as a git remote for offline fetching.
+ - git bundle create app.bundle --all
+ # Retain the SHA of the base of this shallow repository.
+ - cat .git/shallow > shallow.txt
+ # DEBUG: Check sizes.
+ - du -sh .git
+ - du -sh app.bundle
+ # Since this is the only job we have access to the Gitlab remote,
+ # let's get a list of changed files to use in the next jobs.
+ - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt
+ artifacts:
+ paths:
+ - app.bundle
+ - shallow.txt
+ - changedfiles.txt
+ expire_in: 1 hour
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/commit/8f26cea273d8f48e872c0cf1a941b55981ae0a84
--
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/commit/8f26cea273d8f48e872c0cf1a941b55981ae0a84
You're receiving this email because of your account on gitlab.torproject.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.torproject.org/pipermail/tor-commits/attachments/20240812/9a5d3a69/attachment-0001.htm>
More information about the tor-commits
mailing list