Pier Angelo Vendrame pushed to branch base-browser-115.13.0esr-13.5-1 at The Tor Project / Applications / Tor Browser

Commits: 7b915bf9 by Beatriz Rizental at 2024-07-11T11:20:16+02:00 Add CI for Base Browser

- - - - -

3 changed files:

- + .gitlab-ci.yml - + .gitlab/ci/lint.yml - + .gitlab/ci/scripts/run_linters.py

Changes:

===================================== .gitlab-ci.yml ===================================== @@ -0,0 +1,5 @@ +stages: + - lint + +include: + - local: '.gitlab/ci/lint.yml'

===================================== .gitlab/ci/lint.yml ===================================== @@ -0,0 +1,319 @@ +variables: + # This needs to be kept in sync with the max Python version accepted by ./mach + PYTHON_VERSION: "3.11.7" + +.base: + stage: lint + interruptible: true + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + cache: + paths: + - node_modules + - .cache/pip + +eslint: + extends: .base + image: cimg/python:$PYTHON_VERSION-node + script: + - .gitlab/ci/scripts/run_linters.py eslint + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + # Files that are likely audited. + - '**/*.js' + - '**/*.jsm' + - '**/*.json' + - '**/*.jsx' + - '**/*.mjs' + - '**/*.sjs' + - '**/*.html' + - '**/*.xhtml' + - '**/*.xml' + - 'tools/lint/eslint.yml' + # Run when eslint policies change. + - '**/.eslintignore' + - '**/*eslintrc*' + # The plugin implementing custom checks. + - 'tools/lint/eslint/eslint-plugin-mozilla/**' + - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +stylelint: + extends: .base + image: cimg/python:$PYTHON_VERSION-node + script: + - .gitlab/ci/scripts/run_linters.py stylelint + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + # Files that are likely audited. + - '**/*.css' + - 'tools/lint/styleint.yml' + # Run when stylelint policies change. + - '**/.stylelintignore' + - '**/*stylelintrc*' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +py-black: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py black + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + # The list of extensions should match tools/lint/black.yml + - '**/*.py' + - '**/moz.build' + - '**/*.configure' + - '**/*.mozbuild' + - 'pyproject.toml' + - 'tools/lint/black.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +py-ruff: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py ruff + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.py' + - '**/*.configure' + - '**/.ruff.toml' + - 'pyproject.toml' + - 'tools/lint/ruff.yml' + - 'tools/lint/python/ruff.py' + - 'tools/lint/python/ruff_requirements.txt' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +yaml: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py yaml + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.yml' + - '**/*.yaml' + - '**/.ymllint' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +shellcheck: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py shellcheck + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.sh' + - 'tools/lint/shellcheck.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +clang-format: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py clang-format + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.cpp' + - '**/*.c' + - '**/*.cc' + - '**/*.h' + - '**/*.m' + - '**/*.mm' + - 'tools/lint/clang-format.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +rustfmt: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py rustfmt + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.rs' + - 'tools/lint/rustfmt.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +fluent-lint: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py fluent-lint + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.ftl' + - 'tools/lint/fluent-lint.yml' + - 'tools/lint/fluent-lint/exclusions.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +localization: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py l10n + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/locales/en-US/**' + - '**/l10n.toml' + - 'third_party/python/compare-locales/**' + - 'third_party/python/fluent/**' + - 'tools/lint/l10n.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +mingw-capitalization: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py mingw-capitalization + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.cpp' + - '**/*.cc' + - '**/*.c' + - '**/*.h' + - 'tools/lint/mingw-capitalization.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +mscom-init: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py mscom-init + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.cpp' + - '**/*.cc' + - '**/*.c' + - '**/*.h' + - 'tools/lint/mscom-init.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +file-whitespace: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py file-whitespace + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.css' + - '**/*.dtd' + - '**/*.idl' + - '**/*.ftl' + - '**/*.h' + - '**/*.html' + - '**/*.md' + - '**/*.properties' + - '**/*.py' + - '**/*.rs' + - '**/*.rst' + - '**/*.webidl' + - '**/*.xhtml' + - 'tools/lint/file-whitespace.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +test-manifest: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.ini' + - 'python/mozlint/**' + - 'tools/lint/**' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +trojan-source: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py trojan-source + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.h' + - '**/*.py' + - '**/*.rs' + - 'tools/lint/trojan-source.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'

===================================== .gitlab/ci/scripts/run_linters.py ===================================== @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import shlex +import subprocess +import sys + + +def git(command): + result = subprocess.run( + ["git"] + shlex.split(command), check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def get_firefox_tag_from_branch_name(branch_name): + """Extracts the Firefox tag associated with a branch name. + + The "firefox tag" is the tag that marks + the end of the Mozilla commits and the start of the Tor Project commits. + + Know issue: If ever there is more than one tag per Firefox ESR version, + this function may return the incorrect reference number. + + Args: + branch_name: The branch name to extract the tag from. + Expected format is tor-browser-91.2.0esr-11.0-1, + where 91.2.0esr is the Firefox version. + + Returns: + The reference specifier of the matching Firefox tag. + An exception wil be raised if anything goes wrong. + """ + + # Extracts the version number from a branch name. + firefox_version = "" + match = re.search(r"(?<=browser-)([^-]+)", branch_name) + if match: + # TODO: Validate that what we got is actually a valid semver string? + firefox_version = match.group(1) + else: + raise ValueError(f"Failed to extract version from branch name '{branch_name}'.") + + tag = f"FIREFOX_{firefox_version.replace('.', '_')}_" + remote_tags = git("ls-remote --tags") + + # Each line looks like: + # 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1 + pattern = rf"(.*){re.escape(tag)}(.*)$" + match = re.search(pattern, remote_tags, flags=re.MULTILINE) + if match: + return match.group(0).split()[0] + else: + raise ValueError( + f"Failed to find reference specifier for Firefox tag in branch '{branch_name}'." + ) + + +def get_list_of_changed_files(): + """Gets a list of files changed in the working directory. + + This function is meant to be run inside the Gitlab CI environment. + + When running in a default branch, get the list of changed files since the last Firefox tag. + When running for a new MR commit, get a list of changed files in the current MR. + + Returns: + A list of filenames of changed files (excluding deleted files). + An exception wil be raised if anything goes wrong. + """ + + base_reference = "" + + if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event": + # For merge requests, the base_reference is the common ancestor between the MR and the target branch. + base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA") + else: + # When not in merge requests, the base reference is the Firefox tag + base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH")) + + if not base_reference: + raise RuntimeError("No base reference found. There might be more errors above.") + + # Fetch the tag reference + git(f"fetch origin {base_reference} --depth=1 --filter=blob:none") + # Return the list of changed files + return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history." + ) + parser.add_argument( + "linters", metavar="L", type=str, nargs="+", help="A list of linters to run." + ) + args = parser.parse_args() + + command = [ + "./mach", + "lint", + "-v", + *(s for l in args.linters for s in ("-l", l)), + *get_list_of_changed_files(), + ] + result = subprocess.run(command, text=True) + + sys.exit(result.returncode)

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/commit/7b915bf9...