[Git][tpo/applications/tor-browser][tor-browser-115.13.0esr-13.5-1] 3 commits: Revert "Add CI for Tor Browser"
Pier Angelo Vendrame pushed to branch tor-browser-115.13.0esr-13.5-1 at The Tor Project / Applications / Tor Browser
Commits: 8b8656d6 by Beatriz Rizental at 2024-07-09T19:31:40+02:00 Revert "Add CI for Tor Browser"
This reverts commit ec03736019a0337b4a81f7c6566119337c2d53aa.
- - - - - e20fbd3e by Beatriz Rizental at 2024-07-11T10:50:01+02:00 Add CI for Base Browser
- - - - - 065fd124 by Beatriz Rizental at 2024-07-11T10:50:28+02:00 Add CI for Tor Browser
- - - - -
4 changed files:
- .gitlab-ci.yml - + .gitlab/ci/lint.yml - + .gitlab/ci/scripts/run_linters.py - + .gitlab/ci/update-translations.yml
Changes:
===================================== .gitlab-ci.yml ===================================== @@ -1,69 +1,7 @@ stages: + - lint - update-translations
-.update-translation-base: - stage: update-translations - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - changes: - - "**/*.ftl" - - "**/*.properties" - - "**/*.dtd" - - if: $FORCE_UPDATE_TRANSLATIONS == "true" - variables: - TOR_BROWSER_COMBINED_FILES_JSON: "combined-translation-files.json" - - -combine-en-US-translations: - extends: .update-translation-base - image: python - variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - TRANSLATION_FILES: ' - tor-browser:tor-browser.ftl - tor-browser:aboutDialog.dtd - tor-browser:aboutTBUpdate.dtd - tor-browser:aboutTor.dtd - tor-browser:torbutton.dtd - tor-browser:browserOnboarding.properties - tor-browser:cryptoSafetyPrompt.properties - tor-browser:onboarding.properties - tor-browser:onionLocation.properties - tor-browser:rulesets.properties - tor-browser:settings.properties - tor-browser:torbutton.properties - tor-browser:torConnect.properties - tor-browser:torlauncher.properties - base-browser:base-browser.ftl - base-browser:newIdentity.properties - base-browser:securityLevel.properties - ' - cache: - paths: - - .cache/pip - # Artifact is for translation project job - artifacts: - paths: - - "$TOR_BROWSER_COMBINED_FILES_JSON" - expire_in: "60 min" - reports: - dotenv: job_id.env - # Don't load artifacts for this job. - dependencies: [] - script: - # Save this CI_JOB_ID to the dotenv file to be used in the variables for the - # push-en-US-translations job. - - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env - - pip install compare_locales - - python ./tools/torbrowser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$TOR_BROWSER_COMBINED_FILES_JSON" - -push-en-US-translations: - extends: .update-translation-base - needs: - - job: combine-en-US-translations - variables: - TOR_BROWSER_COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${TOR_BROWSER_COMBINED_FILES_JSON}" - trigger: - strategy: depend - project: tor-browser-translation-bot/translation - branch: tor-browser-ci +include: + - local: '.gitlab/ci/lint.yml' + - local: '.gitlab/ci/update-translations.yml'
===================================== .gitlab/ci/lint.yml ===================================== @@ -0,0 +1,319 @@ +variables: + # This needs to be kept in sync with the max Python version accepted by ./mach + PYTHON_VERSION: "3.11.7" + +.base: + stage: lint + interruptible: true + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + cache: + paths: + - node_modules + - .cache/pip + +eslint: + extends: .base + image: cimg/python:$PYTHON_VERSION-node + script: + - .gitlab/ci/scripts/run_linters.py eslint + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + # Files that are likely audited. + - '**/*.js' + - '**/*.jsm' + - '**/*.json' + - '**/*.jsx' + - '**/*.mjs' + - '**/*.sjs' + - '**/*.html' + - '**/*.xhtml' + - '**/*.xml' + - 'tools/lint/eslint.yml' + # Run when eslint policies change. + - '**/.eslintignore' + - '**/*eslintrc*' + # The plugin implementing custom checks. + - 'tools/lint/eslint/eslint-plugin-mozilla/**' + - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +stylelint: + extends: .base + image: cimg/python:$PYTHON_VERSION-node + script: + - .gitlab/ci/scripts/run_linters.py stylelint + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + # Files that are likely audited. + - '**/*.css' + - 'tools/lint/styleint.yml' + # Run when stylelint policies change. + - '**/.stylelintignore' + - '**/*stylelintrc*' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +py-black: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py black + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + # The list of extensions should match tools/lint/black.yml + - '**/*.py' + - '**/moz.build' + - '**/*.configure' + - '**/*.mozbuild' + - 'pyproject.toml' + - 'tools/lint/black.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +py-ruff: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py ruff + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.py' + - '**/*.configure' + - '**/.ruff.toml' + - 'pyproject.toml' + - 'tools/lint/ruff.yml' + - 'tools/lint/python/ruff.py' + - 'tools/lint/python/ruff_requirements.txt' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +yaml: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py yaml + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.yml' + - '**/*.yaml' + - '**/.ymllint' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +shellcheck: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py shellcheck + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.sh' + - 'tools/lint/shellcheck.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +clang-format: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py clang-format + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.cpp' + - '**/*.c' + - '**/*.cc' + - '**/*.h' + - '**/*.m' + - '**/*.mm' + - 'tools/lint/clang-format.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +rustfmt: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py rustfmt + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.rs' + - 'tools/lint/rustfmt.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +fluent-lint: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py fluent-lint + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.ftl' + - 'tools/lint/fluent-lint.yml' + - 'tools/lint/fluent-lint/exclusions.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +localization: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py l10n + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/locales/en-US/**' + - '**/l10n.toml' + - 'third_party/python/compare-locales/**' + - 'third_party/python/fluent/**' + - 'tools/lint/l10n.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +mingw-capitalization: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py mingw-capitalization + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.cpp' + - '**/*.cc' + - '**/*.c' + - '**/*.h' + - 'tools/lint/mingw-capitalization.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +mscom-init: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py mscom-init + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.cpp' + - '**/*.cc' + - '**/*.c' + - '**/*.h' + - 'tools/lint/mscom-init.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +file-whitespace: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py file-whitespace + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.css' + - '**/*.dtd' + - '**/*.idl' + - '**/*.ftl' + - '**/*.h' + - '**/*.html' + - '**/*.md' + - '**/*.properties' + - '**/*.py' + - '**/*.rs' + - '**/*.rst' + - '**/*.webidl' + - '**/*.xhtml' + - 'tools/lint/file-whitespace.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +test-manifest: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.ini' + - 'python/mozlint/**' + - 'tools/lint/**' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true' + +trojan-source: + extends: .base + image: cimg/python:$PYTHON_VERSION + script: + - .gitlab/ci/scripts/run_linters.py trojan-source + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + changes: + # List copied from: taskcluster/ci/source-test/mozlint.yml + # + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.h' + - '**/*.py' + - '**/*.rs' + - 'tools/lint/trojan-source.yml' + # Run job whenever a new tag is created + # or whenever a commit is merged to a protected branch + - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
===================================== .gitlab/ci/scripts/run_linters.py ===================================== @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import shlex +import subprocess +import sys + + +def git(command): + result = subprocess.run( + ["git"] + shlex.split(command), check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def get_firefox_tag_from_branch_name(branch_name): + """Extracts the Firefox tag associated with a branch name. + + The "firefox tag" is the tag that marks + the end of the Mozilla commits and the start of the Tor Project commits. + + Know issue: If ever there is more than one tag per Firefox ESR version, + this function may return the incorrect reference number. + + Args: + branch_name: The branch name to extract the tag from. + Expected format is tor-browser-91.2.0esr-11.0-1, + where 91.2.0esr is the Firefox version. + + Returns: + The reference specifier of the matching Firefox tag. + An exception wil be raised if anything goes wrong. + """ + + # Extracts the version number from a branch name. + firefox_version = "" + match = re.search(r"(?<=browser-)([^-]+)", branch_name) + if match: + # TODO: Validate that what we got is actually a valid semver string? + firefox_version = match.group(1) + else: + raise ValueError(f"Failed to extract version from branch name '{branch_name}'.") + + tag = f"FIREFOX_{firefox_version.replace('.', '_')}_" + remote_tags = git("ls-remote --tags") + + # Each line looks like: + # 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1 + pattern = rf"(.*){re.escape(tag)}(.*)$" + match = re.search(pattern, remote_tags, flags=re.MULTILINE) + if match: + return match.group(0).split()[0] + else: + raise ValueError( + f"Failed to find reference specifier for Firefox tag in branch '{branch_name}'." + ) + + +def get_list_of_changed_files(): + """Gets a list of files changed in the working directory. + + This function is meant to be run inside the Gitlab CI environment. + + When running in a default branch, get the list of changed files since the last Firefox tag. + When running for a new MR commit, get a list of changed files in the current MR. + + Returns: + A list of filenames of changed files (excluding deleted files). + An exception wil be raised if anything goes wrong. + """ + + base_reference = "" + + if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event": + # For merge requests, the base_reference is the common ancestor between the MR and the target branch. + base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA") + else: + # When not in merge requests, the base reference is the Firefox tag + base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH")) + + if not base_reference: + raise RuntimeError("No base reference found. There might be more errors above.") + + # Fetch the tag reference + git(f"fetch origin {base_reference} --depth=1 --filter=blob:none") + # Return the list of changed files + return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history." + ) + parser.add_argument( + "linters", metavar="L", type=str, nargs="+", help="A list of linters to run." + ) + args = parser.parse_args() + + command = [ + "./mach", + "lint", + "-v", + *(s for l in args.linters for s in ("-l", l)), + *get_list_of_changed_files(), + ] + result = subprocess.run(command, text=True) + + sys.exit(result.returncode)
===================================== .gitlab/ci/update-translations.yml ===================================== @@ -0,0 +1,66 @@ +.update-translation-base: + stage: update-translations + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - "**/*.ftl" + - "**/*.properties" + - "**/*.dtd" + - if: $FORCE_UPDATE_TRANSLATIONS == "true" + variables: + TOR_BROWSER_COMBINED_FILES_JSON: "combined-translation-files.json" + + +combine-en-US-translations: + extends: .update-translation-base + image: python + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + TRANSLATION_FILES: ' + tor-browser:tor-browser.ftl + tor-browser:aboutDialog.dtd + tor-browser:aboutTBUpdate.dtd + tor-browser:aboutTor.dtd + tor-browser:torbutton.dtd + tor-browser:browserOnboarding.properties + tor-browser:cryptoSafetyPrompt.properties + tor-browser:onboarding.properties + tor-browser:onionLocation.properties + tor-browser:rulesets.properties + tor-browser:settings.properties + tor-browser:torbutton.properties + tor-browser:torConnect.properties + tor-browser:torlauncher.properties + base-browser:base-browser.ftl + base-browser:newIdentity.properties + base-browser:securityLevel.properties + ' + cache: + paths: + - .cache/pip + # Artifact is for translation project job + artifacts: + paths: + - "$TOR_BROWSER_COMBINED_FILES_JSON" + expire_in: "60 min" + reports: + dotenv: job_id.env + # Don't load artifacts for this job. + dependencies: [] + script: + # Save this CI_JOB_ID to the dotenv file to be used in the variables for the + # push-en-US-translations job. + - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env + - pip install compare_locales + - python ./tools/torbrowser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$TOR_BROWSER_COMBINED_FILES_JSON" + +push-en-US-translations: + extends: .update-translation-base + needs: + - job: combine-en-US-translations + variables: + TOR_BROWSER_COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${TOR_BROWSER_COMBINED_FILES_JSON}" + trigger: + strategy: depend + project: tor-browser-translation-bot/translation + branch: tor-browser-ci
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/dd7037b...
tbb-commits@lists.torproject.org
-
Pier Angelo Vendrame (＠pierov)