[tor-commits] [Git][tpo/applications/tor-browser][tor-browser-115.13.0esr-13.5-1] 3 commits: Revert "Add CI for Tor Browser"

Pier Angelo Vendrame (@pierov) git at gitlab.torproject.org
Thu Jul 11 08:53:37 UTC 2024



Pier Angelo Vendrame pushed to branch tor-browser-115.13.0esr-13.5-1 at The Tor Project / Applications / Tor Browser


Commits:
8b8656d6 by Beatriz Rizental at 2024-07-09T19:31:40+02:00
Revert "Add CI for Tor Browser"

This reverts commit ec03736019a0337b4a81f7c6566119337c2d53aa.

- - - - -
e20fbd3e by Beatriz Rizental at 2024-07-11T10:50:01+02:00
Add CI for Base Browser

- - - - -
065fd124 by Beatriz Rizental at 2024-07-11T10:50:28+02:00
Add CI for Tor Browser

- - - - -


4 changed files:

- .gitlab-ci.yml
- + .gitlab/ci/lint.yml
- + .gitlab/ci/scripts/run_linters.py
- + .gitlab/ci/update-translations.yml


Changes:

=====================================
.gitlab-ci.yml
=====================================
@@ -1,69 +1,7 @@
 stages:
+  - lint
   - update-translations
 
-.update-translation-base:
-  stage: update-translations
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      changes:
-        - "**/*.ftl"
-        - "**/*.properties"
-        - "**/*.dtd"
-    - if: $FORCE_UPDATE_TRANSLATIONS == "true"
-  variables:
-    TOR_BROWSER_COMBINED_FILES_JSON: "combined-translation-files.json"
-
-
-combine-en-US-translations:
-  extends: .update-translation-base
-  image: python
-  variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-    TRANSLATION_FILES: '
-      tor-browser:tor-browser.ftl
-      tor-browser:aboutDialog.dtd
-      tor-browser:aboutTBUpdate.dtd
-      tor-browser:aboutTor.dtd
-      tor-browser:torbutton.dtd
-      tor-browser:browserOnboarding.properties
-      tor-browser:cryptoSafetyPrompt.properties
-      tor-browser:onboarding.properties
-      tor-browser:onionLocation.properties
-      tor-browser:rulesets.properties
-      tor-browser:settings.properties
-      tor-browser:torbutton.properties
-      tor-browser:torConnect.properties
-      tor-browser:torlauncher.properties
-      base-browser:base-browser.ftl
-      base-browser:newIdentity.properties
-      base-browser:securityLevel.properties
-    '
-  cache:
-    paths:
-      - .cache/pip
-  # Artifact is for translation project job
-  artifacts:
-    paths:
-      - "$TOR_BROWSER_COMBINED_FILES_JSON"
-    expire_in: "60 min"
-    reports:
-      dotenv: job_id.env
-  # Don't load artifacts for this job.
-  dependencies: []
-  script:
-    # Save this CI_JOB_ID to the dotenv file to be used in the variables for the
-    # push-en-US-translations job.
-    - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env
-    - pip install compare_locales
-    - python ./tools/torbrowser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$TOR_BROWSER_COMBINED_FILES_JSON"
-
-push-en-US-translations:
-  extends: .update-translation-base
-  needs:
-    - job: combine-en-US-translations
-  variables:
-    TOR_BROWSER_COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${TOR_BROWSER_COMBINED_FILES_JSON}"
-  trigger:
-    strategy: depend
-    project: tor-browser-translation-bot/translation
-    branch: tor-browser-ci
+include:
+  - local: '.gitlab/ci/lint.yml'
+  - local: '.gitlab/ci/update-translations.yml'


=====================================
.gitlab/ci/lint.yml
=====================================
@@ -0,0 +1,319 @@
+variables:
+  # This needs to be kept in sync with the max Python version accepted by ./mach
+  PYTHON_VERSION: "3.11.7"
+
+.base:
+  stage: lint
+  interruptible: true
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  cache:
+    paths:
+      - node_modules
+      - .cache/pip
+
+eslint:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION-node
+  script:
+    - .gitlab/ci/scripts/run_linters.py eslint
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        # Files that are likely audited.
+        - '**/*.js'
+        - '**/*.jsm'
+        - '**/*.json'
+        - '**/*.jsx'
+        - '**/*.mjs'
+        - '**/*.sjs'
+        - '**/*.html'
+        - '**/*.xhtml'
+        - '**/*.xml'
+        - 'tools/lint/eslint.yml'
+        # Run when eslint policies change.
+        - '**/.eslintignore'
+        - '**/*eslintrc*'
+        # The plugin implementing custom checks.
+        - 'tools/lint/eslint/eslint-plugin-mozilla/**'
+        - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+stylelint:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION-node
+  script:
+    - .gitlab/ci/scripts/run_linters.py stylelint
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        # Files that are likely audited.
+        - '**/*.css'
+        - 'tools/lint/styleint.yml'
+        # Run when stylelint policies change.
+        - '**/.stylelintignore'
+        - '**/*stylelintrc*'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+py-black:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py black
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        # The list of extensions should match tools/lint/black.yml
+        - '**/*.py'
+        - '**/moz.build'
+        - '**/*.configure'
+        - '**/*.mozbuild'
+        - 'pyproject.toml'
+        - 'tools/lint/black.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+py-ruff:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py ruff
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.py'
+        - '**/*.configure'
+        - '**/.ruff.toml'
+        - 'pyproject.toml'
+        - 'tools/lint/ruff.yml'
+        - 'tools/lint/python/ruff.py'
+        - 'tools/lint/python/ruff_requirements.txt'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+yaml:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py yaml
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.yml'
+        - '**/*.yaml'
+        - '**/.ymllint'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+shellcheck:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py shellcheck
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.sh'
+        - 'tools/lint/shellcheck.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+clang-format:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py clang-format
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.cpp'
+        - '**/*.c'
+        - '**/*.cc'
+        - '**/*.h'
+        - '**/*.m'
+        - '**/*.mm'
+        - 'tools/lint/clang-format.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+rustfmt:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py rustfmt
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.rs'
+        - 'tools/lint/rustfmt.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+fluent-lint:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py fluent-lint
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.ftl'
+        - 'tools/lint/fluent-lint.yml'
+        - 'tools/lint/fluent-lint/exclusions.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+localization:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py l10n
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/locales/en-US/**'
+        - '**/l10n.toml'
+        - 'third_party/python/compare-locales/**'
+        - 'third_party/python/fluent/**'
+        - 'tools/lint/l10n.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+mingw-capitalization:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py mingw-capitalization
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.cpp'
+        - '**/*.cc'
+        - '**/*.c'
+        - '**/*.h'
+        - 'tools/lint/mingw-capitalization.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+mscom-init:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py mscom-init
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.cpp'
+        - '**/*.cc'
+        - '**/*.c'
+        - '**/*.h'
+        - 'tools/lint/mscom-init.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+file-whitespace:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py file-whitespace
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.c'
+        - '**/*.cc'
+        - '**/*.cpp'
+        - '**/*.css'
+        - '**/*.dtd'
+        - '**/*.idl'
+        - '**/*.ftl'
+        - '**/*.h'
+        - '**/*.html'
+        - '**/*.md'
+        - '**/*.properties'
+        - '**/*.py'
+        - '**/*.rs'
+        - '**/*.rst'
+        - '**/*.webidl'
+        - '**/*.xhtml'
+        - 'tools/lint/file-whitespace.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+test-manifest:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.ini'
+        - 'python/mozlint/**'
+        - 'tools/lint/**'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'
+
+trojan-source:
+  extends: .base
+  image: cimg/python:$PYTHON_VERSION
+  script:
+    - .gitlab/ci/scripts/run_linters.py trojan-source
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        # List copied from: taskcluster/ci/source-test/mozlint.yml
+        #
+        - '**/*.c'
+        - '**/*.cc'
+        - '**/*.cpp'
+        - '**/*.h'
+        - '**/*.py'
+        - '**/*.rs'
+        - 'tools/lint/trojan-source.yml'
+    # Run job whenever a new tag is created
+    # or whenever a commit is merged to a protected branch
+    - if: $CI_COMMIT_TAG || $CI_COMMIT_REF_PROTECTED == 'true'


=====================================
.gitlab/ci/scripts/run_linters.py
=====================================
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import shlex
+import subprocess
+import sys
+
+
+def git(command):
+    result = subprocess.run(
+        ["git"] + shlex.split(command), check=True, capture_output=True, text=True
+    )
+    return result.stdout.strip()
+
+
+def get_firefox_tag_from_branch_name(branch_name):
+    """Extracts the Firefox tag associated with a branch name.
+
+       The "firefox tag" is the tag that marks
+       the end of the Mozilla commits and the start of the Tor Project commits.
+
+       Know issue: If ever there is more than one tag per Firefox ESR version,
+       this function may return the incorrect reference number.
+
+    Args:
+        branch_name: The branch name to extract the tag from.
+        Expected format is tor-browser-91.2.0esr-11.0-1,
+        where 91.2.0esr is the Firefox version.
+
+    Returns:
+        The reference specifier of the matching Firefox tag.
+        An exception wil be raised if anything goes wrong.
+    """
+
+    # Extracts the version number from a branch name.
+    firefox_version = ""
+    match = re.search(r"(?<=browser-)([^-]+)", branch_name)
+    if match:
+        # TODO: Validate that what we got is actually a valid semver string?
+        firefox_version = match.group(1)
+    else:
+        raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")
+
+    tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"
+    remote_tags = git("ls-remote --tags")
+
+    # Each line looks like:
+    # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1
+    pattern = rf"(.*){re.escape(tag)}(.*)$"
+    match = re.search(pattern, remote_tags, flags=re.MULTILINE)
+    if match:
+        return match.group(0).split()[0]
+    else:
+        raise ValueError(
+            f"Failed to find reference specifier for Firefox tag in branch '{branch_name}'."
+        )
+
+
+def get_list_of_changed_files():
+    """Gets a list of files changed in the working directory.
+
+       This function is meant to be run inside the Gitlab CI environment.
+
+       When running in a default branch, get the list of changed files since the last Firefox tag.
+       When running for a new MR commit, get a list of changed files in the current MR.
+
+    Returns:
+        A list of filenames of changed files (excluding deleted files).
+        An exception wil be raised if anything goes wrong.
+    """
+
+    base_reference = ""
+
+    if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
+        # For merge requests, the base_reference is the common ancestor between the MR and the target branch.
+        base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
+    else:
+        # When not in merge requests, the base reference is the Firefox tag
+        base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))
+
+    if not base_reference:
+        raise RuntimeError("No base reference found. There might be more errors above.")
+
+    # Fetch the tag reference
+    git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
+    # Return the list of changed files
+    return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."
+    )
+    parser.add_argument(
+        "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."
+    )
+    args = parser.parse_args()
+
+    command = [
+        "./mach",
+        "lint",
+        "-v",
+        *(s for l in args.linters for s in ("-l", l)),
+        *get_list_of_changed_files(),
+    ]
+    result = subprocess.run(command, text=True)
+
+    sys.exit(result.returncode)


=====================================
.gitlab/ci/update-translations.yml
=====================================
@@ -0,0 +1,66 @@
+.update-translation-base:
+  stage: update-translations
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      changes:
+        - "**/*.ftl"
+        - "**/*.properties"
+        - "**/*.dtd"
+    - if: $FORCE_UPDATE_TRANSLATIONS == "true"
+  variables:
+    TOR_BROWSER_COMBINED_FILES_JSON: "combined-translation-files.json"
+
+
+combine-en-US-translations:
+  extends: .update-translation-base
+  image: python
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    TRANSLATION_FILES: '
+      tor-browser:tor-browser.ftl
+      tor-browser:aboutDialog.dtd
+      tor-browser:aboutTBUpdate.dtd
+      tor-browser:aboutTor.dtd
+      tor-browser:torbutton.dtd
+      tor-browser:browserOnboarding.properties
+      tor-browser:cryptoSafetyPrompt.properties
+      tor-browser:onboarding.properties
+      tor-browser:onionLocation.properties
+      tor-browser:rulesets.properties
+      tor-browser:settings.properties
+      tor-browser:torbutton.properties
+      tor-browser:torConnect.properties
+      tor-browser:torlauncher.properties
+      base-browser:base-browser.ftl
+      base-browser:newIdentity.properties
+      base-browser:securityLevel.properties
+    '
+  cache:
+    paths:
+      - .cache/pip
+  # Artifact is for translation project job
+  artifacts:
+    paths:
+      - "$TOR_BROWSER_COMBINED_FILES_JSON"
+    expire_in: "60 min"
+    reports:
+      dotenv: job_id.env
+  # Don't load artifacts for this job.
+  dependencies: []
+  script:
+    # Save this CI_JOB_ID to the dotenv file to be used in the variables for the
+    # push-en-US-translations job.
+    - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env
+    - pip install compare_locales
+    - python ./tools/torbrowser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$TOR_BROWSER_COMBINED_FILES_JSON"
+
+push-en-US-translations:
+  extends: .update-translation-base
+  needs:
+    - job: combine-en-US-translations
+  variables:
+    TOR_BROWSER_COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${TOR_BROWSER_COMBINED_FILES_JSON}"
+  trigger:
+    strategy: depend
+    project: tor-browser-translation-bot/translation
+    branch: tor-browser-ci



View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/dd7037b5ccfadfb30c0cb7b88987d0aac46abd65...065fd12416f4076fcb8f3abbaba6de60847c0823

-- 
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/dd7037b5ccfadfb30c0cb7b88987d0aac46abd65...065fd12416f4076fcb8f3abbaba6de60847c0823
You're receiving this email because of your account on gitlab.torproject.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.torproject.org/pipermail/tor-commits/attachments/20240711/81055877/attachment-0001.htm>


More information about the tor-commits mailing list