Pier Angelo Vendrame pushed to branch mullvad-browser-128.1.0esr-14.0-1 at The Tor Project / Applications / Mullvad Browser

Commits:

4 changed files:

Changes:

  • .gitlab-ci.yml
    1 1 
     stages:
    2 
    +  - setup
    2 3 
       - lint
    3 4
    4 5 
     variables:
    5 6 
       IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
    6 7
    7 8 
     include:
    9 
    +  - local: '.gitlab/ci/setup.yml'
    8 10 
       - local: '.gitlab/ci/lint.yml'

  • .gitlab/ci/lint.yml
    1 1 
     .base:
    2 2 
       stage: lint
    3 
    +  image: $IMAGE_PATH
    3 4 
       interruptible: true
    5 
    +  needs:
    6 
    +    - job: setup-env
    7 
    +      artifacts: true
    8 
    +    - job: create-bundle
    9 
    +      artifacts: true
    4 10 
       variables:
    5 11 
         MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
    12 
    +    GIT_STRATEGY: "none"
    6 13 
       cache:
    7 14 
         paths:
    8 15 
           - node_modules
    ... ... @@ -11,12 +18,26 @@
    11 18 
         when: 'always'
    12 19 
         # Share the cache throughout all pipelines running for a given branch
    13 20 
         key: $CI_COMMIT_REF_SLUG
    21 
    +  before_script:
    22 
    +    # DEBUG: Are all artifacts here?
    23 
    +    - ls -a
    24 
    +    - mkdir app && cd app
    25 
    +    # Initialize a fresh git repo
    26 
    +    - git init
    27 
    +    # Add app.bundle as the remote. All operations that communicate with the remote will be local.
    28 
    +    - git remote add origin ../app.bundle
    29 
    +    # shallow.txt contains the SHA of the base commit of the bundle.
    30 
    +    # The bundle is shallow, therefore it's base commit will not have a parent.
    31 
    +    # Adding the SHA of the base commit to .git/shallow tells git that it doesn't need
    32 
    +    # to crash when it realizes said base commit doesn't have a parent.
    33 
    +    - cp ../shallow.txt .git/shallow
    34 
    +    # Finally, unpack the bundle. Time it for debugging purposes.
    35 
    +    - time git pull origin $BRANCH_NAME
    14 36
    15 37 
     eslint:
    16 38 
       extends: .base
    17 
    -  image: $IMAGE_PATH
    18 39 
       script:
    19 
    -    - .gitlab/ci/scripts/run_linters.py eslint
    40 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l eslint
    20 41 
       rules:
    21 42 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    22 43 
           changes:
    ... ... @@ -44,9 +65,8 @@ eslint:
    44 65
    45 66 
     stylelint:
    46 67 
       extends: .base
    47 
    -  image: $IMAGE_PATH
    48 68 
       script:
    49 
    -    - .gitlab/ci/scripts/run_linters.py stylelint
    69 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l stylelint
    50 70 
       rules:
    51 71 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    52 72 
           changes:
    ... ... @@ -63,9 +83,8 @@ stylelint:
    63 83
    64 84 
     py-black:
    65 85 
       extends: .base
    66 
    -  image: $IMAGE_PATH
    67 86 
       script:
    68 
    -    - .gitlab/ci/scripts/run_linters.py black
    87 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l black
    69 88 
       rules:
    70 89 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    71 90 
           changes:
    ... ... @@ -83,9 +102,8 @@ py-black:
    83 102
    84 103 
     py-ruff:
    85 104 
       extends: .base
    86 
    -  image: $IMAGE_PATH
    87 105 
       script:
    88 
    -    - .gitlab/ci/scripts/run_linters.py ruff
    106 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l ruff
    89 107 
       rules:
    90 108 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    91 109 
           changes:
    ... ... @@ -103,9 +121,8 @@ py-ruff:
    103 121
    104 122 
     yaml:
    105 123 
       extends: .base
    106 
    -  image: $IMAGE_PATH
    107 124 
       script:
    108 
    -    - .gitlab/ci/scripts/run_linters.py yaml
    125 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l yaml
    109 126 
       rules:
    110 127 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    111 128 
           changes:
    ... ... @@ -119,9 +136,8 @@ yaml:
    119 136
    120 137 
     shellcheck:
    121 138 
       extends: .base
    122 
    -  image: $IMAGE_PATH
    123 139 
       script:
    124 
    -    - .gitlab/ci/scripts/run_linters.py shellcheck
    140 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l shellcheck
    125 141 
       rules:
    126 142 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    127 143 
           changes:
    ... ... @@ -134,10 +150,9 @@ shellcheck:
    134 150
    135 151 
     clang-format:
    136 152 
       extends: .base
    137 
    -  image: $IMAGE_PATH
    138 153 
       script:
    139 154 
         - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
    140 
    -    - .gitlab/ci/scripts/run_linters.py clang-format
    155 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l clang-format
    141 156 
       rules:
    142 157 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    143 158 
           changes:
    ... ... @@ -155,9 +170,8 @@ clang-format:
    155 170
    156 171 
     rustfmt:
    157 172 
       extends: .base
    158 
    -  image: $IMAGE_PATH
    159 173 
       script:
    160 
    -    - .gitlab/ci/scripts/run_linters.py rustfmt
    174 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l rustfmt
    161 175 
       rules:
    162 176 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    163 177 
           changes:
    ... ... @@ -170,9 +184,8 @@ rustfmt:
    170 184
    171 185 
     fluent-lint:
    172 186 
       extends: .base
    173 
    -  image: $IMAGE_PATH
    174 187 
       script:
    175 
    -    - .gitlab/ci/scripts/run_linters.py fluent-lint
    188 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l fluent-lint
    176 189 
       rules:
    177 190 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    178 191 
           changes:
    ... ... @@ -186,9 +199,8 @@ fluent-lint:
    186 199
    187 200 
     localization:
    188 201 
       extends: .base
    189 
    -  image: $IMAGE_PATH
    190 202 
       script:
    191 
    -    - .gitlab/ci/scripts/run_linters.py l10n
    203 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l l10n
    192 204 
       rules:
    193 205 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    194 206 
           changes:
    ... ... @@ -204,9 +216,8 @@ localization:
    204 216
    205 217 
     mingw-capitalization:
    206 218 
       extends: .base
    207 
    -  image: $IMAGE_PATH
    208 219 
       script:
    209 
    -    - .gitlab/ci/scripts/run_linters.py mingw-capitalization
    220 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mingw-capitalization
    210 221 
       rules:
    211 222 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    212 223 
           changes:
    ... ... @@ -222,9 +233,8 @@ mingw-capitalization:
    222 233
    223 234 
     mscom-init:
    224 235 
       extends: .base
    225 
    -  image: $IMAGE_PATH
    226 236 
       script:
    227 
    -    - .gitlab/ci/scripts/run_linters.py mscom-init
    237 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l mscom-init
    228 238 
       rules:
    229 239 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    230 240 
           changes:
    ... ... @@ -240,9 +250,8 @@ mscom-init:
    240 250
    241 251 
     file-whitespace:
    242 252 
       extends: .base
    243 
    -  image: $IMAGE_PATH
    244 253 
       script:
    245 
    -    - .gitlab/ci/scripts/run_linters.py file-whitespace
    254 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l file-whitespace
    246 255 
       rules:
    247 256 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    248 257 
           changes:
    ... ... @@ -270,9 +279,8 @@ file-whitespace:
    270 279
    271 280 
     test-manifest:
    272 281 
       extends: .base
    273 
    -  image: $IMAGE_PATH
    274 282 
       script:
    275 
    -    - .gitlab/ci/scripts/run_linters.py test-manifest-alpha test-manifest-disable test-manifest-skip-if
    283 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
    276 284 
       rules:
    277 285 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    278 286 
           changes:
    ... ... @@ -286,9 +294,8 @@ test-manifest:
    286 294
    287 295 
     trojan-source:
    288 296 
       extends: .base
    289 
    -  image: $IMAGE_PATH
    290 297 
       script:
    291 
    -    - .gitlab/ci/scripts/run_linters.py trojan-source
    298 
    +    - cat ../changedfiles.txt | xargs -d '\n' ./mach lint -l trojan-source
    292 299 
       rules:
    293 300 
         - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    294 301 
           changes:

  • .gitlab/ci/scripts/run_linters.py → .gitlab/ci/scripts/helpers.py
    ... ... @@ -5,7 +5,6 @@ import os
    5 5 
     import re
    6 6 
     import shlex
    7 7 
     import subprocess
    8 
    -import sys
    9 8
    10 9
    11 10 
     def git(command):
    ... ... @@ -15,8 +14,8 @@ def git(command):
    15 14 
         return result.stdout.strip()
    16 15
    17 16
    18 
    -def get_firefox_tag_from_branch_name(branch_name):
    19 
    -    """Extracts the Firefox tag associated with a branch name.
    17 
    +def get_firefox_tag(reference):
    18 
    +    """Extracts the Firefox tag associated with a branch or tag name.
    20 19
    21 20 
            The "firefox tag" is the tag that marks
    22 21 
            the end of the Mozilla commits and the start of the Tor Project commits.
    ... ... @@ -25,26 +24,26 @@ def get_firefox_tag_from_branch_name(branch_name):
    25 24 
            this function may return the incorrect reference number.
    26 25
    27 26 
         Args:
    28 
    -        branch_name: The branch name to extract the tag from.
    27 
    +        reference: The branch or tag name to extract the Firefox tag from.
    29 28 
             Expected format is tor-browser-91.2.0esr-11.0-1,
    30 29 
             where 91.2.0esr is the Firefox version.
    31 30
    32 31 
         Returns:
    33 32 
             The reference specifier of the matching Firefox tag.
    34 
    -        An exception wil be raised if anything goes wrong.
    33 
    +        An exception will be raised if anything goes wrong.
    35 34 
         """
    36 35
    37 
    -    # Extracts the version number from a branch name.
    36 
    +    # Extracts the version number from a branch or tag name.
    38 37 
         firefox_version = ""
    39 
    -    match = re.search(r"(?<=browser-)([^-]+)", branch_name)
    38 
    +    match = re.search(r"(?<=browser-)([^-]+)", reference)
    40 39 
         if match:
    41 40 
             # TODO: Validate that what we got is actually a valid semver string?
    42 41 
             firefox_version = match.group(1)
    43 42 
         else:
    44 
    -        raise ValueError(f"Failed to extract version from branch name '{branch_name}'.")
    43 
    +        raise ValueError(f"Failed to extract version from reference '{reference}'.")
    45 44
    46 45 
         tag = f"FIREFOX_{firefox_version.replace('.', '_')}_"
    47 
    -    remote_tags = git("ls-remote --tags")
    46 
    +    remote_tags = git("ls-remote --tags origin")
    48 47
    49 48 
         # Each line looks like:
    50 49 
         # 9edd658bfd03a6b4743ecb75fd4a9ad968603715  refs/tags/FIREFOX_91_9_0esr_BUILD1
    ... ... @@ -54,7 +53,7 @@ def get_firefox_tag_from_branch_name(branch_name):
    54 53 
             return match.group(0).split()[0]
    55 54 
         else:
    56 55 
             raise ValueError(
    57 
    -            f"Failed to find reference specifier for Firefox tag '{tag}' in branch '{branch_name}'."
    56 
    +            f"Failed to find reference specifier for Firefox tag '{tag}' from '{reference}'."
    58 57 
             )
    59 58
    60 59
    ... ... @@ -74,37 +73,42 @@ def get_list_of_changed_files():
    74 73 
         base_reference = ""
    75 74
    76 75 
         if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
    77 
    -        # For merge requests, the base_reference is the common ancestor between the MR and the target branch.
    76 
    +        # For merge requests, the base_reference is the common ancestor between the MR and the target branch
    78 77 
             base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
    79 78 
         else:
    80 79 
             # When not in merge requests, the base reference is the Firefox tag
    81 
    -        base_reference = get_firefox_tag_from_branch_name(os.getenv("CI_COMMIT_BRANCH"))
    80 
    +        base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
    82 81
    83 82 
         if not base_reference:
    84 83 
             raise RuntimeError("No base reference found. There might be more errors above.")
    85 84
    86 85 
         # Fetch the tag reference
    87 86 
         git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
    88 
    -    # Return the list of changed files
    89 
    -    return git(f"diff --diff-filter=d --name-only {base_reference} HEAD").split("\n")
    87 
    +    # Return but filter the issue_templates files because those file names have spaces which can cause issues
    88 
    +    return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
    90 89
    91 90
    92 91 
     if __name__ == "__main__":
    93 
    -    parser = argparse.ArgumentParser(
    94 
    -        description="Run ./mach linters in CI. Warning: if you run this in your local environment it might mess up your git history."
    92 
    +    parser = argparse.ArgumentParser(description="")
    93 
    +
    94 
    +    parser.add_argument(
    95 
    +        "--get-firefox-tag",
    96 
    +        help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
    97 
    +        type=str,
    95 98 
         )
    96 99 
         parser.add_argument(
    97 
    -        "linters", metavar="L", type=str, nargs="+", help="A list of linters to run."
    100 
    +        "--get-changed-files",
    101 
    +        help="Get list of changed files."
    102 
    +        "When running from a merge request get sthe list of changed files since the merge-base of the current branch."
    103 
    +        "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
    104 
    +        action="store_true",
    98 105 
         )
    106 
    +
    99 107 
         args = parser.parse_args()
    100 108
    101 
    -    changed_files = get_list_of_changed_files()
    102 
    -    if changed_files:
    103 
    -        command = ["./mach", "lint", "-v"]
    104 
    -        for linter in args.linters:
    105 
    -            command.extend(["-l", linter])
    106 
    -        command.extend(changed_files)
    107 
    -        result = subprocess.run(command, text=True)
    108 
    -        sys.exit(result.returncode)
    109 
    +    if args.get_firefox_tag:
    110 
    +        print(get_firefox_tag(args.get_firefox_tag))
    111 
    +    elif args.get_changed_files:
    112 
    +        print("\n".join(get_list_of_changed_files()))
    109 113 
         else:
    110 
    -        print("No files changed, skipping linting.")
    114 
    +        print("No valid option provided.")

  • .gitlab/ci/setup.yml
    1 
    +setup-env:
    2 
    +  stage: setup
    3 
    +  interruptible: true
    4 
    +  variables:
    5 
    +    GIT_STRATEGY: "none"
    6 
    +  rules:
    7 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
    8 
    +  script:
    9 
    +    - |
    10 
    +      if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
    11 
    +        echo "BRANCH_NAME=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" > build.env
    12 
    +      else
    13 
    +        echo "BRANCH_NAME=$CI_COMMIT_REF_NAME" > build.env
    14 
    +      fi
    15 
    +  artifacts:
    16 
    +    reports:
    17 
    +      dotenv:
    18 
    +        - build.env
    19 
    +
    20 
    +create-bundle:
    21 
    +  stage: setup
    22 
    +  # TODO: Find a better suited image, this one just has git.
    23 
    +  image: python
    24 
    +  needs:
    25 
    +    - job: setup-env
    26 
    +      artifacts: true
    27 
    +  rules:
    28 
    +    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_REF_PROTECTED == 'true'
    29 
    +  variables:
    30 
    +    GIT_DEPTH: "1"
    31 
    +  interruptible: true
    32 
    +  script:
    33 
    +    # DEBUG: Check repository status.
    34 
    +    - git status
    35 
    +    # Switch to a named branch.
    36 
    +    - git switch -c $BRANCH_NAME
    37 
    +    # Create a git bundle -- this will generate the app.bundle file,
    38 
    +    # which can be used as a git remote for offline fetching.
    39 
    +    - git bundle create app.bundle --all
    40 
    +    # Retain the SHA of the base of this shallow repository.
    41 
    +    - cat .git/shallow > shallow.txt
    42 
    +    # DEBUG: Check sizes.
    43 
    +    - du -sh .git
    44 
    +    - du -sh app.bundle
    45 
    +    # Since this is the only job we have access to the Gitlab remote,
    46 
    +    # let's get a list of changed files to use in the next jobs.
    47 
    +    - .gitlab/ci/scripts/helpers.py --get-changed-files > changedfiles.txt
    48 
    +  artifacts:
    49 
    +    paths:
    50 
    +      - app.bundle
    51 
    +      - shallow.txt
    52 
    +      - changedfiles.txt
    53 
    +    expire_in: 1 hour

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.torproject.org. Manage all notifications · Help