GitLab

at 2024-12-11T20:00:05+01:00

fixup! Add CI for Base Browser

Small refactor of file organization.

fixup! Add CI for Tor Browser

Small refactor of file organization.

fixup! Add CI for Tor Browser

Multi-platform startup test for nightly builds.

This commit is desktop only.

fixup! Add CI for Tor Browser

Disable Windows and MacOS startup tests.

These can only be enabled once we have Windows and MacOS
runners available.

Relates to: https://gitlab.torproject.org/tpo/tpa/team/-/issues/41829

 stages:

   - update-container-images

   - lint

+  - startup-test

   - update-translations

 variables:

   LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git

 include:

-  - local: '.gitlab/ci/lint.yml'

-  - local: '.gitlab/ci/update-containers.yml'

-  - local: '.gitlab/ci/update-translations.yml'
+  - local: '.gitlab/ci/mixins.yml'

+  - local: '.gitlab/ci/jobs/lint/lint.yml'

+  - local: '.gitlab/ci/jobs/startup-test/startup-test.yml'

+  - local: '.gitlab/ci/jobs/update-containers.yml'

+  - local: '.gitlab/ci/jobs/update-translations.yml'
 .base:

+  extends: .with-local-repo-bash

   stage: lint

   image: $IMAGE_PATH

   interruptible: true

   variables:

     MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"

-    # A copy of the repository already is available in the runner.

-    GIT_STRATEGY: "none"

   cache:

     paths:

       - node_modules

   tags:

     # Run these jobs in the browser dedicated runners.

     - firefox

-  before_script:

-    - git init

-    - git remote add local "$LOCAL_REPO_PATH"

-    - git fetch --depth 500 local

-    - git remote add origin "$CI_REPOSITORY_URL"

-    - |

-      if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then

-          echo "No branch specified. Stopping the pipeline."

-          exit 1

-      fi

-    - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

-    - git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

-    - git checkout origin/${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}

 eslint:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 stylelint:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 py-black:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 py-ruff:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 yaml:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 shellcheck:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

   extends: .base

   script:

     - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 rustfmt:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 fluent-lint:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 localization:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 mingw-capitalization:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 mscom-init:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 file-whitespace:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 test-manifest:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

 trojan-source:

   extends: .base

   script:

-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source

+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source

   rules:

     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

       changes:

+#!/usr/bin/env python3

+

+import argparse

+import subprocess

+from datetime import datetime, timedelta

+

+PLATFORM_TO_ARCH = {

+    "linux": ["x86_64", "i686"],

+    "macos": ["x86_64", "aarch64"],

+    "windows": ["x86_64", "i686"],

+}

+

+

+class DynamicArchAction(argparse.Action):

+    def __call__(self, parser, namespace, values, option_string=None):

+        platform = getattr(namespace, "platform", None)

+        if not platform:

+            raise argparse.ArgumentError(

+                self, "The --platform argument must be provided before --arch."

+            )

+

+        valid_archs = PLATFORM_TO_ARCH.get(platform, [])

+        if values not in valid_archs:

+            raise argparse.ArgumentError(

+                self,

+                f"Invalid architecture '{values}' for platform '{platform}'. "

+                f"Valid options are: {', '.join(valid_archs)}",

+            )

+        setattr(namespace, self.dest, values)

+

+

+parser = argparse.ArgumentParser(

+    description="Downloads and executes yesterday's build of Tor or Mullvad browser nightly."

+)

+

+parser.add_argument(

+    "--platform",

+    required=True,

+    help="Specify the platform (linux, macos or windows). Must be provided before --arch.",

+    choices=PLATFORM_TO_ARCH.keys(),

+)

+parser.add_argument(

+    "--arch",

+    required=True,

+    help="Specify the architecture (validated dynamically based on --platform).",

+    action=DynamicArchAction,

+)

+parser.add_argument(

+    "--browser",

+    required=True,

+    choices=["tor", "mullvad"],

+    help="Specify the browser (tor or mullvad)",

+)

+

+args = parser.parse_args()

+arch = f"-{args.arch}"

+extra = ""

+

+if args.platform == "linux":

+    archive_extension = "tar.xz"

+    binary = f"Browser/start-{args.browser}-browser"

+elif args.platform == "macos":

+    archive_extension = "dmg"

+    # The URL doesn't include the architecture for MacOS,

+    # because it's a universal build.

+    arch = ""

+    if args.browser == "tor":

+        binary = "Contents/MacOS/firefox"

+    else:

+        binary = "Contents/MacOS/mullvadbrowser"

+elif args.platform == "windows":

+    archive_extension = "exe"

+

+    if args.browser == "tor":

+        extra = "-portable"

+        binary = "Browser/firefox.exe"

+    else:

+        binary = "mullvadbrowser.exe"

+

+yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y.%m.%d")

+

+download_url_base = (

+    "https://nightlies.tbb.torproject.org/nightly-builds/tor-browser-builds"

+)

+if args.browser == "tor":

+    download_url = f"{download_url_base}/tbb-nightly.{yesterday}/nightly-{args.platform}{arch}/{args.browser}-browser-{args.platform}{arch}{extra}-tbb-nightly.{yesterday}.{archive_extension}"

+else:

+    download_url = f"{download_url_base}/tbb-nightly.{yesterday}/mullvadbrowser-nightly-{args.platform}{arch}/{args.browser}-browser-{args.platform}{arch}-tbb-nightly.{yesterday}.{archive_extension}"

+

+subprocess.run(

+    [

+        "python3",

+        "testing/mozharness/scripts/does_it_crash.py",

+        "--run-for",

+        "30",

+        "--thing-url",

+        download_url,

+        "--thing-to-run",

+        binary,

+    ]

+)
+# startup-test-windows:

+#   extends: .with-local-repo-pwsh

+#   variables:

+#     LOCAL_REPO_PATH: "C:\\Users\\windoes\\tor-browser.git"

+#   stage: startup-test

+#   interruptible: true

+#   parallel:

+#     matrix:

+#       - BROWSER: ["tor", "mullvad"]

+#   tags:

+#     - x86-win11

+#   script:

+#     - ./mach python testing/mozbase/setup_development.py

+#     - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform windows --arch x86_64 --browser tor

+#   rules:

+#     - if: $CI_PIPELINE_SOURCE == "schedule"

+

+# startup-test-macos:

+#   extends: .with-local-repo-bash

+#   variables:

+#     LOCAL_REPO_PATH: "/Users/gitlab-runner/tor-browser.git"

+#   stage: startup-test

+#   interruptible: true

+#   parallel:

+#     matrix:

+#       - BROWSER: ["tor", "mullvad"]

+#   tags:

+#     -  x86-macos

+#   script:

+#     - ./mach python testing/mozbase/setup_development.py

+#     - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform macos --arch x86_64 --browser tor

+#   rules:

+#     - if: $CI_PIPELINE_SOURCE == "schedule"

+

+startup-test-linux:

+  extends: .with-local-repo-bash

+  image: $IMAGE_PATH

+  stage: startup-test

+  interruptible: true

+  parallel:

+    matrix:

+      - BROWSER: ["tor", "mullvad"]

+  tags:

+    -  firefox

+  script:

+    - Xvfb :99 -screen 0 1400x900x24 &

+    - export DISPLAY=:99

+    - ./mach python testing/mozbase/setup_development.py

+    - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform linux --arch x86_64 --browser tor

+  rules:

+    - if: $CI_PIPELINE_SOURCE == "schedule"
+.with-local-repo-bash:

+  variables:

+      GIT_STRATEGY: "none"

+  before_script:

+    - git init

+    - git remote add local "$LOCAL_REPO_PATH"

+    - git fetch --depth 500 local

+    - git remote add origin "$CI_REPOSITORY_URL"

+    - |

+      if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then

+          echo "No branch specified. Stopping the pipeline."

+          exit 1

+      fi

+    - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

+    - git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

+    - git checkout origin/${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}

+

+.with-local-repo-pwsh:

+  variables:

+      GIT_STRATEGY: "none"

+  before_script:

+    - git init

+    - git remote add local $env:LOCAL_REPO_PATH

+    - git fetch --depth 500 local

+    - git remote add origin $env:CI_REPOSITORY_URL

+    - |

+      $branchName = $env:CI_COMMIT_BRANCH

+      if ([string]::IsNullOrEmpty($branchName)) {

+          $branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME

+      }

+      if ([string]::IsNullOrEmpty($branchName)) {

+          Write-Output "No branch specified. Stopping the pipeline."

+          exit 1

+      }

+    - Write-Output "Fetching from remote branch $branchName"

+    - git fetch origin $branchName

+    - git checkout origin/$branchName

     # possibly gets around UAC in vista (still need to run as administrator)

     os.environ["__compat_layer"] = "RunAsInvoker"

-    cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest))

+    cmd = '"%s" /S /D=%s' % (src, os.path.realpath(dest))

+    # cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest))

     subprocess.check_call(cmd)

         os.environ.get("PATH", "").strip(os.path.pathsep),

     )

+    current_file_path = os.path.abspath(__file__)

+    topobjdir = os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))

+    mach = str(os.path.join(topobjdir, "mach"))

+

     # install non-mozbase dependencies

     # these need to be installed separately and the --no-deps flag

     # subsequently used due to a bug in setuptools; see

     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836

     pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled])

     for package, version in pypi_deps.items():

-        # easy_install should be available since we rely on setuptools

-        call(["easy_install", version])

+        # Originally, Mozilla used easy_install here.

+        # That tool is deprecated, therefore we swich to pip.

+        call([sys.executable, mach, "python", "-m", "pip", "install", version])

     # install packages required for unit testing

     for package in test_packages:

-        call(["easy_install", package])

+        call([sys.executable, mach, "python", "-m", "pip", "install", package])

     # install extra non-mozbase packages if desired

     if options.extra:

         for package in extra_packages:

-            call(["easy_install", package])

+            call([sys.executable, mach, "python", "-m", "pip", "install", package])

 if __name__ == "__main__":

         for retry in range(3):

             if is_win:

                 proc.send_signal(signal.CTRL_BREAK_EVENT)

+

+                # Manually kill all processes we spawned,

+                # not sure why this is required, but without it we hang forever.

+                process_name = self.config["thing_to_run"].split("/")[-1]

+                subprocess.run(

+                    ["taskkill", "/T", "/F", "/IM", process_name], check=True

+                )

             else:

                 os.killpg(proc.pid, signal.SIGKILL)

             try:

Pier Angelo Vendrame pushed to branch tor-browser-128.5.0esr-14.5-1 at The Tor Project / Applications / Tor Browser

Commits:

11 changed files:

Changes: