[Git][tpo/applications/tor-browser][tor-browser-128.5.0esr-14.5-1] 4 commits: fixup! Add CI for Base Browser

12 Dec 2024


      Pier Angelo Vendrame pushed to branch tor-browser-128.5.0esr-14.5-1 at The Tor Project / Applications / Tor Browser


Commits:
67cfb4f1 by Beatriz Rizental at 2024-12-11T20:00:05+01:00
fixup! Add CI for Base Browser

Small refactor of file organization.

- - - - -
94bb2ef1 by Beatriz Rizental at 2024-12-11T20:00:06+01:00
fixup! Add CI for Tor Browser

Small refactor of file organization.

- - - - -
19a0c991 by Beatriz Rizental at 2024-12-12T05:00:14+01:00
fixup! Add CI for Tor Browser

Multi-platform startup test for nightly builds.

This commit is desktop only.

- - - - -
7dc11c34 by Beatriz Rizental at 2024-12-12T05:00:15+01:00
fixup! Add CI for Tor Browser

Disable Windows and MacOS startup tests.

These can only be enabled once we have Windows and MacOS
runners available.

Relates to: https://gitlab.torproject.org/tpo/tpa/team/-/issues/41829

- - - - -


11 changed files:

- .gitlab-ci.yml
- .gitlab/ci/scripts/helpers.py → .gitlab/ci/jobs/lint/helpers.py
- .gitlab/ci/lint.yml → .gitlab/ci/jobs/lint/lint.yml
- + .gitlab/ci/jobs/startup-test/startup-test.py
- + .gitlab/ci/jobs/startup-test/startup-test.yml
- .gitlab/ci/update-containers.yml → .gitlab/ci/jobs/update-containers.yml
- .gitlab/ci/update-translations.yml → .gitlab/ci/jobs/update-translations.yml
- + .gitlab/ci/mixins.yml
- testing/mozbase/mozinstall/mozinstall/mozinstall.py
- testing/mozbase/setup_development.py
- testing/mozharness/scripts/does_it_crash.py


Changes:

=====================================
.gitlab-ci.yml
=====================================
@@ -1,6 +1,7 @@
 stages:
   - update-container-images
   - lint
+  - startup-test
   - update-translations
 
 variables:
@@ -8,6 +9,8 @@ variables:
   LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git
 
 include:
-  - local: '.gitlab/ci/lint.yml'
-  - local: '.gitlab/ci/update-containers.yml'
-  - local: '.gitlab/ci/update-translations.yml'
+  - local: '.gitlab/ci/mixins.yml'
+  - local: '.gitlab/ci/jobs/lint/lint.yml'
+  - local: '.gitlab/ci/jobs/startup-test/startup-test.yml'
+  - local: '.gitlab/ci/jobs/update-containers.yml'
+  - local: '.gitlab/ci/jobs/update-translations.yml'


=====================================
.gitlab/ci/scripts/helpers.py → .gitlab/ci/jobs/lint/helpers.py
=====================================


=====================================
.gitlab/ci/lint.yml → .gitlab/ci/jobs/lint/lint.yml
=====================================
@@ -1,11 +1,10 @@
 .base:
+  extends: .with-local-repo-bash
   stage: lint
   image: $IMAGE_PATH
   interruptible: true
   variables:
     MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild"
-    # A copy of the repository already is available in the runner.
-    GIT_STRATEGY: "none"
   cache:
     paths:
       - node_modules
@@ -17,24 +16,11 @@
   tags:
     # Run these jobs in the browser dedicated runners.
     - firefox
-  before_script:
-    - git init
-    - git remote add local "$LOCAL_REPO_PATH"
-    - git fetch --depth 500 local
-    - git remote add origin "$CI_REPOSITORY_URL"
-    - |
-      if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then
-          echo "No branch specified. Stopping the pipeline."
-          exit 1
-      fi
-    - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
-    - git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
-    - git checkout origin/${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
 
 eslint:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -63,7 +49,7 @@ eslint:
 stylelint:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -81,7 +67,7 @@ stylelint:
 py-black:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -100,7 +86,7 @@ py-black:
 py-ruff:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -119,7 +105,7 @@ py-ruff:
 yaml:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -134,7 +120,7 @@ yaml:
 shellcheck:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -149,7 +135,7 @@ clang-format:
   extends: .base
   script:
     - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -168,7 +154,7 @@ clang-format:
 rustfmt:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -182,7 +168,7 @@ rustfmt:
 fluent-lint:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -197,7 +183,7 @@ fluent-lint:
 localization:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -214,7 +200,7 @@ localization:
 mingw-capitalization:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -231,7 +217,7 @@ mingw-capitalization:
 mscom-init:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -248,7 +234,7 @@ mscom-init:
 file-whitespace:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -278,7 +264,7 @@ file-whitespace:
 test-manifest:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
@@ -293,7 +279,7 @@ test-manifest:
 trojan-source:
   extends: .base
   script:
-    - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source
+    - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:


=====================================
.gitlab/ci/jobs/startup-test/startup-test.py
=====================================
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+from datetime import datetime, timedelta
+
+PLATFORM_TO_ARCH = {
+    "linux": ["x86_64", "i686"],
+    "macos": ["x86_64", "aarch64"],
+    "windows": ["x86_64", "i686"],
+}
+
+
+class DynamicArchAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        platform = getattr(namespace, "platform", None)
+        if not platform:
+            raise argparse.ArgumentError(
+                self, "The --platform argument must be provided before --arch."
+            )
+
+        valid_archs = PLATFORM_TO_ARCH.get(platform, [])
+        if values not in valid_archs:
+            raise argparse.ArgumentError(
+                self,
+                f"Invalid architecture '{values}' for platform '{platform}'. "
+                f"Valid options are: {', '.join(valid_archs)}",
+            )
+        setattr(namespace, self.dest, values)
+
+
+parser = argparse.ArgumentParser(
+    description="Downloads and executes yesterday's build of Tor or Mullvad browser nightly."
+)
+
+parser.add_argument(
+    "--platform",
+    required=True,
+    help="Specify the platform (linux, macos or windows). Must be provided before --arch.",
+    choices=PLATFORM_TO_ARCH.keys(),
+)
+parser.add_argument(
+    "--arch",
+    required=True,
+    help="Specify the architecture (validated dynamically based on --platform).",
+    action=DynamicArchAction,
+)
+parser.add_argument(
+    "--browser",
+    required=True,
+    choices=["tor", "mullvad"],
+    help="Specify the browser (tor or mullvad)",
+)
+
+args = parser.parse_args()
+arch = f"-{args.arch}"
+extra = ""
+
+if args.platform == "linux":
+    archive_extension = "tar.xz"
+    binary = f"Browser/start-{args.browser}-browser"
+elif args.platform == "macos":
+    archive_extension = "dmg"
+    # The URL doesn't include the architecture for MacOS,
+    # because it's a universal build.
+    arch = ""
+    if args.browser == "tor":
+        binary = "Contents/MacOS/firefox"
+    else:
+        binary = "Contents/MacOS/mullvadbrowser"
+elif args.platform == "windows":
+    archive_extension = "exe"
+
+    if args.browser == "tor":
+        extra = "-portable"
+        binary = "Browser/firefox.exe"
+    else:
+        binary = "mullvadbrowser.exe"
+
+yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y.%m.%d")
+
+download_url_base = (
+    "https://nightlies.tbb.torproject.org/nightly-builds/tor-browser-builds"
+)
+if args.browser == "tor":
+    download_url = f"{download_url_base}/tbb-nightly.{yesterday}/nightly-{args.platform}{arch}/{args.browser}-browser-{args.platform}{arch}{extra}-tbb-nightly.{yesterday}.{archive_extension}"
+else:
+    download_url = f"{download_url_base}/tbb-nightly.{yesterday}/mullvadbrowser-nightly-{args.platform}{arch}/{args.browser}-browser-{args.platform}{arch}-tbb-nightly.{yesterday}.{archive_extension}"
+
+subprocess.run(
+    [
+        "python3",
+        "testing/mozharness/scripts/does_it_crash.py",
+        "--run-for",
+        "30",
+        "--thing-url",
+        download_url,
+        "--thing-to-run",
+        binary,
+    ]
+)


=====================================
.gitlab/ci/jobs/startup-test/startup-test.yml
=====================================
@@ -0,0 +1,51 @@
+# startup-test-windows:
+#   extends: .with-local-repo-pwsh
+#   variables:
+#     LOCAL_REPO_PATH: "C:\\Users\\windoes\\tor-browser.git"
+#   stage: startup-test
+#   interruptible: true
+#   parallel:
+#     matrix:
+#       - BROWSER: ["tor", "mullvad"]
+#   tags:
+#     - x86-win11
+#   script:
+#     - ./mach python testing/mozbase/setup_development.py
+#     - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform windows --arch x86_64 --browser tor
+#   rules:
+#     - if: $CI_PIPELINE_SOURCE == "schedule"
+
+# startup-test-macos:
+#   extends: .with-local-repo-bash
+#   variables:
+#     LOCAL_REPO_PATH: "/Users/gitlab-runner/tor-browser.git"
+#   stage: startup-test
+#   interruptible: true
+#   parallel:
+#     matrix:
+#       - BROWSER: ["tor", "mullvad"]
+#   tags:
+#     -  x86-macos
+#   script:
+#     - ./mach python testing/mozbase/setup_development.py
+#     - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform macos --arch x86_64 --browser tor
+#   rules:
+#     - if: $CI_PIPELINE_SOURCE == "schedule"
+
+startup-test-linux:
+  extends: .with-local-repo-bash
+  image: $IMAGE_PATH
+  stage: startup-test
+  interruptible: true
+  parallel:
+    matrix:
+      - BROWSER: ["tor", "mullvad"]
+  tags:
+    -  firefox
+  script:
+    - Xvfb :99 -screen 0 1400x900x24 &
+    - export DISPLAY=:99
+    - ./mach python testing/mozbase/setup_development.py
+    - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform linux --arch x86_64 --browser tor
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"


=====================================
.gitlab/ci/update-containers.yml → .gitlab/ci/jobs/update-containers.yml
=====================================


=====================================
.gitlab/ci/update-translations.yml → .gitlab/ci/jobs/update-translations.yml
=====================================


=====================================
.gitlab/ci/mixins.yml
=====================================
@@ -0,0 +1,37 @@
+.with-local-repo-bash:
+  variables:
+      GIT_STRATEGY: "none"
+  before_script:
+    - git init
+    - git remote add local "$LOCAL_REPO_PATH"
+    - git fetch --depth 500 local
+    - git remote add origin "$CI_REPOSITORY_URL"
+    - |
+      if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then
+          echo "No branch specified. Stopping the pipeline."
+          exit 1
+      fi
+    - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
+    - git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
+    - git checkout origin/${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
+
+.with-local-repo-pwsh:
+  variables:
+      GIT_STRATEGY: "none"
+  before_script:
+    - git init
+    - git remote add local $env:LOCAL_REPO_PATH
+    - git fetch --depth 500 local
+    - git remote add origin $env:CI_REPOSITORY_URL
+    - |
+      $branchName = $env:CI_COMMIT_BRANCH
+      if ([string]::IsNullOrEmpty($branchName)) {
+          $branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      }
+      if ([string]::IsNullOrEmpty($branchName)) {
+          Write-Output "No branch specified. Stopping the pipeline."
+          exit 1
+      }
+    - Write-Output "Fetching from remote branch $branchName"
+    - git fetch origin $branchName
+    - git checkout origin/$branchName


=====================================
testing/mozbase/mozinstall/mozinstall/mozinstall.py
=====================================
@@ -352,7 +352,8 @@ def _install_exe(src, dest):
 
     # possibly gets around UAC in vista (still need to run as administrator)
     os.environ["__compat_layer"] = "RunAsInvoker"
-    cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest))
+    cmd = '"%s" /S /D=%s' % (src, os.path.realpath(dest))
+    # cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest))
 
     subprocess.check_call(cmd)
 


=====================================
testing/mozbase/setup_development.py
=====================================
@@ -267,23 +267,28 @@ def main(args=sys.argv[1:]):
         os.environ.get("PATH", "").strip(os.path.pathsep),
     )
 
+    current_file_path = os.path.abspath(__file__)
+    topobjdir = os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))
+    mach = str(os.path.join(topobjdir, "mach"))
+
     # install non-mozbase dependencies
     # these need to be installed separately and the --no-deps flag
     # subsequently used due to a bug in setuptools; see
     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
     pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled])
     for package, version in pypi_deps.items():
-        # easy_install should be available since we rely on setuptools
-        call(["easy_install", version])
+        # Originally, Mozilla used easy_install here.
+        # That tool is deprecated, therefore we swich to pip.
+        call([sys.executable, mach, "python", "-m", "pip", "install", version])
 
     # install packages required for unit testing
     for package in test_packages:
-        call(["easy_install", package])
+        call([sys.executable, mach, "python", "-m", "pip", "install", package])
 
     # install extra non-mozbase packages if desired
     if options.extra:
         for package in extra_packages:
-            call(["easy_install", package])
+            call([sys.executable, mach, "python", "-m", "pip", "install", package])
 
 
 if __name__ == "__main__":


=====================================
testing/mozharness/scripts/does_it_crash.py
=====================================
@@ -112,6 +112,13 @@ class DoesItCrash(BaseScript):
         for retry in range(3):
             if is_win:
                 proc.send_signal(signal.CTRL_BREAK_EVENT)
+
+                # Manually kill all processes we spawned,
+                # not sure why this is required, but without it we hang forever.
+                process_name = self.config["thing_to_run"].split("/")[-1]
+                subprocess.run(
+                    ["taskkill", "/T", "/F", "/IM", process_name], check=True
+                )
             else:
                 os.killpg(proc.pid, signal.SIGKILL)
             try:



View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/653c14c...

-- 
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/653c14c...
You're receiving this email because of your account on gitlab.torproject.org.

    

Pier Angelo Vendrame (＠pierov)

tags

participants (1)