Pier Angelo Vendrame pushed to branch tor-browser-128.5.0esr-14.5-1 at The Tor Project / Applications / Tor Browser

Commits: 67cfb4f1 by Beatriz Rizental at 2024-12-11T20:00:05+01:00 fixup! Add CI for Base Browser

Small refactor of file organization.

- - - - - 94bb2ef1 by Beatriz Rizental at 2024-12-11T20:00:06+01:00 fixup! Add CI for Tor Browser

Small refactor of file organization.

- - - - - 19a0c991 by Beatriz Rizental at 2024-12-12T05:00:14+01:00 fixup! Add CI for Tor Browser

Multi-platform startup test for nightly builds.

This commit is desktop only.

- - - - - 7dc11c34 by Beatriz Rizental at 2024-12-12T05:00:15+01:00 fixup! Add CI for Tor Browser

Disable Windows and MacOS startup tests.

These can only be enabled once we have Windows and MacOS runners available.

Relates to: https://gitlab.torproject.org/tpo/tpa/team/-/issues/41829

- - - - -

11 changed files:

- .gitlab-ci.yml - .gitlab/ci/scripts/helpers.py → .gitlab/ci/jobs/lint/helpers.py - .gitlab/ci/lint.yml → .gitlab/ci/jobs/lint/lint.yml - + .gitlab/ci/jobs/startup-test/startup-test.py - + .gitlab/ci/jobs/startup-test/startup-test.yml - .gitlab/ci/update-containers.yml → .gitlab/ci/jobs/update-containers.yml - .gitlab/ci/update-translations.yml → .gitlab/ci/jobs/update-translations.yml - + .gitlab/ci/mixins.yml - testing/mozbase/mozinstall/mozinstall/mozinstall.py - testing/mozbase/setup_development.py - testing/mozharness/scripts/does_it_crash.py

Changes:

===================================== .gitlab-ci.yml ===================================== @@ -1,6 +1,7 @@ stages: - update-container-images - lint + - startup-test - update-translations

variables: @@ -8,6 +9,8 @@ variables: LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git

include: - - local: '.gitlab/ci/lint.yml' - - local: '.gitlab/ci/update-containers.yml' - - local: '.gitlab/ci/update-translations.yml' + - local: '.gitlab/ci/mixins.yml' + - local: '.gitlab/ci/jobs/lint/lint.yml' + - local: '.gitlab/ci/jobs/startup-test/startup-test.yml' + - local: '.gitlab/ci/jobs/update-containers.yml' + - local: '.gitlab/ci/jobs/update-translations.yml'

===================================== .gitlab/ci/scripts/helpers.py → .gitlab/ci/jobs/lint/helpers.py =====================================

===================================== .gitlab/ci/lint.yml → .gitlab/ci/jobs/lint/lint.yml ===================================== @@ -1,11 +1,10 @@ .base: + extends: .with-local-repo-bash stage: lint image: $IMAGE_PATH interruptible: true variables: MOZBUILD_STATE_PATH: "$CI_PROJECT_DIR/.cache/mozbuild" - # A copy of the repository already is available in the runner. - GIT_STRATEGY: "none" cache: paths: - node_modules @@ -17,24 +16,11 @@ tags: # Run these jobs in the browser dedicated runners. - firefox - before_script: - - git init - - git remote add local "$LOCAL_REPO_PATH" - - git fetch --depth 500 local - - git remote add origin "$CI_REPOSITORY_URL" - - | - if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then - echo "No branch specified. Stopping the pipeline." - exit 1 - fi - - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" - - git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" - - git checkout origin/${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}

eslint: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l eslint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -63,7 +49,7 @@ eslint: stylelint: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l stylelint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -81,7 +67,7 @@ stylelint: py-black: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l black rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -100,7 +86,7 @@ py-black: py-ruff: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l ruff rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -119,7 +105,7 @@ py-ruff: yaml: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l yaml rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -134,7 +120,7 @@ yaml: shellcheck: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l shellcheck rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -149,7 +135,7 @@ clang-format: extends: .base script: - ./mach configure --without-wasm-sandboxed-libraries --with-base-browser-version=0.0.0 - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l clang-format rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -168,7 +154,7 @@ clang-format: rustfmt: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l rustfmt rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -182,7 +168,7 @@ rustfmt: fluent-lint: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l fluent-lint rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -197,7 +183,7 @@ fluent-lint: localization: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l l10n rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -214,7 +200,7 @@ localization: mingw-capitalization: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mingw-capitalization rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -231,7 +217,7 @@ mingw-capitalization: mscom-init: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l mscom-init rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -248,7 +234,7 @@ mscom-init: file-whitespace: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l file-whitespace rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -278,7 +264,7 @@ file-whitespace: test-manifest: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l test-manifest-alpha -l test-manifest-disable -l test-manifest-skip-if rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes: @@ -293,7 +279,7 @@ test-manifest: trojan-source: extends: .base script: - - .gitlab/ci/scripts/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source + - .gitlab/ci/jobs/lint/helpers.py --get-changed-files | xargs -d '\n' ./mach lint -l trojan-source rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' changes:

===================================== .gitlab/ci/jobs/startup-test/startup-test.py ===================================== @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import argparse +import subprocess +from datetime import datetime, timedelta + +PLATFORM_TO_ARCH = { + "linux": ["x86_64", "i686"], + "macos": ["x86_64", "aarch64"], + "windows": ["x86_64", "i686"], +} + + +class DynamicArchAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + platform = getattr(namespace, "platform", None) + if not platform: + raise argparse.ArgumentError( + self, "The --platform argument must be provided before --arch." + ) + + valid_archs = PLATFORM_TO_ARCH.get(platform, []) + if values not in valid_archs: + raise argparse.ArgumentError( + self, + f"Invalid architecture '{values}' for platform '{platform}'. " + f"Valid options are: {', '.join(valid_archs)}", + ) + setattr(namespace, self.dest, values) + + +parser = argparse.ArgumentParser( + description="Downloads and executes yesterday's build of Tor or Mullvad browser nightly." +) + +parser.add_argument( + "--platform", + required=True, + help="Specify the platform (linux, macos or windows). Must be provided before --arch.", + choices=PLATFORM_TO_ARCH.keys(), +) +parser.add_argument( + "--arch", + required=True, + help="Specify the architecture (validated dynamically based on --platform).", + action=DynamicArchAction, +) +parser.add_argument( + "--browser", + required=True, + choices=["tor", "mullvad"], + help="Specify the browser (tor or mullvad)", +) + +args = parser.parse_args() +arch = f"-{args.arch}" +extra = "" + +if args.platform == "linux": + archive_extension = "tar.xz" + binary = f"Browser/start-{args.browser}-browser" +elif args.platform == "macos": + archive_extension = "dmg" + # The URL doesn't include the architecture for MacOS, + # because it's a universal build. + arch = "" + if args.browser == "tor": + binary = "Contents/MacOS/firefox" + else: + binary = "Contents/MacOS/mullvadbrowser" +elif args.platform == "windows": + archive_extension = "exe" + + if args.browser == "tor": + extra = "-portable" + binary = "Browser/firefox.exe" + else: + binary = "mullvadbrowser.exe" + +yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y.%m.%d") + +download_url_base = ( + "https://nightlies.tbb.torproject.org/nightly-builds/tor-browser-builds" +) +if args.browser == "tor": + download_url = f"{download_url_base}/tbb-nightly.{yesterday}/nightly-{args.platform}{arch}/{args.browser}-browser-{args.platform}{arch}{extra}-tbb-nightly.{yesterday}.{archive_extension}" +else: + download_url = f"{download_url_base}/tbb-nightly.{yesterday}/mullvadbrowser-nightly-{args.platform}{arch}/{args.browser}-browser-{args.platform}{arch}-tbb-nightly.{yesterday}.{archive_extension}" + +subprocess.run( + [ + "python3", + "testing/mozharness/scripts/does_it_crash.py", + "--run-for", + "30", + "--thing-url", + download_url, + "--thing-to-run", + binary, + ] +)

===================================== .gitlab/ci/jobs/startup-test/startup-test.yml ===================================== @@ -0,0 +1,51 @@ +# startup-test-windows: +# extends: .with-local-repo-pwsh +# variables: +# LOCAL_REPO_PATH: "C:\Users\windoes\tor-browser.git" +# stage: startup-test +# interruptible: true +# parallel: +# matrix: +# - BROWSER: ["tor", "mullvad"] +# tags: +# - x86-win11 +# script: +# - ./mach python testing/mozbase/setup_development.py +# - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform windows --arch x86_64 --browser tor +# rules: +# - if: $CI_PIPELINE_SOURCE == "schedule" + +# startup-test-macos: +# extends: .with-local-repo-bash +# variables: +# LOCAL_REPO_PATH: "/Users/gitlab-runner/tor-browser.git" +# stage: startup-test +# interruptible: true +# parallel: +# matrix: +# - BROWSER: ["tor", "mullvad"] +# tags: +# - x86-macos +# script: +# - ./mach python testing/mozbase/setup_development.py +# - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform macos --arch x86_64 --browser tor +# rules: +# - if: $CI_PIPELINE_SOURCE == "schedule" + +startup-test-linux: + extends: .with-local-repo-bash + image: $IMAGE_PATH + stage: startup-test + interruptible: true + parallel: + matrix: + - BROWSER: ["tor", "mullvad"] + tags: + - firefox + script: + - Xvfb :99 -screen 0 1400x900x24 & + - export DISPLAY=:99 + - ./mach python testing/mozbase/setup_development.py + - ./mach python .gitlab/ci/jobs/startup-test/startup-test.py --platform linux --arch x86_64 --browser tor + rules: + - if: $CI_PIPELINE_SOURCE == "schedule"

===================================== .gitlab/ci/update-containers.yml → .gitlab/ci/jobs/update-containers.yml =====================================

===================================== .gitlab/ci/update-translations.yml → .gitlab/ci/jobs/update-translations.yml =====================================

===================================== .gitlab/ci/mixins.yml ===================================== @@ -0,0 +1,37 @@ +.with-local-repo-bash: + variables: + GIT_STRATEGY: "none" + before_script: + - git init + - git remote add local "$LOCAL_REPO_PATH" + - git fetch --depth 500 local + - git remote add origin "$CI_REPOSITORY_URL" + - | + if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then + echo "No branch specified. Stopping the pipeline." + exit 1 + fi + - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" + - git fetch origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" + - git checkout origin/${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} + +.with-local-repo-pwsh: + variables: + GIT_STRATEGY: "none" + before_script: + - git init + - git remote add local $env:LOCAL_REPO_PATH + - git fetch --depth 500 local + - git remote add origin $env:CI_REPOSITORY_URL + - | + $branchName = $env:CI_COMMIT_BRANCH + if ([string]::IsNullOrEmpty($branchName)) { + $branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME + } + if ([string]::IsNullOrEmpty($branchName)) { + Write-Output "No branch specified. Stopping the pipeline." + exit 1 + } + - Write-Output "Fetching from remote branch $branchName" + - git fetch origin $branchName + - git checkout origin/$branchName

===================================== testing/mozbase/mozinstall/mozinstall/mozinstall.py ===================================== @@ -352,7 +352,8 @@ def _install_exe(src, dest):

# possibly gets around UAC in vista (still need to run as administrator) os.environ["__compat_layer"] = "RunAsInvoker" - cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest)) + cmd = '"%s" /S /D=%s' % (src, os.path.realpath(dest)) + # cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest))

subprocess.check_call(cmd)

===================================== testing/mozbase/setup_development.py ===================================== @@ -267,23 +267,28 @@ def main(args=sys.argv[1:]): os.environ.get("PATH", "").strip(os.path.pathsep), )

+ current_file_path = os.path.abspath(__file__) + topobjdir = os.path.dirname(os.path.dirname(os.path.dirname(current_file_path))) + mach = str(os.path.join(topobjdir, "mach")) + # install non-mozbase dependencies # these need to be installed separately and the --no-deps flag # subsequently used due to a bug in setuptools; see # https://bugzilla.mozilla.org/show_bug.cgi?id=759836 pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled]) for package, version in pypi_deps.items(): - # easy_install should be available since we rely on setuptools - call(["easy_install", version]) + # Originally, Mozilla used easy_install here. + # That tool is deprecated, therefore we swich to pip. + call([sys.executable, mach, "python", "-m", "pip", "install", version])

# install packages required for unit testing for package in test_packages: - call(["easy_install", package]) + call([sys.executable, mach, "python", "-m", "pip", "install", package])

# install extra non-mozbase packages if desired if options.extra: for package in extra_packages: - call(["easy_install", package]) + call([sys.executable, mach, "python", "-m", "pip", "install", package])

if __name__ == "__main__":

===================================== testing/mozharness/scripts/does_it_crash.py ===================================== @@ -112,6 +112,13 @@ class DoesItCrash(BaseScript): for retry in range(3): if is_win: proc.send_signal(signal.CTRL_BREAK_EVENT) + + # Manually kill all processes we spawned, + # not sure why this is required, but without it we hang forever. + process_name = self.config["thing_to_run"].split("/")[-1] + subprocess.run( + ["taskkill", "/T", "/F", "/IM", process_name], check=True + ) else: os.killpg(proc.pid, signal.SIGKILL) try:

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/653c14c...