brizental pushed to branch tor-browser-140.2.0esr-15.0-1 at The Tor Project / Applications / Tor Browser Commits: 5f6737f0 by Beatriz Rizental at 2025-08-21T15:13:13+02:00 BB 43564: Modify ./mach bootstrap for Base Browser - - - - - 2b7a0550 by Beatriz Rizental at 2025-08-21T15:13:16+02:00 fixup! BB 43564: Modify ./mach bootstrap for Base Browser EXTRA: Stop asking to configure git during bootstrap. - - - - - c1827fba by Beatriz Rizental at 2025-08-21T16:01:36+02:00 TB 43564: Modify ./mach bootstrap for Tor Browser - - - - - 10 changed files: - + build/moz.configure/basebrowser-resources.configure - build/moz.configure/bootstrap.configure - build/moz.configure/init.configure - + build/moz.configure/torbrowser-resources.configure - moz.configure - python/mozboot/mozboot/bootstrap.py - python/mozbuild/mozbuild/action/tooltool.py - python/mozbuild/mozbuild/artifact_commands.py - python/mozbuild/mozbuild/backend/base.py - + python/mozbuild/mozbuild/tbbutils.py Changes: ===================================== build/moz.configure/basebrowser-resources.configure ===================================== @@ -0,0 +1,88 @@ +# Helpers +# ------------------------------------------------- + + +@depends(build_project) +def is_desktop_build(build_project): + return build_project == "browser" + + +# Bootstrap resources +# ------------------------------------------------- + + +option( + "--with-noscript", + env="NOSCRIPT", + nargs=1, + default=None, + help="Path to noscript .xpi extension archive.", +) + + +@depends( + "--with-noscript", + mozbuild_state_path, + bootstrap_path( + "noscript", no_unpack=True, when=depends("--with-noscript")(lambda x: not x) + ), +) +@checking("for noscript") +@imports(_from="pathlib", _import="Path") +def noscript(value, mozbuild_state_path, _bootstrapped): + if value: + path = Path(value[0]) + if path.is_file() and path.suffix == ".xpi": + return value[0] + else: + die("--with-noscript must be an existing .xpi file") + + bootstrapped_location = Path(mozbuild_state_path) / "browser" + for file in bootstrapped_location.glob(f"*.xpi"): + if "noscript" in file.name: + return str(bootstrapped_location / file) + + # noscript is not required for building. + return None + + +set_config("NOSCRIPT", noscript) + + +option( + "--with-tor-browser-fonts", + env="TOR_BROWSER_FONTS", + nargs=1, + default=None, + help="Path to location of fonts directory.", +) + + +@depends( + "--with-tor-browser-fonts", + mozbuild_state_path, + bootstrap_path( + "fonts", + when=depends("--with-tor-browser-fonts")(lambda x: not x) & is_desktop_build, + ), +) +@checking("for tor-browser fonts directory") +@imports(_from="pathlib", _import="Path") +def tor_browser_fonts(value, mozbuild_state_path, _bootstrapped): + if value: + path = Path(value[0]) + # TODO: Do a more thorough check on the directory. + if path.is_dir(): + return value[0] + else: + die("--with-tor-browser-fonts must point to a real directory.") + + bootstrapped_location = Path(mozbuild_state_path) / "fonts" + if bootstrapped_location.is_dir(): + return str(bootstrapped_location) + + # tor browser fonts directory is not required for building. + return None + + +set_config("TOR_BROWSER_FONTS", tor_browser_fonts) ===================================== build/moz.configure/bootstrap.configure ===================================== @@ -4,6 +4,29 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +option( + "--with-tor-browser-build-out", + env="TOR_BROWSER_BUILD_OUT", + nargs=1, + default="https://tb-build-06.torproject.org/~tb-builder/tor-browser-build/out", + help="URL pointing to a Tor Browser Build out folder, served over HTTP[S].", +) + + +@depends("--with-tor-browser-build-out") +def tor_browser_build_out(value): + if value: + return value[0] + + +option( + "--enable-tor-browser-build-only-bootstrap", + env="TBB_ONLY_BOOTSTRAP", + default=False, + help="Flag that disables bootstrapping any artifact from Mozilla's Taskcluster. Will only bootstrap artifacts from tor-browser-build.", +) + + option( env="MOZ_FETCHES_DIR", nargs=1, @@ -115,9 +138,10 @@ def bootstrap_toolchain_tasks(host): def bootstrap_path(path, **kwargs): when = kwargs.pop("when", None) allow_failure = kwargs.pop("allow_failure", None) + no_unpack = kwargs.pop("no_unpack", False) if kwargs: configure_error( - "bootstrap_path only takes `when` and `allow_failure` as a keyword argument" + "bootstrap_path only takes `when`, `allow_failure` and `no_unpack` as keyword arguments" ) @depends( @@ -129,11 +153,16 @@ def bootstrap_path(path, **kwargs): build_environment, dependable(path), dependable(allow_failure), + dependable(no_unpack), + tor_browser_build_out, + "--enable-tor-browser-build-only-bootstrap", + target, when=when, ) @imports("os") @imports("subprocess") @imports("sys") + @imports("mozbuild.tbbutils") @imports(_from="mozbuild.dirutils", _import="ensureParentDir") @imports(_from="importlib", _import="import_module") @imports(_from="shutil", _import="rmtree") @@ -148,6 +177,10 @@ def bootstrap_path(path, **kwargs): build_env, path, allow_failure, + no_unpack, + tor_browser_build_out, + tbb_only_bootstrap, + target, ): if not path: return @@ -158,6 +191,81 @@ def bootstrap_path(path, **kwargs): if path_parts[0] == "clang-tools": path_prefix = path_parts.pop(0) + # Small hack because noscript is inside the browser folder. + if path_parts[0] == "noscript": + path_prefix = "browser" + + def try_tbb_bootstrap(exists): + if not tor_browser_build_out: + return False + + # Tor browser build doesn't have artifacts for all targets supported + # by the Firefox build system. When this is empty it means we are + # building for a platform which tbb doesn't support. + if not target.tor_browser_build_alias: + return False + + artifact = mozbuild.tbbutils.get_artifact_name( + path_parts[0], target, tasks.prefix + ) + if not artifact: + log.info("%s is not mapped to a tbb artifact", path_parts[0]) + return False + + artifact_path = mozbuild.tbbutils.get_artifact_path( + tor_browser_build_out, artifact, target, prefix=path_prefix + ) + if not artifact_path: + log.info("no path found in tbb/out for %s", artifact) + return False + + # We will use the name of the artifact as the index. + # + # It's usually unique to the artifact version, but each artifact follows + # a different naming convention, so we can't really get more specific here. + artifact_index = artifact_path.rsplit("/", 1)[-1] + index_file = os.path.join(toolchains_base_dir, "indices", artifact) + try: + with open(index_file) as fh: + index = fh.read().strip() + except Exception: + index = None + if index == artifact_index and exists: + log.debug("%s is up-to-date", artifact) + return True + + command = ["artifact", "toolchain", "--from-url", artifact_path] + + if no_unpack: + command.append("--no-unpack") + + # Note to rebasers: + # From here on, it's a slightly modified copy/paste + # from the end of the try_bootstrap function + log.info( + "%s bootstrapped toolchain from TBB in %s", + "Updating" if exists else "Installing", + os.path.join(toolchains_base_dir, path_prefix, artifact), + ) + os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True) + proc = subprocess.run( + [ + sys.executable, + os.path.join(build_env.topsrcdir, "mach"), + "--log-no-times", + ] + + command, + cwd=os.path.join(toolchains_base_dir, path_prefix), + check=not allow_failure, + ) + if proc.returncode != 0 and allow_failure: + return False + ensureParentDir(index_file) + with open(index_file, "w") as fh: + fh.write(artifact_index) + + return True + def try_bootstrap(exists): if not tasks: return False @@ -280,9 +388,10 @@ def bootstrap_path(path, **kwargs): try: # With --enable-bootstrap=no-update, we don't `try_bootstrap`, except # when the toolchain can't be found. - if ( - "no-update" not in enable_bootstrap or not exists - ) and not try_bootstrap(exists): + if ("no-update" not in enable_bootstrap or not exists) and not ( + try_tbb_bootstrap(exists) + or (not tbb_only_bootstrap and try_bootstrap(exists)) + ): # If there aren't toolchain artifacts to use for this build, # don't return a path. return None ===================================== build/moz.configure/init.configure ===================================== @@ -590,6 +590,21 @@ def split_triplet(triplet, allow_wasi=False): else: toolchain = "%s-%s" % (cpu, os) + # In tor-browser-build we use slightly different terminology for + # the supported platforms. Let's prepare that OS string here. + # + # Not all possible platforms listed here are supported in tbb, + # so this value will be empty sometimes. + tor_browser_build_alias = None + if canonical_os == "Android" and canonical_kernel == "Linux": + tor_browser_build_alias = f"android" + elif canonical_os == "GNU" and canonical_kernel == "Linux": + tor_browser_build_alias = f"linux" + elif canonical_os == "OSX" and canonical_kernel == "Darwin": + tor_browser_build_alias = f"macos" + elif canonical_os == "WINNT" and canonical_kernel == "WINNT": + tor_browser_build_alias = f"windows" + return namespace( alias=triplet, cpu=CPU(canonical_cpu), @@ -604,6 +619,7 @@ def split_triplet(triplet, allow_wasi=False): toolchain=toolchain, vendor=vendor, sub_configure_alias=sub_configure_alias, + tor_browser_build_alias=tor_browser_build_alias, ) ===================================== build/moz.configure/torbrowser-resources.configure ===================================== @@ -0,0 +1,37 @@ +option( + "--with-tor-expert-bundle", + env="TOR_EXPERT_BUNDLE", + nargs=1, + default=None, + help="Path to location of tor-expert-bundle directory.", +) + + +@depends( + "--with-tor-expert-bundle", + mozbuild_state_path, + bootstrap_path( + "tor-expert-bundle", + when=depends("--with-tor-expert-bundle")(lambda x: not x) & is_desktop_build, + ), +) +@checking("for tor-expert-bundle") +@imports(_from="pathlib", _import="Path") +def tor_expert_bundle(value, mozbuild_state_path, _bootstrapped): + if value: + path = Path(value[0]) + # TODO: Do a more thorough check on the directory. + if path.is_dir(): + return value[0] + else: + die("--with-tor-expert-bundle must point to a real directory.") + + bootstrapped_location = Path(mozbuild_state_path) / "tor-expert-bundle" + if bootstrapped_location.is_dir(): + return str(bootstrapped_location) + + # tor-expert-bundle is not required for building. + return None + + +set_config("TOR_EXPERT_BUNDLE", tor_expert_bundle) ===================================== moz.configure ===================================== @@ -229,6 +229,8 @@ check_prog("WGET", ("wget",), allow_missing=True) include("build/moz.configure/toolchain.configure", when="--enable-compile-environment") +include("build/moz.configure/basebrowser-resources.configure") +include("build/moz.configure/torbrowser-resources.configure") include("build/moz.configure/pkg.configure") include("build/moz.configure/memory.configure", when="--enable-compile-environment") ===================================== python/mozboot/mozboot/bootstrap.py ===================================== @@ -52,21 +52,28 @@ Note on Artifact Mode: Artifact builds download prebuilt C++ components rather than building them locally. Artifact builds are faster! -Artifact builds are recommended for people working on Firefox or -Firefox for Android frontends, or the GeckoView Java API. They are unsuitable +Artifact builds are recommended for people working on Tor Browser or +Tor Browser for Android frontends, or the GeckoView Java API. They are unsuitable for those working on C++ code. For more information see: https://firefox-source-docs.mozilla.org/contributing/build/artifact_builds.h.... -Please choose the version of Firefox you want to build (see note above): +# Note to Tor Browser developers + +This is still highly experimental. Expect bugs! + +Please choose the version of Tor Browser you want to build (see note above): %s Your choice: """ APPLICATIONS = OrderedDict( [ - ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"), - ("Firefox for Desktop", "browser"), - ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"), - ("GeckoView/Firefox for Android", "mobile_android"), + ("Tor Browser for Desktop Artifact Mode", "browser_artifact_mode"), + ("Tor Browser for Desktop", "browser"), + ( + "GeckoView/Tor Browser for Android Artifact Mode", + "mobile_android_artifact_mode", + ), + ("GeckoView/Tor Browser for Android", "mobile_android"), ("SpiderMonkey JavaScript engine", "js"), ] ) @@ -360,6 +367,8 @@ class Bootstrapper: getattr(self.instance, "ensure_%s_packages" % application)() def check_code_submission(self, checkout_root: Path): + return + if self.instance.no_interactive or which("moz-phab"): return @@ -474,8 +483,7 @@ class Bootstrapper: configure_mercurial(hg, state_dir) # Offer to configure Git, if the current checkout or repo type is Git. - elif git and checkout_type == "git": - should_configure_git = False + elif False and git and checkout_type == "git": if not self.instance.no_interactive: should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT) else: ===================================== python/mozbuild/mozbuild/action/tooltool.py ===================================== @@ -1029,14 +1029,29 @@ def unpack_file(filename): """Untar `filename`, assuming it is uncompressed or compressed with bzip2, xz, gzip, zst, or unzip a zip file. The file is assumed to contain a single directory with a name matching the base of the given filename. - Xz support is handled by shelling out to 'tar'.""" + Xz support is handled by shelling out to 'tar'. + + tor-browser#41564 - For supporting tor-browser-build artifacts that contain + multiple directories, the archive is extracted into a directory with the + same name as the base of the filename. This modification is only applied to + tar archives, because that is all that was necessary. + """ if os.path.isfile(filename) and tarfile.is_tarfile(filename): tar_file, zip_ext = os.path.splitext(filename) base_file, tar_ext = os.path.splitext(tar_file) clean_path(base_file) log.info('untarring "%s"' % filename) with TarFile.open(filename) as tar: - safe_extract(tar) + top_level_directories = set() + for name in tar.getnames(): + dir = name.split("/", 1)[0] + top_level_directories.add(dir) + if len(top_level_directories) == 1: + safe_extract(tar) + else: + safe_extract( + tar, path=os.path.join(os.path.dirname(filename), base_file) + ) elif os.path.isfile(filename) and filename.endswith(".tar.zst"): import zstandard ===================================== python/mozbuild/mozbuild/artifact_commands.py ===================================== @@ -244,6 +244,12 @@ def artifact_clear_cache(command_context, tree=None, job=None, verbose=False): nargs="+", help="Download toolchain artifact from a given task.", ) +@CommandArgument( + "--from-url", + metavar="URL", + nargs="+", + help="Download toolchain artifact from an arbitrary address.", +) @CommandArgument( "--tooltool-manifest", metavar="MANIFEST", @@ -273,6 +279,7 @@ def artifact_toolchain( skip_cache=False, from_build=(), from_task=(), + from_url=[], tooltool_manifest=None, no_unpack=False, retry=0, @@ -504,6 +511,13 @@ def artifact_toolchain( record = ArtifactRecord(task_id, name) records[record.filename] = record + if from_url: + for file in from_url: + record = DownloadRecord( + file, file.rsplit("/", 1)[-1], None, None, None, True + ) + records[record.filename] = record + for record in records.values(): command_context.log( logging.INFO, ===================================== python/mozbuild/mozbuild/backend/base.py ===================================== @@ -2,11 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import errno import itertools +import logging import os import time from abc import ABCMeta, abstractmethod from contextlib import contextmanager +from pathlib import Path import mozpack.path as mozpath from mach.mixin.logging import LoggingMixin @@ -239,6 +242,129 @@ class BuildBackend(LoggingMixin): with open(mozpath.join(dir, ".purgecaches"), "w") as f: f.write("\n") + def _setup_tor_browser_environment(self, config): + app = config.substs["MOZ_BUILD_APP"] + + noscript_target_filename = "{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi" + noscript_location = Path(config.substs["NOSCRIPT"]) + + def _infallible_symlink(src, dst): + try: + os.symlink(src, dst) + except OSError as e: + if e.errno == errno.EEXIST: + # If the symlink already exists, remove it and try again. + os.remove(dst) + os.symlink(src, dst) + else: + return + + if app == "browser": + tbdir = Path(config.topobjdir) / "dist" / "bin" + + if config.substs.get("OS_TARGET") == "Darwin": + tbdir = next(tbdir.glob("*.app")) + paths = { + "docs": tbdir / "Contents/Resources/TorBrowser/Docs", + "exts": tbdir / "Contents/Resources/distribution/extensions", + "tor_bin": tbdir / "Contents/MacOS/tor", + "tor_config": tbdir / "Contents/Resources/TorBrowser/Tor", + "fonts": tbdir / "Resources/fonts", + } + else: + paths = { + "docs": tbdir / "TorBrowser/Docs", + "exts": tbdir / "distribution/extensions", + "tor_bin": tbdir / "TorBrowser/Tor", + "tor_config": tbdir / "TorBrowser/Data/Tor", + "fonts": tbdir / "fonts", + } + + fonts_location = Path(config.substs["TOR_BROWSER_FONTS"]) + if fonts_location.is_dir(): + self.log( + logging.INFO, + "_setup_tor_browser_environment", + { + "fonts_location": str(fonts_location), + "fonts_target": str(paths["fonts"]), + }, + "Creating symlink for fonts files from {fonts_location} to {fonts_target}", + ) + + for file in fonts_location.iterdir(): + target = paths["fonts"] / file.name + _infallible_symlink(file, target) + + # Set up NoScript extension + if noscript_location.is_file(): + noscript_target = paths["exts"] / noscript_target_filename + self.log( + logging.INFO, + "_setup_tor_browser_environment", + { + "noscript_location": str(noscript_location), + "noscript_target": str(noscript_target), + }, + "Creating symlink for NoScript from {noscript_location} to {noscript_target}", + ) + + paths["exts"].mkdir(parents=True, exist_ok=True) + _infallible_symlink(noscript_location, noscript_target) + + expert_bundle_location = Path(config.substs["TOR_EXPERT_BUNDLE"]) + if expert_bundle_location.is_dir(): + self.log( + logging.INFO, + "_setup_tor_browser_environment", + { + "expert_bundle_location": str(expert_bundle_location), + }, + "Setting up tor-expert-bundle resources from {expert_bundle_location}", + ) + + # Set up Tor configuration files + paths["tor_config"].mkdir(parents=True, exist_ok=True) + for file in ["geoip", "geoip6"]: + target = paths["tor_config"] / file + _infallible_symlink(expert_bundle_location / "data" / file, target) + + # Set up Conjure documentation + conjust_docs_location = paths["docs"] / "conjure" + conjust_docs_location.mkdir(parents=True, exist_ok=True) + conjure_readme = conjust_docs_location / "README.CONJURE.md" + _infallible_symlink( + expert_bundle_location + / "tor/pluggable_transports/README.CONJURE.md", + conjure_readme, + ) + + # Set up pluggable transports + paths["tor_bin"].mkdir(parents=True, exist_ok=True) + pluggable_transports_location = ( + expert_bundle_location / "tor/pluggable_transports" + ) + pluggable_transports_target = paths["tor_bin"] / "PluggableTransports" + pluggable_transports_target.mkdir(parents=True, exist_ok=True) + for file in pluggable_transports_location.iterdir(): + # We only want the PT executables. + if os.access(file, os.X_OK) or file.suffix.lower() == ".exe": + target = pluggable_transports_target / file.name + _infallible_symlink(file, target) + + # Setup Tor binary + for item in (expert_bundle_location / "tor").iterdir(): + target = paths["tor_bin"] / item.name + if target.is_file(): + _infallible_symlink(item, target) + + # Set up licenses + licenses_location = paths["docs"] / "Licenses" + licenses_location.mkdir(parents=True, exist_ok=True) + for item in (expert_bundle_location / "docs").iterdir(): + target = licenses_location / item.name + _infallible_symlink(item, target) + def post_build(self, config, output, jobs, verbose, status): """Called late during 'mach build' execution, after `build(...)` has finished. @@ -257,6 +383,9 @@ class BuildBackend(LoggingMixin): """ self._write_purgecaches(config) + if status == 0: + self._setup_tor_browser_environment(config) + return status @contextmanager ===================================== python/mozbuild/mozbuild/tbbutils.py ===================================== @@ -0,0 +1,97 @@ +import re +from urllib.request import Request, urlopen + + +def list_files_http(url): + try: + req = Request(url, method="GET") + with urlopen(req) as response: + if response.status != 200: + return [] + html = response.read().decode() + except Exception: + return [] + + links = [] + for href in re.findall(r'<a href="([^"]+)"', html): + if href == "../": + continue + + if "tor-expert-bundle" in href: + href = f"{href}/tor-expert-bundle.tar.gz" + + links.append(href) + + return links + + +TOR_BROWSER_BUILD_ARTIFACTS = [ + # Tor Browser Build-only artifacts, these artifacts are not common with Firefox. + "noscript", + "fonts", + "tor-expert-bundle", +] + +# Mapping of artifacts from taskcluster to tor-browser-build. +ARTIFACT_NAME_MAP = { + "cbindgen": "cbindgen", + # FIXME (tor-browser-build#41471): nasm is more or less ready to go, but it needs to have the + # executable in the root of the artifact folder instead of nasm/bin. + # "nasm": "nasm", + # FIXME (tor-browser-build#41421): the clang project as is, is not ready to use. It needs + # to be repackaged with a bunch of things that differ per platform. Fun stuff. + # "clang": "clang", + "node": "node", +} + + +def get_artifact_name(original_artifact_name, target, host): + # These are not build artifacts, they are pre-built artifacts to be added to the final build, + # therefore this check can come before the host check. + if original_artifact_name in TOR_BROWSER_BUILD_ARTIFACTS: + return original_artifact_name + + if host != "linux64": + # Tor browser build only has development artifacts for linux64 host systems. + return None + + return ARTIFACT_NAME_MAP.get(original_artifact_name) + + +def get_artifact_path(url, artifact, target, prefix=""): + if prefix: + path = prefix + else: + path = artifact + + # The `?C=M;O=D` parameters make it so links are ordered by + # the last modified date. This here to make us get the latest + # version of file in the case there are multiple and we just + # grab the first one. + files = list_files_http(f"{url}/{path}?C=M;O=D") + + if not files: + return None + + def filter_files(files, keyword): + return [file for file in files if keyword in file] + + artifact_files = filter_files(files, artifact) + + if len(artifact_files) == 1: + return f"{url}/{path}/{artifact_files[0]}" + + files_per_os = filter_files(artifact_files, target.tor_browser_build_alias) + + # If there are files in the folder, but they don't have the OS in the name + # it means we can get any of them because they can be used to build for any OS. + # So let's just get the first one. + if len(files_per_os) == 0: + return f"{url}/{artifact}/{artifact_files[0]}" + + elif len(files_per_os) == 1: + return f"{url}/{artifact}/{files_per_os[0]}" + + matches = filter_files(files_per_os, target.cpu) + + return f"{url}/{artifact}/{matches[0]}" if matches else None View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/1b62300... -- View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/1b62300... You're receiving this email because of your account on gitlab.torproject.org.