+# Helpers

+# -------------------------------------------------

+

+

+@depends(build_project)

+def is_desktop_build(build_project):

+    return build_project == "browser"

+

+

+# Bootstrap resources

+# -------------------------------------------------

+

+

+option(

+    "--with-noscript",

+    env="NOSCRIPT",

+    nargs=1,

+    default=None,

+    help="Path to noscript .xpi extension archive.",

+)

+

+

+@depends(

+    "--with-noscript",

+    mozbuild_state_path,

+    bootstrap_path(

+        "noscript", no_unpack=True, when=depends("--with-noscript")(lambda x: not x)

+    ),

+)

+@checking("for noscript")

+@imports(_from="pathlib", _import="Path")

+def noscript(value, mozbuild_state_path, _bootstrapped):

+    if value:

+        path = Path(value[0])

+        if path.is_file() and path.suffix == ".xpi":

+            return value[0]

+        else:

+            die("--with-noscript must be an existing .xpi file")

+

+    bootstrapped_location = Path(mozbuild_state_path) / "browser"

+    for file in bootstrapped_location.glob(f"*.xpi"):

+        if "noscript" in file.name:

+            return str(bootstrapped_location / file)

+

+    # noscript is not required for building.

+    return None

+

+

+set_config("NOSCRIPT", noscript)

+

+

+option(

+    "--with-tor-browser-fonts",

+    env="TOR_BROWSER_FONTS",

+    nargs=1,

+    default=None,

+    help="Path to location of fonts directory.",

+)

+

+

+@depends(

+    "--with-tor-browser-fonts",

+    mozbuild_state_path,

+    bootstrap_path(

+        "fonts",

+        when=depends("--with-tor-browser-fonts")(lambda x: not x) & is_desktop_build,

+    ),

+)

+@checking("for tor-browser fonts directory")

+@imports(_from="pathlib", _import="Path")

+def tor_browser_fonts(value, mozbuild_state_path, _bootstrapped):

+    if value:

+        path = Path(value[0])

+        # TODO: Do a more thorough check on the directory.

+        if path.is_dir():

+            return value[0]

+        else:

+            die("--with-tor-browser-fonts must point to a real directory.")

+

+    bootstrapped_location = Path(mozbuild_state_path) / "fonts"

+    if bootstrapped_location.is_dir():

+        return str(bootstrapped_location)

+

+    # tor browser fonts directory is not required for building.

+    return None

+

+

+set_config("TOR_BROWSER_FONTS", tor_browser_fonts)

+option(

+    "--with-tor-browser-build-out",

+    env="TOR_BROWSER_BUILD_OUT",

+    nargs=1,

+    default="https://tb-build-06.torproject.org/~tb-builder/tor-browser-build/out",

+    help="URL pointing to a Tor Browser Build out folder, served over HTTP[S].",

+)

+

+

+@depends("--with-tor-browser-build-out")

+def tor_browser_build_out(value):

+    if value:

+        return value[0]

+

+

+option(

+    "--enable-tor-browser-build-only-bootstrap",

+    env="TBB_ONLY_BOOTSTRAP",

+    default=False,

+    help="Flag that disables bootstrapping any artifact from Mozilla's Taskcluster. Will only bootstrap artifacts from tor-browser-build.",

+)

+

+

+    no_unpack = kwargs.pop("no_unpack", False)

+            "bootstrap_path only takes `when`, `allow_failure` and `no_unpack` as keyword arguments"

+        dependable(no_unpack),

+        tor_browser_build_out,

+        "--enable-tor-browser-build-only-bootstrap",

+        target,

+    @imports("mozbuild.tbbutils")

+        no_unpack,

+        tor_browser_build_out,

+        tbb_only_bootstrap,

+        target,

+        # Small hack because noscript is inside the browser folder.

+        if path_parts[0] == "noscript":

+            path_prefix = "browser"

+

+        def try_tbb_bootstrap(exists):

+            if not tor_browser_build_out:

+                return False

+

+            # Tor browser build doesn't have artifacts for all targets supported

+            # by the Firefox build system. When this is empty it means we are

+            # building for a platform which tbb doesn't support.

+            if not target.tor_browser_build_alias:

+                return False

+

+            artifact = mozbuild.tbbutils.get_artifact_name(path_parts[0], tasks.prefix)

+            if not artifact:

+                log.info("%s is not mapped to a tbb artifact", path_parts[0])

+                return False

+

+            artifact_path = mozbuild.tbbutils.get_artifact_path(

+                tor_browser_build_out,

+                artifact,

+                target,

+                prefix=path_prefix,

+                log=log.warning,

+            )

+            if not artifact_path:

+                log.info("no path found in tbb/out for %s", artifact)

+                return False

+

+            # We will use the name of the artifact as the index.

+            #

+            # It's usually unique to the artifact version, but each artifact follows

+            # a different naming convention, so we can't really get more specific here.

+            artifact_index = artifact_path.rsplit("/", 1)[-1]

+            index_file = os.path.join(toolchains_base_dir, "indices", artifact)

+            try:

+                with open(index_file) as fh:

+                    index = fh.read().strip()

+            except Exception:

+                index = None

+            if index == artifact_index and exists:

+                log.debug("%s is up-to-date", artifact)

+                return True

+

+            command = ["artifact", "toolchain", "--from-url", artifact_path]

+

+            if no_unpack:

+                command.append("--no-unpack")

+

+            # Note to rebasers:

+            # From here on, it's a slightly modified copy/paste

+            # from the end of the try_bootstrap function

+            log.info(

+                "%s bootstrapped toolchain from TBB in %s",

+                "Updating" if exists else "Installing",

+                os.path.join(toolchains_base_dir, path_prefix, artifact),

+            )

+            os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)

+            proc = subprocess.run(

+                [

+                    sys.executable,

+                    os.path.join(build_env.topsrcdir, "mach"),

+                    "--log-no-times",

+                ]

+                + command,

+                cwd=os.path.join(toolchains_base_dir, path_prefix),

+                check=not allow_failure,

+            )

+            if proc.returncode != 0 and allow_failure:

+                return False

+            ensureParentDir(index_file)

+            with open(index_file, "w") as fh:

+                fh.write(artifact_index)

+

+            return True

+

+                if ("no-update" not in enable_bootstrap or not exists) and not (

+                    try_tbb_bootstrap(exists)

+                    or (not tbb_only_bootstrap and try_bootstrap(exists))

+                ):

+    # In tor-browser-build we use slightly different terminology for

+    # the supported platforms. Let's prepare that OS string here.

+    #

+    # Not all possible platforms listed here are supported in tbb,

+    # so this value will be empty sometimes.

+    tor_browser_build_alias = None

+    if canonical_os == "Android" and canonical_kernel == "Linux":

+        tor_browser_build_alias = f"android"

+    elif canonical_os == "GNU" and canonical_kernel == "Linux":

+        tor_browser_build_alias = f"linux"

+    elif canonical_os == "OSX" and canonical_kernel == "Darwin":

+        tor_browser_build_alias = f"macos"

+    elif canonical_os == "WINNT" and canonical_kernel == "WINNT":

+        tor_browser_build_alias = f"windows"

+

+        tor_browser_build_alias=tor_browser_build_alias,

+include("build/moz.configure/basebrowser-resources.configure")

+Artifact builds are recommended for people working on Tor Browser or

+Base Browser for Android frontends, or the GeckoView Java API. They are unsuitable

+# Note to Base Browser developers

+

+This is still highly experimental. Expect bugs!

+

+Please choose the version of Base Browser you want to build (see note above):

+        ("Base Browser for Desktop Artifact Mode", "browser_artifact_mode"),

+        ("Base Browser for Desktop", "browser"),

+        (

+            "GeckoView/Base Browser for Android Artifact Mode",

+            "mobile_android_artifact_mode",

+        ),

+        ("GeckoView/Base Browser for Android", "mobile_android"),

+        return

+

+        elif False and git and checkout_type == "git":

+    Xz support is handled by shelling out to 'tar'.

+

+    tor-browser#41564 - For supporting tor-browser-build artifacts that contain

+    multiple directories, the archive is extracted into a directory with the

+    same name as the base of the filename. This modification is only applied to

+    tar archives, because that is all that was necessary.

+    """

+            top_level_directories = set()

+            for name in tar.getnames():

+                dir = name.split("/", 1)[0]

+                top_level_directories.add(dir)

+            if len(top_level_directories) == 1:

+                safe_extract(tar)

+            else:

+                safe_extract(

+                    tar, path=os.path.join(os.path.dirname(filename), base_file)

+                )

+@CommandArgument(

+    "--from-url",

+    metavar="URL",

+    nargs="+",

+    help="Download toolchain artifact from an arbitrary address.",

+)

+    from_url=[],

+    if from_url:

+        for file in from_url:

+            record = DownloadRecord(

+                file, file.rsplit("/", 1)[-1], None, None, None, True

+            )

+            records[record.filename] = record

+

+import errno

+import logging

+from pathlib import Path

+    def _setup_tor_browser_environment(self, config):

+        app = config.substs["MOZ_BUILD_APP"]

+

+        noscript_target_filename = "{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi"

+        noscript_location = config.substs.get("NOSCRIPT")

+

+        def _infallible_symlink(src, dst):

+            try:

+                os.symlink(src, dst)

+            except OSError as e:

+                if e.errno == errno.EEXIST:

+                    # If the symlink already exists, remove it and try again.

+                    os.remove(dst)

+                    os.symlink(src, dst)

+                else:

+                    return

+

+        if app == "browser":

+            tbdir = Path(config.topobjdir) / "dist" / "bin"

+

+            if config.substs.get("OS_TARGET") == "Darwin":

+                tbdir = next(tbdir.glob("*.app"))

+                paths = {

+                    "docs": tbdir / "Contents/Resources/TorBrowser/Docs",

+                    "exts": tbdir / "Contents/Resources/distribution/extensions",

+                    "fonts": tbdir / "Resources/fonts",

+                }

+            else:

+                paths = {

+                    "docs": tbdir / "TorBrowser/Docs",

+                    "exts": tbdir / "distribution/extensions",

+                    "fonts": tbdir / "fonts",

+                }

+

+            fonts_location = config.substs.get("TOR_BROWSER_FONTS")

+            if fonts_location:

+                self.log(

+                    logging.INFO,

+                    "_setup_tor_browser_environment",

+                    {

+                        "fonts_location": fonts_location,

+                        "fonts_target": str(paths["fonts"]),

+                    },

+                    "Creating symlink for fonts files from {fonts_location} to {fonts_target}",

+                )

+

+                for file in Path(fonts_location).iterdir():

+                    target = paths["fonts"] / file.name

+                    _infallible_symlink(file, target)

+

+            # Set up NoScript extension

+            if noscript_location:

+                noscript_target = paths["exts"] / noscript_target_filename

+                self.log(

+                    logging.INFO,

+                    "_setup_tor_browser_environment",

+                    {

+                        "noscript_location": noscript_location,

+                        "noscript_target": str(noscript_target),

+                    },

+                    "Creating symlink for NoScript from {noscript_location} to {noscript_target}",

+                )

+

+                paths["exts"].mkdir(parents=True, exist_ok=True)

+                _infallible_symlink(noscript_location, noscript_target)

+

+        if status == 0:

+            self._setup_tor_browser_environment(config)

+

+import re

+from urllib.request import Request, urlopen

+

+

+def list_files_http(url):

+    try:

+        req = Request(url, method="GET")

+        with urlopen(req) as response:

+            if response.status != 200:

+                return []

+            html = response.read().decode()

+    except Exception:

+        return []

+

+    links = []

+    for href in re.findall(r'<a href="([^"]+)"', html):

+        if href == "../":

+            continue

+

+        links.append(href)

+

+    return links

+

+

+TOR_BROWSER_BUILD_ARTIFACTS = [

+    # Tor Browser Build-only artifacts, these artifacts are not common with Firefox.

+    "noscript",

+    "fonts",

+]

+

+# Mapping of artifacts from taskcluster to tor-browser-build.

+ARTIFACT_NAME_MAP = {

+    "cbindgen": "cbindgen",

+    # FIXME (tor-browser-build#41471): nasm is more or less ready to go, but it needs to have the

+    # executable in the root of the artifact folder instead of nasm/bin.

+    # "nasm": "nasm",

+    # FIXME (tor-browser-build#41421): the clang project as is, is not ready to use. It needs

+    # to be repackaged with a bunch of things that differ per platform. Fun stuff.

+    # "clang": "clang",

+    "node": "node",

+}

+

+

+def get_artifact_name(original_artifact_name, host):

+    # These are not build artifacts, they are pre-built artifacts to be added to the final build,

+    # therefore this check can come before the host check.

+    if original_artifact_name in TOR_BROWSER_BUILD_ARTIFACTS:

+        return original_artifact_name

+

+    if host != "linux64":

+        # Tor browser build only has development artifacts for linux64 host systems.

+        return None

+

+    return ARTIFACT_NAME_MAP.get(original_artifact_name)

+

+

+def get_artifact_path(url, artifact, target, prefix="", log=lambda *args, **kwargs: {}):

+    if prefix:

+        path = prefix

+    else:

+        path = artifact

+

+    # The `?C=M;O=D` parameters make it so links are ordered by

+    # the last modified date. This here to make us get the latest

+    # version of file in the case there are multiple and we just

+    # grab the first one.

+    files = list_files_http(f"{url}/{path}?C=M;O=D")

+

+    if not files:

+        log(f"No files found in {url} for {artifact}.")

+        return None

+

+    def filter_files(files, keyword):

+        return [file for file in files if keyword in file]

+

+    artifact_files = [file for file in files if file.startswith(artifact)]

+

+    if len(artifact_files) == 0:

+        log(f"No files found in {url} for {artifact}.")

+        return None

+

+    if len(artifact_files) == 1:

+        return f"{url}/{path}/{artifact_files[0]}"

+

+    files_per_os = filter_files(artifact_files, target.tor_browser_build_alias)

+

+    # If there are files in the folder, but they don't have the OS in the name

+    # it probably means we can get any of them because they can be used to build

+    # for any OS. So let's just get the first one.

+    #

+    # Note: It could be the case that the artifact _is_ OS dependant, but there

+    # just are no files for the OS we are looking for. In that case, this will

+    # return an incorrect artifact. This should not happen often though and is

+    # something we cannot address until artifact names are standardized on tbb.

+    if len(files_per_os) == 0:

+        return f"{url}/{artifact}/{artifact_files[0]}"

+

+    elif len(files_per_os) == 1:

+        return f"{url}/{artifact}/{files_per_os[0]}"

+

+    matches = filter_files(files_per_os, target.cpu)

+

+    return f"{url}/{artifact}/{matches[0]}" if matches else None

+["test_tbbutils.py"]

+subsuite = "base-browser"

+

+import unittest

+from types import SimpleNamespace

+from unittest.mock import MagicMock, patch

+

+import mozunit

+

+from mozbuild.tbbutils import get_artifact_path, list_files_http

+

+

+class TestGetArtifactName(unittest.TestCase):

+    def setUp(self):

+        self.artifact = "artifact"

+        self.host = "linux64"

+

+    @patch("mozbuild.tbbutils.TOR_BROWSER_BUILD_ARTIFACTS", new=["artifact"])

+    def test_artifact_in_tbb_artifacts(self):

+        from mozbuild.tbbutils import get_artifact_name

+

+        result = get_artifact_name(self.artifact, self.host)

+        self.assertEqual(result, self.artifact)

+

+    @patch("mozbuild.tbbutils.ARTIFACT_NAME_MAP", new={"artifact": "tcafitra"})

+    def test_host_is_not_linux64(self):

+        from mozbuild.tbbutils import get_artifact_name

+

+        result = get_artifact_name(self.artifact, "linux64-aarch64")

+        self.assertIsNone(result)

+

+    @patch("mozbuild.tbbutils.ARTIFACT_NAME_MAP", new={"artifact": "tcafitra"})

+    def test_mapped_artifact(self):

+        from mozbuild.tbbutils import get_artifact_name

+

+        result = get_artifact_name(self.artifact, self.host)

+        self.assertEqual(result, self.artifact[::-1])

+

+

+class TestGetArtifactPath(unittest.TestCase):

+    def setUp(self):

+        self.url = "http://example.com"

+        self.artifact = "artifact"

+        # This is just an example target which is valid. But it doesn't make

+        # any difference and could be anything for these tests.

+        self.target = SimpleNamespace(tor_browser_build_alias="linux", cpu="x86_64")

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_no_files_returns_none(self, mock_list_files):

+        mock_list_files.return_value = []

+        result = get_artifact_path(self.url, self.artifact, self.target)

+        self.assertIsNone(result)

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_no_matching_files_returns_none(self, mock_list_files):

+        mock_list_files.return_value = ["somethingelse.zip", "yetanotherthing.zip"]

+        result = get_artifact_path(self.url, self.artifact, self.target)

+        self.assertIsNone(result)

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_single_artifact_match(self, mock_list_files):

+        mock_list_files.return_value = ["artifact-1.zip"]

+        result = get_artifact_path(self.url, self.artifact, self.target)

+        self.assertEqual(result, f"{self.url}/{self.artifact}/artifact-1.zip")

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_artifact_without_os_returns_first(self, mock_list_files):

+        mock_list_files.return_value = ["artifact-1.zip", "artifact-2.zip"]

+        result = get_artifact_path(self.url, self.artifact, self.target)

+        self.assertTrue(result.startswith(f"{self.url}/{self.artifact}/"))

+        self.assertIn("artifact-", result)

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_artifact_with_os_match(self, mock_list_files):

+        mock_list_files.return_value = [

+            "artifact-windows.zip",

+            "artifact-linux.zip",

+        ]

+        result = get_artifact_path(self.url, self.artifact, self.target)

+        self.assertEqual(result, f"{self.url}/{self.artifact}/artifact-linux.zip")

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_artifact_with_cpu_match(self, mock_list_files):

+        mock_list_files.return_value = [

+            "artifact-linux-arm.zip",

+            "artifact-linux-x86_64.zip",

+        ]

+        result = get_artifact_path(self.url, self.artifact, self.target)

+        self.assertEqual(

+            result, f"{self.url}/{self.artifact}/artifact-linux-x86_64.zip"

+        )

+

+    @patch("mozbuild.tbbutils.list_files_http")

+    def test_artifact_with_prefix(self, mock_list_files):

+        mock_list_files.return_value = ["artifact-1.zip"]

+

+        prefix = "prefix"

+        result = get_artifact_path(self.url, self.artifact, self.target, prefix=prefix)

+        self.assertEqual(result, f"{self.url}/{prefix}/artifact-1.zip")

+        mock_list_files.assert_called_with(f"{self.url}/{prefix}?C=M;O=D")

+

+

+class TestListFilesHttp(unittest.TestCase):

+    def setUp(self):

+        self.url = "http://example.com"

+

+    @patch("mozbuild.tbbutils.urlopen")

+    def test_non_200_status_returns_empty(self, mock_urlopen):

+        mock_resp = MagicMock()

+        mock_resp.status = 404

+        mock_resp.read.return_value = b""

+        mock_urlopen.return_value.__enter__.return_value = mock_resp

+

+        result = list_files_http(self.url)

+        self.assertEqual(result, [])

+

+    @patch("mozbuild.tbbutils.urlopen")

+    def test_exception_returns_empty(self, mock_urlopen):

+        mock_urlopen.side_effect = Exception("network error")

+        result = list_files_http(self.url)

+        self.assertEqual(result, [])

+

+    @patch("mozbuild.tbbutils.urlopen")

+    def test_regular_links(self, mock_urlopen):

+        html = b"""

+        <html><body>

+        <a href="../">Parent</a>

+        <a href="file1.zip">file1</a>

+        <a href="file2.zip">file2</a>

+        </body></html>

+        """

+        mock_resp = MagicMock()

+        mock_resp.status = 200

+        mock_resp.read.return_value = html

+        mock_urlopen.return_value.__enter__.return_value = mock_resp

+

+        result = list_files_http(self.url)

+        self.assertEqual(result, ["file1.zip", "file2.zip"])

+

+

+if __name__ == "__main__":

+    mozunit.main()

+[DEFAULT]

+subsuite = "base-browser"

+

+["test_android.py"]

+

+["test_dtd.py"]