brizental pushed to branch tor-browser-140.2.0esr-15.0-1 at The Tor Project / Applications / Tor Browser

Commits:

10 changed files:

Changes:

  • build/moz.configure/basebrowser-resources.configure
    1 
    +# Helpers
    2 
    +# -------------------------------------------------
    3 
    +
    4 
    +
    5 
    +@depends(build_project)
    6 
    +def is_desktop_build(build_project):
    7 
    +    return build_project == "browser"
    8 
    +
    9 
    +
    10 
    +# Bootstrap resources
    11 
    +# -------------------------------------------------
    12 
    +
    13 
    +
    14 
    +option(
    15 
    +    "--with-noscript",
    16 
    +    env="NOSCRIPT",
    17 
    +    nargs=1,
    18 
    +    default=None,
    19 
    +    help="Path to noscript .xpi extension archive.",
    20 
    +)
    21 
    +
    22 
    +
    23 
    +@depends(
    24 
    +    "--with-noscript",
    25 
    +    mozbuild_state_path,
    26 
    +    bootstrap_path(
    27 
    +        "noscript", no_unpack=True, when=depends("--with-noscript")(lambda x: not x)
    28 
    +    ),
    29 
    +)
    30 
    +@checking("for noscript")
    31 
    +@imports(_from="pathlib", _import="Path")
    32 
    +def noscript(value, mozbuild_state_path, _bootstrapped):
    33 
    +    if value:
    34 
    +        path = Path(value[0])
    35 
    +        if path.is_file() and path.suffix == ".xpi":
    36 
    +            return value[0]
    37 
    +        else:
    38 
    +            die("--with-noscript must be an existing .xpi file")
    39 
    +
    40 
    +    bootstrapped_location = Path(mozbuild_state_path) / "browser"
    41 
    +    for file in bootstrapped_location.glob(f"*.xpi"):
    42 
    +        if "noscript" in file.name:
    43 
    +            return str(bootstrapped_location / file)
    44 
    +
    45 
    +    # noscript is not required for building.
    46 
    +    return None
    47 
    +
    48 
    +
    49 
    +set_config("NOSCRIPT", noscript)
    50 
    +
    51 
    +
    52 
    +option(
    53 
    +    "--with-tor-browser-fonts",
    54 
    +    env="TOR_BROWSER_FONTS",
    55 
    +    nargs=1,
    56 
    +    default=None,
    57 
    +    help="Path to location of fonts directory.",
    58 
    +)
    59 
    +
    60 
    +
    61 
    +@depends(
    62 
    +    "--with-tor-browser-fonts",
    63 
    +    mozbuild_state_path,
    64 
    +    bootstrap_path(
    65 
    +        "fonts",
    66 
    +        when=depends("--with-tor-browser-fonts")(lambda x: not x) & is_desktop_build,
    67 
    +    ),
    68 
    +)
    69 
    +@checking("for tor-browser fonts directory")
    70 
    +@imports(_from="pathlib", _import="Path")
    71 
    +def tor_browser_fonts(value, mozbuild_state_path, _bootstrapped):
    72 
    +    if value:
    73 
    +        path = Path(value[0])
    74 
    +        # TODO: Do a more thorough check on the directory.
    75 
    +        if path.is_dir():
    76 
    +            return value[0]
    77 
    +        else:
    78 
    +            die("--with-tor-browser-fonts must point to a real directory.")
    79 
    +
    80 
    +    bootstrapped_location = Path(mozbuild_state_path) / "fonts"
    81 
    +    if bootstrapped_location.is_dir():
    82 
    +        return str(bootstrapped_location)
    83 
    +
    84 
    +    # tor browser fonts directory is not required for building.
    85 
    +    return None
    86 
    +
    87 
    +
    88 
    +set_config("TOR_BROWSER_FONTS", tor_browser_fonts)

  • build/moz.configure/bootstrap.configure
    ... ... @@ -4,6 +4,29 @@
    4 4 
     # License, v. 2.0. If a copy of the MPL was not distributed with this
    5 5 
     # file, You can obtain one at http://mozilla.org/MPL/2.0/.
    6 6
    7 
    +option(
    8 
    +    "--with-tor-browser-build-out",
    9 
    +    env="TOR_BROWSER_BUILD_OUT",
    10 
    +    nargs=1,
    11 
    +    default="https://tb-build-06.torproject.org/~tb-builder/tor-browser-build/out",
    12 
    +    help="URL pointing to a Tor Browser Build out folder, served over HTTP[S].",
    13 
    +)
    14 
    +
    15 
    +
    16 
    +@depends("--with-tor-browser-build-out")
    17 
    +def tor_browser_build_out(value):
    18 
    +    if value:
    19 
    +        return value[0]
    20 
    +
    21 
    +
    22 
    +option(
    23 
    +    "--enable-tor-browser-build-only-bootstrap",
    24 
    +    env="TBB_ONLY_BOOTSTRAP",
    25 
    +    default=False,
    26 
    +    help="Flag that disables bootstrapping any artifact from Mozilla's Taskcluster. Will only bootstrap artifacts from tor-browser-build.",
    27 
    +)
    28 
    +
    29 
    +
    7 30 
     option(
    8 31 
         env="MOZ_FETCHES_DIR",
    9 32 
         nargs=1,
    ... ... @@ -115,9 +138,10 @@ def bootstrap_toolchain_tasks(host):
    115 138 
     def bootstrap_path(path, **kwargs):
    116 139 
         when = kwargs.pop("when", None)
    117 140 
         allow_failure = kwargs.pop("allow_failure", None)
    141 
    +    no_unpack = kwargs.pop("no_unpack", False)
    118 142 
         if kwargs:
    119 143 
             configure_error(
    120 
    -            "bootstrap_path only takes `when` and `allow_failure` as a keyword argument"
    144 
    +            "bootstrap_path only takes `when`, `allow_failure` and `no_unpack` as keyword arguments"
    121 145 
             )
    122 146
    123 147 
         @depends(
    ... ... @@ -129,11 +153,16 @@ def bootstrap_path(path, **kwargs):
    129 153 
             build_environment,
    130 154 
             dependable(path),
    131 155 
             dependable(allow_failure),
    156 
    +        dependable(no_unpack),
    157 
    +        tor_browser_build_out,
    158 
    +        "--enable-tor-browser-build-only-bootstrap",
    159 
    +        target,
    132 160 
             when=when,
    133 161 
         )
    134 162 
         @imports("os")
    135 163 
         @imports("subprocess")
    136 164 
         @imports("sys")
    165 
    +    @imports("mozbuild.tbbutils")
    137 166 
         @imports(_from="mozbuild.dirutils", _import="ensureParentDir")
    138 167 
         @imports(_from="importlib", _import="import_module")
    139 168 
         @imports(_from="shutil", _import="rmtree")
    ... ... @@ -148,6 +177,10 @@ def bootstrap_path(path, **kwargs):
    148 177 
             build_env,
    149 178 
             path,
    150 179 
             allow_failure,
    180 
    +        no_unpack,
    181 
    +        tor_browser_build_out,
    182 
    +        tbb_only_bootstrap,
    183 
    +        target,
    151 184 
         ):
    152 185 
             if not path:
    153 186 
                 return
    ... ... @@ -158,6 +191,81 @@ def bootstrap_path(path, **kwargs):
    158 191 
             if path_parts[0] == "clang-tools":
    159 192 
                 path_prefix = path_parts.pop(0)
    160 193
    194 
    +        # Small hack because noscript is inside the browser folder.
    195 
    +        if path_parts[0] == "noscript":
    196 
    +            path_prefix = "browser"
    197 
    +
    198 
    +        def try_tbb_bootstrap(exists):
    199 
    +            if not tor_browser_build_out:
    200 
    +                return False
    201 
    +
    202 
    +            # Tor browser build doesn't have artifacts for all targets supported
    203 
    +            # by the Firefox build system. When this is empty it means we are
    204 
    +            # building for a platform which tbb doesn't support.
    205 
    +            if not target.tor_browser_build_alias:
    206 
    +                return False
    207 
    +
    208 
    +            artifact = mozbuild.tbbutils.get_artifact_name(
    209 
    +                path_parts[0], target, tasks.prefix
    210 
    +            )
    211 
    +            if not artifact:
    212 
    +                log.info("%s is not mapped to a tbb artifact", path_parts[0])
    213 
    +                return False
    214 
    +
    215 
    +            artifact_path = mozbuild.tbbutils.get_artifact_path(
    216 
    +                tor_browser_build_out, artifact, target, prefix=path_prefix
    217 
    +            )
    218 
    +            if not artifact_path:
    219 
    +                log.info("no path found in tbb/out for %s", artifact)
    220 
    +                return False
    221 
    +
    222 
    +            # We will use the name of the artifact as the index.
    223 
    +            #
    224 
    +            # It's usually unique to the artifact version, but each artifact follows
    225 
    +            # a different naming convention, so we can't really get more specific here.
    226 
    +            artifact_index = artifact_path.rsplit("/", 1)[-1]
    227 
    +            index_file = os.path.join(toolchains_base_dir, "indices", artifact)
    228 
    +            try:
    229 
    +                with open(index_file) as fh:
    230 
    +                    index = fh.read().strip()
    231 
    +            except Exception:
    232 
    +                index = None
    233 
    +            if index == artifact_index and exists:
    234 
    +                log.debug("%s is up-to-date", artifact)
    235 
    +                return True
    236 
    +
    237 
    +            command = ["artifact", "toolchain", "--from-url", artifact_path]
    238 
    +
    239 
    +            if no_unpack:
    240 
    +                command.append("--no-unpack")
    241 
    +
    242 
    +            # Note to rebasers:
    243 
    +            # From here on, it's a slightly modified copy/paste
    244 
    +            # from the end of the try_bootstrap function
    245 
    +            log.info(
    246 
    +                "%s bootstrapped toolchain from TBB in %s",
    247 
    +                "Updating" if exists else "Installing",
    248 
    +                os.path.join(toolchains_base_dir, path_prefix, artifact),
    249 
    +            )
    250 
    +            os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
    251 
    +            proc = subprocess.run(
    252 
    +                [
    253 
    +                    sys.executable,
    254 
    +                    os.path.join(build_env.topsrcdir, "mach"),
    255 
    +                    "--log-no-times",
    256 
    +                ]
    257 
    +                + command,
    258 
    +                cwd=os.path.join(toolchains_base_dir, path_prefix),
    259 
    +                check=not allow_failure,
    260 
    +            )
    261 
    +            if proc.returncode != 0 and allow_failure:
    262 
    +                return False
    263 
    +            ensureParentDir(index_file)
    264 
    +            with open(index_file, "w") as fh:
    265 
    +                fh.write(artifact_index)
    266 
    +
    267 
    +            return True
    268 
    +
    161 269 
             def try_bootstrap(exists):
    162 270 
                 if not tasks:
    163 271 
                     return False
    ... ... @@ -280,9 +388,10 @@ def bootstrap_path(path, **kwargs):
    280 388 
                 try:
    281 389 
                     # With --enable-bootstrap=no-update, we don't `try_bootstrap`, except
    282 390 
                     # when the toolchain can't be found.
    283 
    -                if (
    284 
    -                    "no-update" not in enable_bootstrap or not exists
    285 
    -                ) and not try_bootstrap(exists):
    391 
    +                if ("no-update" not in enable_bootstrap or not exists) and not (
    392 
    +                    try_tbb_bootstrap(exists)
    393 
    +                    or (not tbb_only_bootstrap and try_bootstrap(exists))
    394 
    +                ):
    286 395 
                         # If there aren't toolchain artifacts to use for this build,
    287 396 
                         # don't return a path.
    288 397 
                         return None

  • build/moz.configure/init.configure
    ... ... @@ -590,6 +590,21 @@ def split_triplet(triplet, allow_wasi=False):
    590 590 
         else:
    591 591 
             toolchain = "%s-%s" % (cpu, os)
    592 592
    593 
    +    # In tor-browser-build we use slightly different terminology for
    594 
    +    # the supported platforms. Let's prepare that OS string here.
    595 
    +    #
    596 
    +    # Not all possible platforms listed here are supported in tbb,
    597 
    +    # so this value will be empty sometimes.
    598 
    +    tor_browser_build_alias = None
    599 
    +    if canonical_os == "Android" and canonical_kernel == "Linux":
    600 
    +        tor_browser_build_alias = f"android"
    601 
    +    elif canonical_os == "GNU" and canonical_kernel == "Linux":
    602 
    +        tor_browser_build_alias = f"linux"
    603 
    +    elif canonical_os == "OSX" and canonical_kernel == "Darwin":
    604 
    +        tor_browser_build_alias = f"macos"
    605 
    +    elif canonical_os == "WINNT" and canonical_kernel == "WINNT":
    606 
    +        tor_browser_build_alias = f"windows"
    607 
    +
    593 608 
         return namespace(
    594 609 
             alias=triplet,
    595 610 
             cpu=CPU(canonical_cpu),
    ... ... @@ -604,6 +619,7 @@ def split_triplet(triplet, allow_wasi=False):
    604 619 
             toolchain=toolchain,
    605 620 
             vendor=vendor,
    606 621 
             sub_configure_alias=sub_configure_alias,
    622 
    +        tor_browser_build_alias=tor_browser_build_alias,
    607 623 
         )
    608 624
    609 625

  • build/moz.configure/torbrowser-resources.configure
    1 
    +option(
    2 
    +    "--with-tor-expert-bundle",
    3 
    +    env="TOR_EXPERT_BUNDLE",
    4 
    +    nargs=1,
    5 
    +    default=None,
    6 
    +    help="Path to location of tor-expert-bundle directory.",
    7 
    +)
    8 
    +
    9 
    +
    10 
    +@depends(
    11 
    +    "--with-tor-expert-bundle",
    12 
    +    mozbuild_state_path,
    13 
    +    bootstrap_path(
    14 
    +        "tor-expert-bundle",
    15 
    +        when=depends("--with-tor-expert-bundle")(lambda x: not x) & is_desktop_build,
    16 
    +    ),
    17 
    +)
    18 
    +@checking("for tor-expert-bundle")
    19 
    +@imports(_from="pathlib", _import="Path")
    20 
    +def tor_expert_bundle(value, mozbuild_state_path, _bootstrapped):
    21 
    +    if value:
    22 
    +        path = Path(value[0])
    23 
    +        # TODO: Do a more thorough check on the directory.
    24 
    +        if path.is_dir():
    25 
    +            return value[0]
    26 
    +        else:
    27 
    +            die("--with-tor-expert-bundle must point to a real directory.")
    28 
    +
    29 
    +    bootstrapped_location = Path(mozbuild_state_path) / "tor-expert-bundle"
    30 
    +    if bootstrapped_location.is_dir():
    31 
    +        return str(bootstrapped_location)
    32 
    +
    33 
    +    # tor-expert-bundle is not required for building.
    34 
    +    return None
    35 
    +
    36 
    +
    37 
    +set_config("TOR_EXPERT_BUNDLE", tor_expert_bundle)

  • moz.configure
    ... ... @@ -229,6 +229,8 @@ check_prog("WGET", ("wget",), allow_missing=True)
    229 229
    230 230
    231 231 
     include("build/moz.configure/toolchain.configure", when="--enable-compile-environment")
    232 
    +include("build/moz.configure/basebrowser-resources.configure")
    233 
    +include("build/moz.configure/torbrowser-resources.configure")
    232 234
    233 235 
     include("build/moz.configure/pkg.configure")
    234 236 
     include("build/moz.configure/memory.configure", when="--enable-compile-environment")

  • python/mozboot/mozboot/bootstrap.py
    ... ... @@ -52,21 +52,28 @@ Note on Artifact Mode:
    52 52 
     Artifact builds download prebuilt C++ components rather than building
    53 53 
     them locally. Artifact builds are faster!
    54 54
    55 
    -Artifact builds are recommended for people working on Firefox or
    56 
    -Firefox for Android frontends, or the GeckoView Java API. They are unsuitable
    55 
    +Artifact builds are recommended for people working on Tor Browser or
    56 
    +Tor Browser for Android frontends, or the GeckoView Java API. They are unsuitable
    57 57 
     for those working on C++ code. For more information see:
    58 58 
     https://firefox-source-docs.mozilla.org/contributing/build/artifact_builds.html.
    59 59
    60 
    -Please choose the version of Firefox you want to build (see note above):
    60 
    +# Note to Tor Browser developers
    61 
    +
    62 
    +This is still highly experimental. Expect bugs!
    63 
    +
    64 
    +Please choose the version of Tor Browser you want to build (see note above):
    61 65 
     %s
    62 66 
     Your choice: """
    63 67
    64 68 
     APPLICATIONS = OrderedDict(
    65 69 
         [
    66 
    -        ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"),
    67 
    -        ("Firefox for Desktop", "browser"),
    68 
    -        ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"),
    69 
    -        ("GeckoView/Firefox for Android", "mobile_android"),
    70 
    +        ("Tor Browser for Desktop Artifact Mode", "browser_artifact_mode"),
    71 
    +        ("Tor Browser for Desktop", "browser"),
    72 
    +        (
    73 
    +            "GeckoView/Tor Browser for Android Artifact Mode",
    74 
    +            "mobile_android_artifact_mode",
    75 
    +        ),
    76 
    +        ("GeckoView/Tor Browser for Android", "mobile_android"),
    70 77 
             ("SpiderMonkey JavaScript engine", "js"),
    71 78 
         ]
    72 79 
     )
    ... ... @@ -360,6 +367,8 @@ class Bootstrapper:
    360 367 
             getattr(self.instance, "ensure_%s_packages" % application)()
    361 368
    362 369 
         def check_code_submission(self, checkout_root: Path):
    370 
    +        return
    371 
    +
    363 372 
             if self.instance.no_interactive or which("moz-phab"):
    364 373 
                 return
    365 374
    ... ... @@ -474,8 +483,7 @@ class Bootstrapper:
    474 483 
                     configure_mercurial(hg, state_dir)
    475 484
    476 485 
             # Offer to configure Git, if the current checkout or repo type is Git.
    477 
    -        elif git and checkout_type == "git":
    478 
    -            should_configure_git = False
    486 
    +        elif False and git and checkout_type == "git":
    479 487 
                 if not self.instance.no_interactive:
    480 488 
                     should_configure_git = self.instance.prompt_yesno(prompt=CONFIGURE_GIT)
    481 489 
                 else:

  • python/mozbuild/mozbuild/action/tooltool.py
    ... ... @@ -1029,14 +1029,29 @@ def unpack_file(filename):
    1029 1029 
         """Untar `filename`, assuming it is uncompressed or compressed with bzip2,
    1030 1030 
         xz, gzip, zst, or unzip a zip file. The file is assumed to contain a single
    1031 1031 
         directory with a name matching the base of the given filename.
    1032 
    -    Xz support is handled by shelling out to 'tar'."""
    1032 
    +    Xz support is handled by shelling out to 'tar'.
    1033 
    +
    1034 
    +    tor-browser#41564 - For supporting tor-browser-build artifacts that contain
    1035 
    +    multiple directories, the archive is extracted into a directory with the
    1036 
    +    same name as the base of the filename. This modification is only applied to
    1037 
    +    tar archives, because that is all that was necessary.
    1038 
    +    """
    1033 1039 
         if os.path.isfile(filename) and tarfile.is_tarfile(filename):
    1034 1040 
             tar_file, zip_ext = os.path.splitext(filename)
    1035 1041 
             base_file, tar_ext = os.path.splitext(tar_file)
    1036 1042 
             clean_path(base_file)
    1037 1043 
             log.info('untarring "%s"' % filename)
    1038 1044 
             with TarFile.open(filename) as tar:
    1039 
    -            safe_extract(tar)
    1045 
    +            top_level_directories = set()
    1046 
    +            for name in tar.getnames():
    1047 
    +                dir = name.split("/", 1)[0]
    1048 
    +                top_level_directories.add(dir)
    1049 
    +            if len(top_level_directories) == 1:
    1050 
    +                safe_extract(tar)
    1051 
    +            else:
    1052 
    +                safe_extract(
    1053 
    +                    tar, path=os.path.join(os.path.dirname(filename), base_file)
    1054 
    +                )
    1040 1055 
         elif os.path.isfile(filename) and filename.endswith(".tar.zst"):
    1041 1056 
             import zstandard
    1042 1057

  • python/mozbuild/mozbuild/artifact_commands.py
    ... ... @@ -244,6 +244,12 @@ def artifact_clear_cache(command_context, tree=None, job=None, verbose=False):
    244 244 
         nargs="+",
    245 245 
         help="Download toolchain artifact from a given task.",
    246 246 
     )
    247 
    +@CommandArgument(
    248 
    +    "--from-url",
    249 
    +    metavar="URL",
    250 
    +    nargs="+",
    251 
    +    help="Download toolchain artifact from an arbitrary address.",
    252 
    +)
    247 253 
     @CommandArgument(
    248 254 
         "--tooltool-manifest",
    249 255 
         metavar="MANIFEST",
    ... ... @@ -273,6 +279,7 @@ def artifact_toolchain(
    273 279 
         skip_cache=False,
    274 280 
         from_build=(),
    275 281 
         from_task=(),
    282 
    +    from_url=[],
    276 283 
         tooltool_manifest=None,
    277 284 
         no_unpack=False,
    278 285 
         retry=0,
    ... ... @@ -504,6 +511,13 @@ def artifact_toolchain(
    504 511 
             record = ArtifactRecord(task_id, name)
    505 512 
             records[record.filename] = record
    506 513
    514 
    +    if from_url:
    515 
    +        for file in from_url:
    516 
    +            record = DownloadRecord(
    517 
    +                file, file.rsplit("/", 1)[-1], None, None, None, True
    518 
    +            )
    519 
    +            records[record.filename] = record
    520 
    +
    507 521 
         for record in records.values():
    508 522 
             command_context.log(
    509 523 
                 logging.INFO,

  • python/mozbuild/mozbuild/backend/base.py
    ... ... @@ -2,11 +2,14 @@
    2 2 
     # License, v. 2.0. If a copy of the MPL was not distributed with this
    3 3 
     # file, You can obtain one at http://mozilla.org/MPL/2.0/.
    4 4
    5 
    +import errno
    5 6 
     import itertools
    7 
    +import logging
    6 8 
     import os
    7 9 
     import time
    8 10 
     from abc import ABCMeta, abstractmethod
    9 11 
     from contextlib import contextmanager
    12 
    +from pathlib import Path
    10 13
    11 14 
     import mozpack.path as mozpath
    12 15 
     from mach.mixin.logging import LoggingMixin
    ... ... @@ -239,6 +242,129 @@ class BuildBackend(LoggingMixin):
    239 242 
                 with open(mozpath.join(dir, ".purgecaches"), "w") as f:
    240 243 
                     f.write("\n")
    241 244
    245 
    +    def _setup_tor_browser_environment(self, config):
    246 
    +        app = config.substs["MOZ_BUILD_APP"]
    247 
    +
    248 
    +        noscript_target_filename = "{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi"
    249 
    +        noscript_location = Path(config.substs["NOSCRIPT"])
    250 
    +
    251 
    +        def _infallible_symlink(src, dst):
    252 
    +            try:
    253 
    +                os.symlink(src, dst)
    254 
    +            except OSError as e:
    255 
    +                if e.errno == errno.EEXIST:
    256 
    +                    # If the symlink already exists, remove it and try again.
    257 
    +                    os.remove(dst)
    258 
    +                    os.symlink(src, dst)
    259 
    +                else:
    260 
    +                    return
    261 
    +
    262 
    +        if app == "browser":
    263 
    +            tbdir = Path(config.topobjdir) / "dist" / "bin"
    264 
    +
    265 
    +            if config.substs.get("OS_TARGET") == "Darwin":
    266 
    +                tbdir = next(tbdir.glob("*.app"))
    267 
    +                paths = {
    268 
    +                    "docs": tbdir / "Contents/Resources/TorBrowser/Docs",
    269 
    +                    "exts": tbdir / "Contents/Resources/distribution/extensions",
    270 
    +                    "tor_bin": tbdir / "Contents/MacOS/tor",
    271 
    +                    "tor_config": tbdir / "Contents/Resources/TorBrowser/Tor",
    272 
    +                    "fonts": tbdir / "Resources/fonts",
    273 
    +                }
    274 
    +            else:
    275 
    +                paths = {
    276 
    +                    "docs": tbdir / "TorBrowser/Docs",
    277 
    +                    "exts": tbdir / "distribution/extensions",
    278 
    +                    "tor_bin": tbdir / "TorBrowser/Tor",
    279 
    +                    "tor_config": tbdir / "TorBrowser/Data/Tor",
    280 
    +                    "fonts": tbdir / "fonts",
    281 
    +                }
    282 
    +
    283 
    +            fonts_location = Path(config.substs["TOR_BROWSER_FONTS"])
    284 
    +            if fonts_location.is_dir():
    285 
    +                self.log(
    286 
    +                    logging.INFO,
    287 
    +                    "_setup_tor_browser_environment",
    288 
    +                    {
    289 
    +                        "fonts_location": str(fonts_location),
    290 
    +                        "fonts_target": str(paths["fonts"]),
    291 
    +                    },
    292 
    +                    "Creating symlink for fonts files from {fonts_location} to {fonts_target}",
    293 
    +                )
    294 
    +
    295 
    +                for file in fonts_location.iterdir():
    296 
    +                    target = paths["fonts"] / file.name
    297 
    +                    _infallible_symlink(file, target)
    298 
    +
    299 
    +            # Set up NoScript extension
    300 
    +            if noscript_location.is_file():
    301 
    +                noscript_target = paths["exts"] / noscript_target_filename
    302 
    +                self.log(
    303 
    +                    logging.INFO,
    304 
    +                    "_setup_tor_browser_environment",
    305 
    +                    {
    306 
    +                        "noscript_location": str(noscript_location),
    307 
    +                        "noscript_target": str(noscript_target),
    308 
    +                    },
    309 
    +                    "Creating symlink for NoScript from {noscript_location} to {noscript_target}",
    310 
    +                )
    311 
    +
    312 
    +                paths["exts"].mkdir(parents=True, exist_ok=True)
    313 
    +                _infallible_symlink(noscript_location, noscript_target)
    314 
    +
    315 
    +            expert_bundle_location = Path(config.substs["TOR_EXPERT_BUNDLE"])
    316 
    +            if expert_bundle_location.is_dir():
    317 
    +                self.log(
    318 
    +                    logging.INFO,
    319 
    +                    "_setup_tor_browser_environment",
    320 
    +                    {
    321 
    +                        "expert_bundle_location": str(expert_bundle_location),
    322 
    +                    },
    323 
    +                    "Setting up tor-expert-bundle resources from {expert_bundle_location}",
    324 
    +                )
    325 
    +
    326 
    +                # Set up Tor configuration files
    327 
    +                paths["tor_config"].mkdir(parents=True, exist_ok=True)
    328 
    +                for file in ["geoip", "geoip6"]:
    329 
    +                    target = paths["tor_config"] / file
    330 
    +                    _infallible_symlink(expert_bundle_location / "data" / file, target)
    331 
    +
    332 
    +                # Set up Conjure documentation
    333 
    +                conjust_docs_location = paths["docs"] / "conjure"
    334 
    +                conjust_docs_location.mkdir(parents=True, exist_ok=True)
    335 
    +                conjure_readme = conjust_docs_location / "README.CONJURE.md"
    336 
    +                _infallible_symlink(
    337 
    +                    expert_bundle_location
    338 
    +                    / "tor/pluggable_transports/README.CONJURE.md",
    339 
    +                    conjure_readme,
    340 
    +                )
    341 
    +
    342 
    +                # Set up pluggable transports
    343 
    +                paths["tor_bin"].mkdir(parents=True, exist_ok=True)
    344 
    +                pluggable_transports_location = (
    345 
    +                    expert_bundle_location / "tor/pluggable_transports"
    346 
    +                )
    347 
    +                pluggable_transports_target = paths["tor_bin"] / "PluggableTransports"
    348 
    +                pluggable_transports_target.mkdir(parents=True, exist_ok=True)
    349 
    +                for file in pluggable_transports_location.iterdir():
    350 
    +                    # We only want the PT executables.
    351 
    +                    if os.access(file, os.X_OK) or file.suffix.lower() == ".exe":
    352 
    +                        target = pluggable_transports_target / file.name
    353 
    +                        _infallible_symlink(file, target)
    354 
    +
    355 
    +                # Setup Tor binary
    356 
    +                for item in (expert_bundle_location / "tor").iterdir():
    357 
    +                    target = paths["tor_bin"] / item.name
    358 
    +                    if target.is_file():
    359 
    +                        _infallible_symlink(item, target)
    360 
    +
    361 
    +                # Set up licenses
    362 
    +                licenses_location = paths["docs"] / "Licenses"
    363 
    +                licenses_location.mkdir(parents=True, exist_ok=True)
    364 
    +                for item in (expert_bundle_location / "docs").iterdir():
    365 
    +                    target = licenses_location / item.name
    366 
    +                    _infallible_symlink(item, target)
    367 
    +
    242 368 
         def post_build(self, config, output, jobs, verbose, status):
    243 369 
             """Called late during 'mach build' execution, after `build(...)` has finished.
    244 370
    ... ... @@ -257,6 +383,9 @@ class BuildBackend(LoggingMixin):
    257 383 
             """
    258 384 
             self._write_purgecaches(config)
    259 385
    386 
    +        if status == 0:
    387 
    +            self._setup_tor_browser_environment(config)
    388 
    +
    260 389 
             return status
    261 390
    262 391 
         @contextmanager

  • python/mozbuild/mozbuild/tbbutils.py
    1 
    +import re
    2 
    +from urllib.request import Request, urlopen
    3 
    +
    4 
    +
    5 
    +def list_files_http(url):
    6 
    +    try:
    7 
    +        req = Request(url, method="GET")
    8 
    +        with urlopen(req) as response:
    9 
    +            if response.status != 200:
    10 
    +                return []
    11 
    +            html = response.read().decode()
    12 
    +    except Exception:
    13 
    +        return []
    14 
    +
    15 
    +    links = []
    16 
    +    for href in re.findall(r'<a href="([^"]+)"', html):
    17 
    +        if href == "../":
    18 
    +            continue
    19 
    +
    20 
    +        if "tor-expert-bundle" in href:
    21 
    +            href = f"{href}/tor-expert-bundle.tar.gz"
    22 
    +
    23 
    +        links.append(href)
    24 
    +
    25 
    +    return links
    26 
    +
    27 
    +
    28 
    +TOR_BROWSER_BUILD_ARTIFACTS = [
    29 
    +    # Tor Browser Build-only artifacts, these artifacts are not common with Firefox.
    30 
    +    "noscript",
    31 
    +    "fonts",
    32 
    +    "tor-expert-bundle",
    33 
    +]
    34 
    +
    35 
    +# Mapping of artifacts from taskcluster to tor-browser-build.
    36 
    +ARTIFACT_NAME_MAP = {
    37 
    +    "cbindgen": "cbindgen",
    38 
    +    # FIXME (tor-browser-build#41471): nasm is more or less ready to go, but it needs to have the
    39 
    +    # executable in the root of the artifact folder instead of nasm/bin.
    40 
    +    # "nasm": "nasm",
    41 
    +    # FIXME (tor-browser-build#41421): the clang project as is, is not ready to use. It needs
    42 
    +    # to be repackaged with a bunch of things that differ per platform. Fun stuff.
    43 
    +    # "clang": "clang",
    44 
    +    "node": "node",
    45 
    +}
    46 
    +
    47 
    +
    48 
    +def get_artifact_name(original_artifact_name, target, host):
    49 
    +    # These are not build artifacts, they are pre-built artifacts to be added to the final build,
    50 
    +    # therefore this check can come before the host check.
    51 
    +    if original_artifact_name in TOR_BROWSER_BUILD_ARTIFACTS:
    52 
    +        return original_artifact_name
    53 
    +
    54 
    +    if host != "linux64":
    55 
    +        # Tor browser build only has development artifacts for linux64 host systems.
    56 
    +        return None
    57 
    +
    58 
    +    return ARTIFACT_NAME_MAP.get(original_artifact_name)
    59 
    +
    60 
    +
    61 
    +def get_artifact_path(url, artifact, target, prefix=""):
    62 
    +    if prefix:
    63 
    +        path = prefix
    64 
    +    else:
    65 
    +        path = artifact
    66 
    +
    67 
    +    # The `?C=M;O=D` parameters make it so links are ordered by
    68 
    +    # the last modified date. This here to make us get the latest
    69 
    +    # version of file in the case there are multiple and we just
    70 
    +    # grab the first one.
    71 
    +    files = list_files_http(f"{url}/{path}?C=M;O=D")
    72 
    +
    73 
    +    if not files:
    74 
    +        return None
    75 
    +
    76 
    +    def filter_files(files, keyword):
    77 
    +        return [file for file in files if keyword in file]
    78 
    +
    79 
    +    artifact_files = filter_files(files, artifact)
    80 
    +
    81 
    +    if len(artifact_files) == 1:
    82 
    +        return f"{url}/{path}/{artifact_files[0]}"
    83 
    +
    84 
    +    files_per_os = filter_files(artifact_files, target.tor_browser_build_alias)
    85 
    +
    86 
    +    # If there are files in the folder, but they don't have the OS in the name
    87 
    +    # it means we can get any of them because they can be used to build for any OS.
    88 
    +    # So let's just get the first one.
    89 
    +    if len(files_per_os) == 0:
    90 
    +        return f"{url}/{artifact}/{artifact_files[0]}"
    91 
    +
    92 
    +    elif len(files_per_os) == 1:
    93 
    +        return f"{url}/{artifact}/{files_per_os[0]}"
    94 
    +
    95 
    +    matches = filter_files(files_per_os, target.cpu)
    96 
    +
    97 
    +    return f"{url}/{artifact}/{matches[0]}" if matches else None

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.torproject.org. Manage all notifications · Help Notification message regarding https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/1b62300257519c05a8f3c6b0d2a77e132738152b...c1827fba852ac331b5dfad6308931659cdce2724 at 1755785193