morgan pushed to branch maint-13.5 at The Tor Project / Applications / tor-browser-build

Commits: 72f49dc1 by Pier Angelo Vendrame at 2024-11-21T17:35:24+01:00 Bug 41284: Stop sync'ing changelogs between channels.

- - - - - bf089520 by Pier Angelo Vendrame at 2024-11-21T17:45:32+01:00 Bug 41200: Remove the tools for allowed_addons.json.

For tor-browser#42618 and tor-browser#42619, we stopped using the allowed_addons.json, but initially we kept the tools to update it. However, the file creates some confusion, because it is used only for Andorid, but should be updated also when doing desktop-only releases when they update NoScript, or Android nightly builds might fail. So, since as a matter of fact we do not use that file anymore, we decided to stop updating it and remove the related tools.

- - - - - 84e2ad6a by Pier Angelo Vendrame at 2024-11-21T18:10:39+01:00 Bug 41310: 13.5-specific fixes for relprep.py.

When I wrote relprep.py, I did not think about a legacy channel, as we never had one before. Our approach was to keep everything to build releases as stable, so we decided to comment or remove the undesired functionalities rather than implementing a legacy channel in the script.

- - - - -

11 changed files:

- − .gitattributes - .gitlab/issue_templates/Release Prep - Tor Browser Alpha.md - .gitlab/issue_templates/Release Prep - Tor Browser Stable.md - − projects/browser/allowed_addons.json - projects/browser/build.android - projects/browser/config - − projects/browser/verify_allowed_addons.py - projects/manual/config - − tools/fetch_allowed_addons.py - tools/fetch_changelogs.py - tools/relprep.py

Changes:

===================================== .gitattributes deleted ===================================== @@ -1 +0,0 @@ -projects/browser/allowed_addons.json -diff

===================================== .gitlab/issue_templates/Release Prep - Tor Browser Alpha.md ===================================== @@ -59,8 +59,6 @@ Tor Browser Alpha (and Nightly) are on the `main` branch - [ ] `fenix_version` : update to match alpha `firefox-android` build tag - [ ] `browser_branch` : update to match alpha `firefox-android` build tag - [ ] `browser_build` : update to match alpha `firefox-android` build tag - - [ ] Update allowed_addons.json by running (from `tor-browser-build` root): - - `./tools/fetch_allowed_addons.py > projects/browser/allowed_addons.json` - [ ] Update `projects/translation/config`: - [ ] run `make list_translation_updates-alpha` to get updated hashes - [ ] `steps/base-browser/git_hash` : update with `HEAD` commit of project's `base-browser` branch

===================================== .gitlab/issue_templates/Release Prep - Tor Browser Stable.md ===================================== @@ -60,8 +60,6 @@ Tor Browser Stable lives in the various `maint-$(TOR_BROWSER_MAJOR).$(TOR_BROWSE - [ ] `browser_branch` : update to match stable `firefox-android` build tag - [ ] `browser_build` : update to match stable `firefox-android` build tag variant: Beta - - [ ] Update allowed_addons.json by running (from `tor-browser-build` root): - - `./tools/fetch_allowed_addons.py > projects/browser/allowed_addons.json` - [ ] Update `projects/translation/config`: - [ ] run `make list_translation_updates-release` to get updated hashes - [ ] `steps/base-browser/git_hash` : update with `HEAD` commit of project's `base-browser` branch

===================================== projects/browser/allowed_addons.json deleted ===================================== The diff for this file was not included because it is too large.

===================================== projects/browser/build.android ===================================== @@ -39,15 +39,6 @@ unzip ../omni.ja }) %] popd

- -[% IF c("var/verify_allowed_addons") %] - # Check that allowed_addons.json contains the right versions of our bundled extension(s). - # If so, replace the default allowed_addons.json by ours in the apk assets folder. - $rootdir/verify_allowed_addons.py "$rootdir/allowed_addons.json" "$noscript_path" -[% END %] - -mv $rootdir/allowed_addons.json $assets_dir/allowed_addons.json - mkdir apk pushd apk 7zz x "$apk"

===================================== projects/browser/config ===================================== @@ -49,7 +49,6 @@ targets: android: build: '[% INCLUDE build.android %]' var: - verify_allowed_addons: 1 arch_deps: - 7zip - openjdk-17-jdk-headless @@ -159,10 +158,6 @@ input_files: enable: '[% c("var/namecoin") %]' - filename: namecoin.patch enable: '[% c("var/namecoin") %]' - - filename: allowed_addons.json - enable: '[% c("var/android") %]' - - filename: verify_allowed_addons.py - enable: '[% c("var/android") && c("var/verify_allowed_addons") %]' - project: manual name: manual enable: '[% ! c("var/android") && c("var/tor-browser") %]'

===================================== projects/browser/verify_allowed_addons.py deleted ===================================== @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys -import hashlib -import zipfile - -def find_addon(addons, addon_id): - results = addons['results'] - for x in results: - addon = x['addon'] - if addon['guid'] == addon_id: - return addon - sys.exit("Error: cannot find addon " + addon_id) - -def verify_extension_version(addons, addon_id, version): - addon = find_addon(addons, addon_id) - expected_version = addon['current_version']['version'] - if version != expected_version: - sys.exit("Error: version " + version + " != " + expected_version) - -def verify_extension_hash(addons, addon_id, hash): - addon = find_addon(addons, addon_id) - expected_hash = addon["current_version"]["files"][0]["hash"] - if hash != expected_hash: - sys.exit("Error: hash " + hash + " != " + expected_hash) - -def read_extension_manifest(path): - return json.loads(zipfile.ZipFile(path, 'r').read('manifest.json')) - -def main(argv): - allowed_addons_path = argv[0] - noscript_path = argv[1] - - addons = None - with open(allowed_addons_path, 'r') as file: - addons = json.loads(file.read()) - - noscript_hash = None - with open(noscript_path, 'rb') as file: - noscript_hash = "sha256:" + hashlib.sha256(file.read()).hexdigest() - - noscript_version = read_extension_manifest(noscript_path)["version"] - - verify_extension_hash(addons, '{73a6fe31-595d-460b-a920-fcc0f8843232}', noscript_hash) - verify_extension_version(addons, '{73a6fe31-595d-460b-a920-fcc0f8843232}', noscript_version) - -if __name__ == "__main__": - main(sys.argv[1:])

===================================== projects/manual/config ===================================== @@ -1,7 +1,7 @@ # vim: filetype=yaml sw=2 # To update, see doc/how-to-update-the-manual.txt # Remember to update also the package's hash, with the version! -version: 222718 +version: 210938 filename: 'manual-[% c("version") %]-[% c("var/build_id") %].tar.[% c("compress_tar") %]' container: use_container: 1 @@ -23,6 +23,6 @@ input_files: - project: container-image - URL: 'https://build-sources.tbb.torproject.org/manual_%5B% c("version") %].zip' name: manual - sha256sum: 051174ba012fa2241e865cc604658a0af116d3bbf9d02474025277fff1b34636 + sha256sum: eb83259f0525a14dae1a1c3944e1e5ac3a2f8111a42834ab0f401628c8a38791 - filename: packagemanual.py name: package_script

===================================== tools/fetch_allowed_addons.py deleted ===================================== @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -import urllib.request -import json -import base64 -import sys - -NOSCRIPT = "{73a6fe31-595d-460b-a920-fcc0f8843232}" - - -def fetch(x): - with urllib.request.urlopen(x) as response: - return response.read() - - -def find_addon(addons, addon_id): - results = addons["results"] - for x in results: - addon = x["addon"] - if addon["guid"] == addon_id: - return addon - - -def fetch_and_embed_icons(addons): - results = addons["results"] - for x in results: - addon = x["addon"] - icon_data = fetch(addon["icon_url"]) - addon["icon_url"] = "data:image/png;base64," + str( - base64.b64encode(icon_data), "utf8" - ) - - -def fetch_allowed_addons(amo_collection=None): - if amo_collection is None: - amo_collection = "83a9cccfe6e24a34bd7b155ff9ee32" - url = f"https://services.addons.mozilla.org/api/v4/accounts/account/mozilla/collecti..." - data = json.loads(fetch(url)) - fetch_and_embed_icons(data) - data["results"].sort(key=lambda x: x["addon"]["guid"]) - return data - - -def main(argv): - data = fetch_allowed_addons(argv[0] if len(argv) > 1 else None) - # Check that NoScript is present - if find_addon(data, NOSCRIPT) is None: - sys.exit("Error: cannot find NoScript.") - print(json.dumps(data, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main(sys.argv[1:])

===================================== tools/fetch_changelogs.py ===================================== @@ -21,12 +21,10 @@ class EntryType(enum.IntFlag):

class Platform(enum.IntFlag): - WINDOWS = 8 - MACOS = 4 - LINUX = 2 - ANDROID = 1 - DESKTOP = 8 | 4 | 2 - ALL_PLATFORMS = 8 | 4 | 2 | 1 + WINDOWS = 2 + MACOS = 1 + DESKTOP = 2 | 1 + ALL_PLATFORMS = 2 | 1

class ChangelogEntry: @@ -52,10 +50,6 @@ class ChangelogEntry: platforms.append("Windows") if self.platform & Platform.MACOS: platforms.append("macOS") - if self.platform & Platform.LINUX: - platforms.append("Linux") - if self.platform & Platform.ANDROID: - platforms.append("Android") return " + ".join(platforms)

def __lt__(self, other): @@ -78,15 +72,8 @@ class ChangelogEntry:

class UpdateEntry(ChangelogEntry): def __init__(self, name, version, is_mb): - if name == "Firefox" and not is_mb: - platform = Platform.DESKTOP - num_platforms = 3 - elif name == "GeckoView" or name == "Zstandard": - platform = Platform.ANDROID - num_platforms = 1 - else: - platform = Platform.ALL_PLATFORMS - num_platforms = 4 + platform = Platform.ALL_PLATFORMS + num_platforms = 2 super().__init__( EntryType.UPDATE, platform, num_platforms, name == "Go", is_mb ) @@ -107,8 +94,8 @@ class Issue(ChangelogEntry): platform = 0 num_platforms = 0 if "Desktop" in j["labels"]: - platform = Platform.DESKTOP - num_platforms += 3 + platform = Platform.ALL_PLATFORMS + num_platforms += 2 else: if "Windows" in j["labels"]: platform |= Platform.WINDOWS @@ -116,20 +103,13 @@ class Issue(ChangelogEntry): if "MacOS" in j["labels"]: platform |= Platform.MACOS num_platforms += 1 - if "Linux" in j["labels"]: - platform |= Platform.LINUX - num_platforms += 1 - if "Android" in j["labels"]: - if is_mb and num_platforms == 0: + if not platform: + if "Android" in j["labels"] or "Linux" in j["labels"]: raise Exception( - f"Android-only issue on Mullvad Browser: {j['references']['full']}!" + f"The legacy channel should include only fixes for macOS and/or Windows, please check {self.project}#{self.number}." ) - elif not is_mb: - platform |= Platform.ANDROID - num_platforms += 1 - if not platform or (is_mb and platform == Platform.DESKTOP): platform = Platform.ALL_PLATFORMS - num_platforms = 4 + num_platforms = 2 is_build = "Build System" in j["labels"] super().__init__( EntryType.ISSUE, platform, num_platforms, is_build, is_mb

===================================== tools/relprep.py ===================================== @@ -4,7 +4,6 @@ from collections import namedtuple import configparser from datetime import datetime, timezone from hashlib import sha256 -import json import locale import logging from pathlib import Path @@ -16,7 +15,6 @@ from git import Repo import requests import ruamel.yaml

-from fetch_allowed_addons import NOSCRIPT, fetch_allowed_addons, find_addon import fetch_changelogs from update_manual import update_manual

@@ -93,13 +91,10 @@ class ReleasePreparation: self.base_path = Path(repo_path) self.repo = Repo(self.base_path)

- self.tor_browser = bool(kwargs.get("tor_browser", True)) - self.mullvad_browser = bool(kwargs.get("mullvad_browser", True)) - if not self.tor_browser and not self.mullvad_browser: - raise ValueError("Nothing to do") - self.android = kwargs.get("android", self.tor_browser) - if not self.tor_browser and self.android: - raise ValueError("Only Tor Browser supports Android") + # Legacy channel, always do Tor Browser desktop only. + self.tor_browser = True + self.mullvad_browser = False + self.android = False

logger.debug( "Tor Browser: %s; Mullvad Browser: %s; Android: %s", @@ -142,7 +137,8 @@ class ReleasePreparation: # Do not update Go anymore: 1.21.x is not listed anymore in # the download page as it is EOL as of August 13, 2024. # self.update_go() - self.update_manual() + # Freeze the manual to before 14.0. + # self.update_manual()

self.update_changelogs() self.update_rbm_conf() @@ -304,7 +300,6 @@ class ReleasePreparation: targets = ["base-browser"] if self.tor_browser: targets.append("tor-browser") - targets.append("fenix") if self.mullvad_browser: targets.append("mullvad-browser") for i in targets: @@ -319,28 +314,27 @@ class ReleasePreparation: logger.info("Updating addons") config = self.load_config("browser")

- amo_data = fetch_allowed_addons() - logger.debug("Fetched AMO data") - if self.android: - with ( - self.base_path / "projects/browser/allowed_addons.json" - ).open("w") as f: - json.dump(amo_data, f, indent=2) - - noscript = find_addon(amo_data, NOSCRIPT) logger.debug("Updating NoScript") - self.update_addon_amo(config, "noscript", noscript) + self.update_addon_amo( + config, "noscript", "{73a6fe31-595d-460b-a920-fcc0f8843232}" + ) if self.mullvad_browser: logger.debug("Updating uBlock Origin") - ublock = find_addon(amo_data, "uBlock0@raymondhill.net") - self.update_addon_amo(config, "ublock-origin", ublock) + self.update_addon_amo( + config, "ublock-origin", "uBlock0@raymondhill.net" + ) logger.debug("Updating the Mullvad Browser extension") self.update_mullvad_addon(config)

self.save_config("browser", config)

- def update_addon_amo(self, config, name, addon): - addon = addon["current_version"]["files"][0] + def update_addon_amo(self, config, name, addon_id): + r = requests.get( + f"https://services.addons.mozilla.org/api/v4/addons/addon/%7Baddon_id%7D" + ) + r.raise_for_status() + amo_data = r.json() + addon = amo_data["current_version"]["files"][0] assert addon["hash"].startswith("sha256:") addon_input = self.find_input(config, name) addon_input["URL"] = addon["url"] @@ -523,6 +517,10 @@ class ReleasePreparation: self.build_number, ) continue + if version.major > self.version.major: + # Ignore more recent version. + # E.g., for the legacy channel. + continue key = (project, version.channel) if key not in self.last_releases: self.last_releases[key] = [] @@ -590,14 +588,9 @@ class ReleasePreparation: changelogs = cb.create(**kwargs)

path = f"projects/browser/Bundle-Data/Docs-{tag_prefix.upper()}/ChangeLog.txt" - stable_tag = self.last_releases[(tag_prefix, "release")][0].tag - alpha_tag = self.last_releases[(tag_prefix, "alpha")][0].tag - if stable_tag.tagged_date > alpha_tag.tagged_date: - last_tag = stable_tag - else: - last_tag = alpha_tag - logger.debug("Using %s to add the new changelogs to.", last_tag.tag) - last_changelogs = self.repo.git.show(f"{last_tag.tag}:{path}") + # Take HEAD to reset any changes we might already have from a + # previous run. + last_changelogs = self.repo.git.show(f"HEAD:{path}") with (self.base_path / path).open("w") as f: f.write(changelogs + "\n" + last_changelogs + "\n")

@@ -705,8 +698,6 @@ if __name__ == "__main__": default=Path(__file__).parent.parent, help="Path to a tor-browser-build.git clone", ) - parser.add_argument("--tor-browser", action="store_true") - parser.add_argument("--mullvad-browser", action="store_true") parser.add_argument( "--date", help="Release date and optionally time for changelog purposes. " @@ -747,12 +738,7 @@ if __name__ == "__main__": ) logger.addHandler(ch)

- tbb = bool(args.tor_browser) - mb = bool(args.mullvad_browser) kwargs = {} - if tbb or mb: - kwargs["tor_browser"] = tbb - kwargs["mullvad_browser"] = mb if args.date: try: kwargs["changelog_date"] = datetime.fromisoformat(args.date)

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/compare/8...