Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build

Commits: 1fb6aaae by Pier Angelo Vendrame at 2024-07-30T11:10:15+02:00 Bug 41195: Simplify the go update in relprep.py.

Since we do not support Windows 7 anymore, we can use only one version of Go, and simplify the function that looks for updates in relprep.py.

- - - - - f51f78b2 by Pier Angelo Vendrame at 2024-07-30T11:10:28+02:00 Bug 41200: Remove the tools for allowed_addons.json.

For tor-browser#42618 and tor-browser#42619, we stopped using the allowed_addons.json, but initially we kept the tools to update it. However, the file creates some confusion, because it is used only for Andorid, but should be updated also when doing desktop-only releases when they update NoScript, or Android nightly builds might fail. So, since as a matter of fact we do not use that file anymore, we decided to stop updating it and remove the related tools.

- - - - -

9 changed files:

- − .gitattributes - .gitlab/issue_templates/Release Prep - Tor Browser Alpha.md - .gitlab/issue_templates/Release Prep - Tor Browser Stable.md - − projects/browser/allowed_addons.json - projects/browser/build.android - projects/browser/config - − projects/browser/verify_allowed_addons.py - − tools/fetch_allowed_addons.py - tools/relprep.py

Changes:

===================================== .gitattributes deleted ===================================== @@ -1 +0,0 @@ -projects/browser/allowed_addons.json -diff

===================================== .gitlab/issue_templates/Release Prep - Tor Browser Alpha.md ===================================== @@ -59,8 +59,6 @@ Tor Browser Alpha (and Nightly) are on the `main` branch - [ ] `fenix_version` : update to match alpha `firefox-android` build tag - [ ] `browser_branch` : update to match alpha `firefox-android` build tag - [ ] `browser_build` : update to match alpha `firefox-android` build tag - - [ ] Update allowed_addons.json by running (from `tor-browser-build` root): - - `./tools/fetch_allowed_addons.py > projects/browser/allowed_addons.json` - [ ] Update `projects/translation/config`: - [ ] run `make list_translation_updates-alpha` to get updated hashes - [ ] `steps/base-browser/git_hash` : update with `HEAD` commit of project's `base-browser` branch

===================================== .gitlab/issue_templates/Release Prep - Tor Browser Stable.md ===================================== @@ -60,8 +60,6 @@ Tor Browser Stable lives in the various `maint-$(TOR_BROWSER_MAJOR).$(TOR_BROWSE - [ ] `browser_branch` : update to match stable `firefox-android` build tag - [ ] `browser_build` : update to match stable `firefox-android` build tag variant: Beta - - [ ] Update allowed_addons.json by running (from `tor-browser-build` root): - - `./tools/fetch_allowed_addons.py > projects/browser/allowed_addons.json` - [ ] Update `projects/translation/config`: - [ ] run `make list_translation_updates-release` to get updated hashes - [ ] `steps/base-browser/git_hash` : update with `HEAD` commit of project's `base-browser` branch

===================================== projects/browser/allowed_addons.json deleted ===================================== The diff for this file was not included because it is too large.

===================================== projects/browser/build.android ===================================== @@ -39,15 +39,6 @@ unzip ../omni.ja }) %] popd

- -[% IF c("var/verify_allowed_addons") %] - # Check that allowed_addons.json contains the right versions of our bundled extension(s). - # If so, replace the default allowed_addons.json by ours in the apk assets folder. - $rootdir/verify_allowed_addons.py "$rootdir/allowed_addons.json" "$noscript_path" -[% END %] - -mv $rootdir/allowed_addons.json $assets_dir/allowed_addons.json - mkdir apk pushd apk 7zz x "$apk"

===================================== projects/browser/config ===================================== @@ -49,7 +49,6 @@ targets: android: build: '[% INCLUDE build.android %]' var: - verify_allowed_addons: 1 arch_deps: - 7zip - openjdk-17-jdk-headless @@ -160,10 +159,6 @@ input_files: enable: '[% c("var/namecoin") %]' - filename: namecoin.patch enable: '[% c("var/namecoin") %]' - - filename: allowed_addons.json - enable: '[% c("var/android") %]' - - filename: verify_allowed_addons.py - enable: '[% c("var/android") && c("var/verify_allowed_addons") %]' - project: manual name: manual enable: '[% ! c("var/android") && c("var/tor-browser") %]'

===================================== projects/browser/verify_allowed_addons.py deleted ===================================== @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys -import hashlib -import zipfile - -def find_addon(addons, addon_id): - results = addons['results'] - for x in results: - addon = x['addon'] - if addon['guid'] == addon_id: - return addon - sys.exit("Error: cannot find addon " + addon_id) - -def verify_extension_version(addons, addon_id, version): - addon = find_addon(addons, addon_id) - expected_version = addon['current_version']['version'] - if version != expected_version: - sys.exit("Error: version " + version + " != " + expected_version) - -def verify_extension_hash(addons, addon_id, hash): - addon = find_addon(addons, addon_id) - expected_hash = addon["current_version"]["files"][0]["hash"] - if hash != expected_hash: - sys.exit("Error: hash " + hash + " != " + expected_hash) - -def read_extension_manifest(path): - return json.loads(zipfile.ZipFile(path, 'r').read('manifest.json')) - -def main(argv): - allowed_addons_path = argv[0] - noscript_path = argv[1] - - addons = None - with open(allowed_addons_path, 'r') as file: - addons = json.loads(file.read()) - - noscript_hash = None - with open(noscript_path, 'rb') as file: - noscript_hash = "sha256:" + hashlib.sha256(file.read()).hexdigest() - - noscript_version = read_extension_manifest(noscript_path)["version"] - - verify_extension_hash(addons, '{73a6fe31-595d-460b-a920-fcc0f8843232}', noscript_hash) - verify_extension_version(addons, '{73a6fe31-595d-460b-a920-fcc0f8843232}', noscript_version) - -if __name__ == "__main__": - main(sys.argv[1:])

===================================== tools/fetch_allowed_addons.py deleted ===================================== @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -import urllib.request -import json -import base64 -import sys - -NOSCRIPT = "{73a6fe31-595d-460b-a920-fcc0f8843232}" - - -def fetch(x): - with urllib.request.urlopen(x) as response: - return response.read() - - -def find_addon(addons, addon_id): - results = addons["results"] - for x in results: - addon = x["addon"] - if addon["guid"] == addon_id: - return addon - - -def fetch_and_embed_icons(addons): - results = addons["results"] - for x in results: - addon = x["addon"] - icon_data = fetch(addon["icon_url"]) - addon["icon_url"] = "data:image/png;base64," + str( - base64.b64encode(icon_data), "utf8" - ) - - -def fetch_allowed_addons(amo_collection=None): - if amo_collection is None: - amo_collection = "83a9cccfe6e24a34bd7b155ff9ee32" - url = f"https://services.addons.mozilla.org/api/v4/accounts/account/mozilla/collecti..." - data = json.loads(fetch(url)) - fetch_and_embed_icons(data) - data["results"].sort(key=lambda x: x["addon"]["guid"]) - return data - - -def main(argv): - data = fetch_allowed_addons(argv[0] if len(argv) > 1 else None) - # Check that NoScript is present - if find_addon(data, NOSCRIPT) is None: - sys.exit("Error: cannot find NoScript.") - print(json.dumps(data, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main(sys.argv[1:])

===================================== tools/relprep.py ===================================== @@ -4,7 +4,6 @@ from collections import namedtuple import configparser from datetime import datetime, timezone from hashlib import sha256 -import json import locale import logging from pathlib import Path @@ -16,7 +15,6 @@ from git import Repo import requests import ruamel.yaml

-from fetch_allowed_addons import NOSCRIPT, fetch_allowed_addons, find_addon import fetch_changelogs from update_manual import update_manual

@@ -303,28 +301,27 @@ class ReleasePreparation: logger.info("Updating addons") config = self.load_config("browser")

- amo_data = fetch_allowed_addons() - logger.debug("Fetched AMO data") - if self.android: - with ( - self.base_path / "projects/browser/allowed_addons.json" - ).open("w") as f: - json.dump(amo_data, f, indent=2) - - noscript = find_addon(amo_data, NOSCRIPT) logger.debug("Updating NoScript") - self.update_addon_amo(config, "noscript", noscript) + self.update_addon_amo( + config, "noscript", "{73a6fe31-595d-460b-a920-fcc0f8843232}" + ) if self.mullvad_browser: logger.debug("Updating uBlock Origin") - ublock = find_addon(amo_data, "uBlock0@raymondhill.net") - self.update_addon_amo(config, "ublock-origin", ublock) + self.update_addon_amo( + config, "ublock-origin", "uBlock0@raymondhill.net" + ) logger.debug("Updating the Mullvad Browser extension") self.update_mullvad_addon(config)

self.save_config("browser", config)

- def update_addon_amo(self, config, name, addon): - addon = addon["current_version"]["files"][0] + def update_addon_amo(self, config, name, addon_id): + r = requests.get( + f"https://services.addons.mozilla.org/api/v4/addons/addon/%7Baddon_id%7D" + ) + r.raise_for_status() + amo_data = r.json() + addon = amo_data["current_version"]["files"][0] assert addon["hash"].startswith("sha256:") addon_input = self.find_input(config, name) addon_input["URL"] = addon["url"] @@ -448,9 +445,9 @@ class ReleasePreparation: major = major[2:] return major

+ logger.info("Updating Go") config = self.load_config("go") - # TODO: When Windows 7 goes EOL use config["version"] - major = get_major(config["var"]["go_1_21"]) + major = get_major(config["version"])

r = requests.get("https://go.dev/dl/?mode=json") r.raise_for_status() @@ -463,7 +460,7 @@ class ReleasePreparation: if not data: raise KeyError("Could not find information for our Go series.") # Skip the "go" prefix in the version. - config["var"]["go_1_21"] = data["version"][2:] + config["version"] = data["version"][2:]

sha256sum = "" for f in data["files"]: @@ -472,15 +469,7 @@ class ReleasePreparation: break if not sha256sum: raise KeyError("Go source package not found.") - updated_hash = False - for input_ in config["input_files"]: - if "URL" in input_ and "var/go_1_21" in input_["URL"]: - input_["sha256sum"] = sha256sum - updated_hash = True - break - if not updated_hash: - raise KeyError("Could not find a matching entry in input_files.") - + self.find_input(config, "go")["sha256sum"] = sha256sum self.save_config("go", config)

def update_manual(self): @@ -556,18 +545,11 @@ class ReleasePreparation: # Sometimes this might be incorrect for alphas, but let's # keep it for now. kwargs["firefox"] += "esr" - self.check_update_simple(kwargs, prev_tag, "tor") - self.check_update_simple(kwargs, prev_tag, "openssl") - self.check_update_simple(kwargs, prev_tag, "zlib") - self.check_update_simple(kwargs, prev_tag, "zstd") - try: - self.check_update(kwargs, prev_tag, "go", ["var", "go_1_21"]) - except KeyError as e: - logger.warning( - "Go: var/go_1_21 not found, marking Go as not updated.", - exc_info=e, - ) - pass + self.check_update(kwargs, prev_tag, "tor") + self.check_update(kwargs, prev_tag, "openssl") + self.check_update(kwargs, prev_tag, "zlib") + self.check_update(kwargs, prev_tag, "zstd") + self.check_update(kwargs, prev_tag, "go") self.check_update_extensions(kwargs, prev_tag) logger.debug("Changelog arguments for %s: %s", tag_prefix, kwargs) cb = fetch_changelogs.ChangelogBuilder( @@ -587,7 +569,7 @@ class ReleasePreparation: with (self.base_path / path).open("w") as f: f.write(changelogs + "\n" + last_changelogs + "\n")

- def check_update(self, updates, prev_tag, project, key): + def check_update(self, updates, prev_tag, project, key=["version"]): old_val = self.load_old_config(prev_tag.tag, project) new_val = self.load_config(project) for k in key: @@ -596,9 +578,6 @@ class ReleasePreparation: if old_val != new_val: updates[project] = new_val

- def check_update_simple(self, updates, prev_tag, project): - self.check_update(updates, prev_tag, project, ["version"]) - def check_update_extensions(self, updates, prev_tag): old_config = self.load_old_config(prev_tag, "browser") new_config = self.load_config("browser")

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/compare/9...