GitLab

at 2024-07-30T11:10:15+02:00

Bug 41195: Simplify the go update in relprep.py.

Since we do not support Windows 7 anymore, we can use only one version
of Go, and simplify the function that looks for updates in relprep.py.

Bug 41200: Remove the tools for allowed_addons.json.

For tor-browser#42618 and tor-browser#42619, we stopped using the
allowed_addons.json, but initially we kept the tools to update it.
However, the file creates some confusion, because it is used only for
Andorid, but should be updated also when doing desktop-only releases
when they update NoScript, or Android nightly builds might fail.
So, since as a matter of fact we do not use that file anymore, we
decided to stop updating it and remove the related tools.

-projects/browser/allowed_addons.json -diff
     - [ ] `fenix_version` : update to match alpha `firefox-android` build tag

     - [ ] `browser_branch` : update to match alpha `firefox-android` build tag

     - [ ] `browser_build` : update to match alpha `firefox-android` build tag

-  - [ ] Update allowed_addons.json by running (from `tor-browser-build` root):

-    - `./tools/fetch_allowed_addons.py > projects/browser/allowed_addons.json`

 - [ ] Update `projects/translation/config`:

   - [ ] run `make list_translation_updates-alpha` to get updated hashes

   - [ ] `steps/base-browser/git_hash` : update with `HEAD` commit of project's `base-browser` branch

     - [ ] `browser_branch` : update to match stable `firefox-android` build tag

     - [ ] `browser_build` : update to match stable `firefox-android` build tag

   variant: Beta

-  - [ ] Update allowed_addons.json by running (from `tor-browser-build` root):

-    - `./tools/fetch_allowed_addons.py > projects/browser/allowed_addons.json`

 - [ ] Update `projects/translation/config`:

   - [ ] run `make list_translation_updates-release` to get updated hashes

   - [ ] `steps/base-browser/git_hash` : update with `HEAD` commit of project's `base-browser` branch

       }) %]

 popd

-

-[% IF c("var/verify_allowed_addons") %]

-  # Check that allowed_addons.json contains the right versions of our bundled extension(s).

-  # If so, replace the default allowed_addons.json by ours in the apk assets folder.

-  $rootdir/verify_allowed_addons.py "$rootdir/allowed_addons.json" "$noscript_path"

-[% END %]

-

-mv $rootdir/allowed_addons.json $assets_dir/allowed_addons.json

-

 mkdir apk

 pushd apk

 7zz x "$apk"

   android:

     build: '[% INCLUDE build.android %]'

     var:

-      verify_allowed_addons: 1

       arch_deps:

         - 7zip

         - openjdk-17-jdk-headless

     enable: '[% c("var/namecoin") %]'

   - filename: namecoin.patch

     enable: '[% c("var/namecoin") %]'

-  - filename: allowed_addons.json

-    enable: '[% c("var/android") %]'

-  - filename: verify_allowed_addons.py

-    enable: '[% c("var/android") && c("var/verify_allowed_addons") %]'

   - project: manual

     name: manual

     enable: '[% ! c("var/android") && c("var/tor-browser") %]'

-#!/usr/bin/env python3

-

-import json

-import sys

-import hashlib

-import zipfile

-

-def find_addon(addons, addon_id):

-  results = addons['results']

-  for x in results:

-    addon = x['addon']

-    if addon['guid'] == addon_id:

-      return addon

-  sys.exit("Error: cannot find addon " + addon_id)

-

-def verify_extension_version(addons, addon_id, version):

-  addon = find_addon(addons, addon_id)

-  expected_version = addon['current_version']['version']

-  if version != expected_version:

-    sys.exit("Error: version " + version + " != " + expected_version)

-

-def verify_extension_hash(addons, addon_id, hash):

-  addon = find_addon(addons, addon_id)

-  expected_hash = addon["current_version"]["files"][0]["hash"]

-  if hash != expected_hash:

-    sys.exit("Error: hash " + hash + " != " + expected_hash)

-

-def read_extension_manifest(path):

-  return json.loads(zipfile.ZipFile(path, 'r').read('manifest.json'))

-

-def main(argv):

-  allowed_addons_path = argv[0]

-  noscript_path = argv[1]

-

-  addons = None

-  with open(allowed_addons_path, 'r') as file:

-    addons = json.loads(file.read())

-

-  noscript_hash = None

-  with open(noscript_path, 'rb') as file:

-    noscript_hash = "sha256:" + hashlib.sha256(file.read()).hexdigest()

-

-  noscript_version = read_extension_manifest(noscript_path)["version"]

-

-  verify_extension_hash(addons, '{73a6fe31-595d-460b-a920-fcc0f8843232}', noscript_hash)

-  verify_extension_version(addons, '{73a6fe31-595d-460b-a920-fcc0f8843232}', noscript_version)

-

-if __name__ == "__main__":

-   main(sys.argv[1:])
-#!/usr/bin/env python3

-

-import urllib.request

-import json

-import base64

-import sys

-

-NOSCRIPT = "{73a6fe31-595d-460b-a920-fcc0f8843232}"

-

-

-def fetch(x):

-    with urllib.request.urlopen(x) as response:

-        return response.read()

-

-

-def find_addon(addons, addon_id):

-    results = addons["results"]

-    for x in results:

-        addon = x["addon"]

-        if addon["guid"] == addon_id:

-            return addon

-

-

-def fetch_and_embed_icons(addons):

-    results = addons["results"]

-    for x in results:

-        addon = x["addon"]

-        icon_data = fetch(addon["icon_url"])

-        addon["icon_url"] = "data:image/png;base64," + str(

-            base64.b64encode(icon_data), "utf8"

-        )

-

-

-def fetch_allowed_addons(amo_collection=None):

-    if amo_collection is None:

-        amo_collection = "83a9cccfe6e24a34bd7b155ff9ee32"

-    url = f"https://services.addons.mozilla.org/api/v4/accounts/account/mozilla/collections/{amo_collection}/addons/"

-    data = json.loads(fetch(url))

-    fetch_and_embed_icons(data)

-    data["results"].sort(key=lambda x: x["addon"]["guid"])

-    return data

-

-

-def main(argv):

-    data = fetch_allowed_addons(argv[0] if len(argv) > 1 else None)

-    # Check that NoScript is present

-    if find_addon(data, NOSCRIPT) is None:

-        sys.exit("Error: cannot find NoScript.")

-    print(json.dumps(data, indent=2, ensure_ascii=False))

-

-

-if __name__ == "__main__":

-    main(sys.argv[1:])
 import configparser

 from datetime import datetime, timezone

 from hashlib import sha256

-import json

 import locale

 import logging

 from pathlib import Path

 import requests

 import ruamel.yaml

-from fetch_allowed_addons import NOSCRIPT, fetch_allowed_addons, find_addon

 import fetch_changelogs

 from update_manual import update_manual

         logger.info("Updating addons")

         config = self.load_config("browser")

-        amo_data = fetch_allowed_addons()

-        logger.debug("Fetched AMO data")

-        if self.android:

-            with (

-                self.base_path / "projects/browser/allowed_addons.json"

-            ).open("w") as f:

-                json.dump(amo_data, f, indent=2)

-

-        noscript = find_addon(amo_data, NOSCRIPT)

         logger.debug("Updating NoScript")

-        self.update_addon_amo(config, "noscript", noscript)

+        self.update_addon_amo(

+            config, "noscript", "{73a6fe31-595d-460b-a920-fcc0f8843232}"

+        )

         if self.mullvad_browser:

             logger.debug("Updating uBlock Origin")

-            ublock = find_addon(amo_data, "uBlock0@raymondhill.net")

-            self.update_addon_amo(config, "ublock-origin", ublock)

+            self.update_addon_amo(

+                config, "ublock-origin", "uBlock0@raymondhill.net"

+            )

             logger.debug("Updating the Mullvad Browser extension")

             self.update_mullvad_addon(config)

         self.save_config("browser", config)

-    def update_addon_amo(self, config, name, addon):

-        addon = addon["current_version"]["files"][0]

+    def update_addon_amo(self, config, name, addon_id):

+        r = requests.get(

+            f"https://services.addons.mozilla.org/api/v4/addons/addon/{addon_id}"

+        )

+        r.raise_for_status()

+        amo_data = r.json()

+        addon = amo_data["current_version"]["files"][0]

         assert addon["hash"].startswith("sha256:")

         addon_input = self.find_input(config, name)

         addon_input["URL"] = addon["url"]

                 major = major[2:]

             return major

+        logger.info("Updating Go")

         config = self.load_config("go")

-        # TODO: When Windows 7 goes EOL use config["version"]

-        major = get_major(config["var"]["go_1_21"])

+        major = get_major(config["version"])

         r = requests.get("https://go.dev/dl/?mode=json")

         r.raise_for_status()

         if not data:

             raise KeyError("Could not find information for our Go series.")

         # Skip the "go" prefix in the version.

-        config["var"]["go_1_21"] = data["version"][2:]

+        config["version"] = data["version"][2:]

         sha256sum = ""

         for f in data["files"]:

                 break

         if not sha256sum:

             raise KeyError("Go source package not found.")

-        updated_hash = False

-        for input_ in config["input_files"]:

-            if "URL" in input_ and "var/go_1_21" in input_["URL"]:

-                input_["sha256sum"] = sha256sum

-                updated_hash = True

-                break

-        if not updated_hash:

-            raise KeyError("Could not find a matching entry in input_files.")

-

+        self.find_input(config, "go")["sha256sum"] = sha256sum

         self.save_config("go", config)

     def update_manual(self):

             # Sometimes this might be incorrect for alphas, but let's

             # keep it for now.

             kwargs["firefox"] += "esr"

-        self.check_update_simple(kwargs, prev_tag, "tor")

-        self.check_update_simple(kwargs, prev_tag, "openssl")

-        self.check_update_simple(kwargs, prev_tag, "zlib")

-        self.check_update_simple(kwargs, prev_tag, "zstd")

-        try:

-            self.check_update(kwargs, prev_tag, "go", ["var", "go_1_21"])

-        except KeyError as e:

-            logger.warning(

-                "Go: var/go_1_21 not found, marking Go as not updated.",

-                exc_info=e,

-            )

-            pass

+        self.check_update(kwargs, prev_tag, "tor")

+        self.check_update(kwargs, prev_tag, "openssl")

+        self.check_update(kwargs, prev_tag, "zlib")

+        self.check_update(kwargs, prev_tag, "zstd")

+        self.check_update(kwargs, prev_tag, "go")

         self.check_update_extensions(kwargs, prev_tag)

         logger.debug("Changelog arguments for %s: %s", tag_prefix, kwargs)

         cb = fetch_changelogs.ChangelogBuilder(

         with (self.base_path / path).open("w") as f:

             f.write(changelogs + "\n" + last_changelogs + "\n")

-    def check_update(self, updates, prev_tag, project, key):

+    def check_update(self, updates, prev_tag, project, key=["version"]):

         old_val = self.load_old_config(prev_tag.tag, project)

         new_val = self.load_config(project)

         for k in key:

         if old_val != new_val:

             updates[project] = new_val

-    def check_update_simple(self, updates, prev_tag, project):

-        self.check_update(updates, prev_tag, project, ["version"])

-

     def check_update_extensions(self, updates, prev_tag):

         old_config = self.load_old_config(prev_tag, "browser")

         new_config = self.load_config("browser")

Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build

Commits:

9 changed files:

Changes: