Pier Angelo Vendrame pushed to branch tor-browser-115.8.0esr-13.5-1 at The Tor Project / Applications / Tor Browser

Commits: 708a2c47 by Henry Wilkes at 2024-03-07T10:37:42+00:00 fixup! Tor Browser localization migration scripts.

Bug 42305: Move localization scripts into new folder.

- - - - - ad71ddb2 by Henry Wilkes at 2024-03-07T10:37:42+00:00 Bug 42305: Add script to combine translation files across versions.

- - - - - f13a9d22 by Henry Wilkes at 2024-03-07T10:37:42+00:00 Add CI for Tor Browser

- - - - -

13 changed files:

- + .gitlab-ci.yml - + tools/torbrowser/l10n/combine-translation-versions.py - + tools/torbrowser/l10n/combine/__init__.py - + tools/torbrowser/l10n/combine/combine.py - + tools/torbrowser/l10n/combine/tests/README - tools/torbrowser/l10n_migrations/__init__.py → tools/torbrowser/l10n/combine/tests/__init__.py - + tools/torbrowser/l10n/combine/tests/test_android.py - + tools/torbrowser/l10n/combine/tests/test_dtd.py - + tools/torbrowser/l10n/combine/tests/test_fluent.py - + tools/torbrowser/l10n/combine/tests/test_properties.py - tools/torbrowser/migrate_l10n.py → tools/torbrowser/l10n/migrate.py - + tools/torbrowser/l10n/migrations/__init__.py - tools/torbrowser/l10n_migrations/bug-41333-new-about-tor.py → tools/torbrowser/l10n/migrations/bug-41333-new-about-tor.py

Changes:

===================================== .gitlab-ci.yml ===================================== @@ -0,0 +1,69 @@ +stages: + - update-translations + +.update-translation-base: + stage: update-translations + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - "**/*.ftl" + - "**/*.properties" + - "**/*.dtd" + - if: $FORCE_UPDATE_TRANSLATIONS == "true" + variables: + TOR_BROWSER_COMBINED_FILES_JSON: "combined-translation-files.json" + + +combine-en-US-translations: + extends: .update-translation-base + image: python + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + TRANSLATION_FILES: ' + tor-browser:tor-browser.ftl + tor-browser:aboutDialog.dtd + tor-browser:aboutTBUpdate.dtd + tor-browser:aboutTor.dtd + tor-browser:torbutton.dtd + tor-browser:browserOnboarding.properties + tor-browser:cryptoSafetyPrompt.properties + tor-browser:onboarding.properties + tor-browser:onionLocation.properties + tor-browser:rulesets.properties + tor-browser:settings.properties + tor-browser:torbutton.properties + tor-browser:torConnect.properties + tor-browser:torlauncher.properties + base-browser:base-browser.ftl + base-browser:newIdentity.properties + base-browser:securityLevel.properties + ' + cache: + paths: + - .cache/pip + # Artifact is for translation project job + artifacts: + paths: + - "$TOR_BROWSER_COMBINED_FILES_JSON" + expire_in: "60 min" + reports: + dotenv: job_id.env + # Don't load artifacts for this job. + dependencies: [] + script: + # Save this CI_JOB_ID to the dotenv file to be used in the variables for the + # push-en-US-translations job. + - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env + - pip install compare_locales + - python ./tools/torbrowser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$TOR_BROWSER_COMBINED_FILES_JSON" + +push-en-US-translations: + extends: .update-translation-base + needs: + - job: combine-en-US-translations + variables: + TOR_BROWSER_COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${TOR_BROWSER_COMBINED_FILES_JSON}" + trigger: + strategy: depend + project: tor-browser-translation-bot/translation + branch: tor-browser-ci

===================================== tools/torbrowser/l10n/combine-translation-versions.py ===================================== @@ -0,0 +1,208 @@ +import argparse +import json +import logging +import os +import re +import subprocess + +from combine import combine_files + +arg_parser = argparse.ArgumentParser( + description="Combine a translation file across two different versions" +) + +arg_parser.add_argument( + "current_branch", metavar="<current-branch>", help="branch for the newest version" +) +arg_parser.add_argument( + "filenames", metavar="<filenames>", help="name of the translation files" +) +arg_parser.add_argument("outname", metavar="<json>", help="name of the json output") + +args = arg_parser.parse_args() + +logging.basicConfig() +logger = logging.getLogger("combine-translation-versions") +logger.setLevel(logging.INFO) + + +def in_pink(msg: str) -> str: + """Present a message as pink in the terminal output. + + :param msg: The message to wrap in pink. + :returns: The message to print to terminal. + """ + # Pink and bold. + return f"\x1b[1;38;5;212m{msg}\x1b[0m" + + +def git_run(git_args: list[str]) -> None: + """Run a git command. + + :param git_args: The arguments that should follow "git". + """ + # Add some text to give context to git's stderr appearing in log. + logger.info("Running: " + in_pink("git " + " ".join(git_args))) + subprocess.run(["git", *git_args], check=True) + + +def git_text(git_args: list[str]) -> str: + """Get the text output for a git command. + + :param git_args: The arguments that should follow "git". + :returns: The stdout of the command. + """ + logger.info("Running: " + in_pink("git " + " ".join(git_args))) + return subprocess.run( + ["git", *git_args], text=True, check=True, stdout=subprocess.PIPE + ).stdout + + +def git_lines(git_args: list[str]) -> list[str]: + """Get the lines from a git command. + + :param git_args: The arguments that should follow "git". + :returns: The non-empty lines from stdout of the command. + """ + return [line for line in git_text(git_args).split("\n") if line] + + +def git_file_paths(git_ref: str) -> list[str]: + """Get the full list of file paths found under the given tree. + + :param git_ref: The git reference for the tree to search. + :returns: The found file paths. + """ + return git_lines(["ls-tree", "-r", "--format=%(path)", git_ref]) + + +def matching_path(search_paths: list[str], filename: str) -> str | None: + """Get the matching file path with the given filename, if it exists. + + :param search_paths: The file paths to search through. + :param filename: The file name to match. + :returns: The unique file path with the matching name, or None if no such + match was found. + :throws Exception: If multiple paths shared the same file name. + """ + matching = [path for path in search_paths if os.path.basename(path) == filename] + if not matching: + return None + if len(matching) > 1: + raise Exception("Multiple occurrences of {filename}") + return matching[0] + + +def git_file_content(git_ref: str, path: str | None) -> str | None: + """Get the file content of the specified git blob object. + + :param git_ref: The reference for the tree to find the file under. + :param path: The file path for the object, or None if there is no path. + :returns: The file content, or None if no path was given. + """ + if path is None: + return None + return git_text(["cat-file", "blob", f"{git_ref}:{path}"]) + + +def get_stable_branch(branch_prefix: str) -> str: + """Find the most recent stable branch in the origin repository. + + :param branch_prefix: The prefix that the stable branch should have. + :returns: The branch name. + """ + tag_glob = f"{branch_prefix}-*-build1" + # To speed up, only fetch the tags without blobs. + git_run( + ["fetch", "--depth=1", "--filter=object:type=tag", "origin", "tag", tag_glob] + ) + # Get most recent stable tag. + for build_tag, annotation in ( + line.split(" ", 1) + for line in git_lines(["tag", "-n1", "--list", tag_glob, "--sort=-taggerdate"]) + ): + if "stable" in annotation: + # Branch name is the same as the tag, minus "-build1". + return re.sub(r"-build1$", "", build_tag) + raise Exception("No stable build1 tag found") + + +def get_version_from_branch_name(branch_name: str) -> tuple[str, float]: + """Get the branch prefix and version from its name. + + :param branch_name: The branch to extract from. + :returns: The branch prefix and its version number. + """ + version_match = re.match( + r"([a-z-]+)-[^-]*-([0-9]+.[05])-", + branch_name, + ) + + if not version_match: + raise ValueError(f"Unable to parse the version from the branch {branch_name}") + + return (version_match.group(1), float(version_match.group(2))) + + +branch_prefix, current_version = get_version_from_branch_name(args.current_branch) + +stable_branch = get_stable_branch(branch_prefix) +_, stable_version = get_version_from_branch_name(stable_branch) + +if stable_version > current_version or stable_version < current_version - 0.5: + raise Exception( + f"Version of stable branch {stable_branch} is not within 0.5 of the " + f"current branch {args.current_branch}" + ) + +# Minimal fetch of stable_branch. +# Individual file blobs will be downloaded as needed. +git_run(["fetch", "--depth=1", "--filter=blob:none", "origin", stable_branch]) + +current_file_paths = git_file_paths("HEAD") +old_file_paths = git_file_paths(f"origin/{stable_branch}") + +ci_commit = os.environ.get("CI_COMMIT_SHA", "") +ci_url_base = os.environ.get("CI_PROJECT_URL", "") + +json_data = { + "commit": ci_commit, + "commit-url": f"{ci_url_base}/-/commit/{ci_commit}" + if (ci_commit and ci_url_base) + else "", + "project-path": os.environ.get("CI_PROJECT_PATH", ""), + "current-branch": args.current_branch, + "stable-branch": stable_branch, + "files": [], +} + +for translation_branch, name in ( + part.strip().split(":", 1) for part in args.filenames.split(" ") if part.strip() +): + current_path = matching_path(current_file_paths, name) + old_path = matching_path(old_file_paths, name) + + if current_path is None and old_path is None: + # No file in either branch. + logger.warning(f"{name} does not exist in either the current or stable branch") + elif current_path is None: + logger.warning(f"{name} deleted in the current branch") + elif old_path is None: + logger.warning(f"{name} does not exist in the stable branch") + + content = combine_files( + name, + git_file_content("HEAD", current_path), + git_file_content(f"origin/{stable_branch}", old_path), + f"Will be unused in Tor Browser {current_version}!", + ) + json_data["files"].append( + { + "name": name, + "branch": translation_branch, + "content": content, + } + ) + +with open(args.outname, "w") as file: + json.dump(json_data, file)

===================================== tools/torbrowser/l10n/combine/__init__.py ===================================== @@ -0,0 +1,3 @@ +# flake8: noqa + +from .combine import combine_files

===================================== tools/torbrowser/l10n/combine/combine.py ===================================== @@ -0,0 +1,181 @@ +import re +from typing import TYPE_CHECKING, Any + +from compare_locales.parser import getParser +from compare_locales.parser.android import AndroidEntity, DocumentWrapper +from compare_locales.parser.base import Comment, Entity, Junk, Whitespace +from compare_locales.parser.dtd import DTDEntity +from compare_locales.parser.fluent import FluentComment, FluentEntity +from compare_locales.parser.properties import PropertiesEntity + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def combine_files( + filename: str, + new_content: str | None, + old_content: str | None, + comment_prefix: str, +) -> str | None: + """Combine two translation files into one to include all strings from both. + The new content is presented first, and any strings only found in the old + content are placed at the end with an additional comment. + + :param filename: The filename for the file, determines the format. + :param new_content: The new content for the file, or None if it has been + deleted. + :param old_content: The old content for the file, or None if it did not + exist before. + :comment_prefix: A comment to include for any strings that are only found in + the old content. This will be placed before any other comments for the + string. + + :returns: The combined content, or None if both given contents are None. + """ + if new_content is None and old_content is None: + return None + + # getParser from compare_locale returns the same instance for the same file + # extension. + parser = getParser(filename) + + is_android = filename.endswith(".xml") + if new_content is None: + if is_android: + # File was deleted, add some document parts. + content_start = ( + '<?xml version="1.0" encoding="utf-8" standalone="yes"?>\n<resources>\n' + ) + content_end = "</resources>\n" + else: + # Treat as an empty file. + content_start = "" + content_end = "" + existing_keys = [] + else: + parser.readUnicode(new_content) + + # Start with the same content as the current file. + # For android strings, we want to keep the final "</resources>" until after. + if is_android: + closing_match = re.match( + r"^(.*)(</resources>\s*)$", parser.ctx.contents, re.DOTALL + ) + if not closing_match: + raise ValueError("Missing a final </resources>") + content_start = closing_match.group(1) + content_end = closing_match.group(2) + else: + content_start = parser.ctx.contents + content_end = "" + existing_keys = [entry.key for entry in parser.walk(only_localizable=True)] + + # For Fluent, we want to prefix the strings using GroupComments. + # On weblate this will cause all the strings that fall under the GroupComment's + # scope to have the prefix added to their "notes". + # We set up an initial GroupComment for the first string we find. This will also + # end the scope of the last GroupComment in the new translation file. + # This will be replaced with a the next GroupComment when it is found. + fluent_group_comment_prefix = f"\n## {comment_prefix}\n" + fluent_group_comment: str | None = fluent_group_comment_prefix + + # For other formats, we want to keep all the comment lines that come directly + # before the string. + # In compare_locales.parser, only the comment line directly before an Entity + # counts as the pre_comment for that Entity. I.e. only this line will be + # included in Entity.all + # However, in weblate every comment line that comes before the Entity is + # included as a comment. So we also want to keep these additional comments to + # preserve them for weblate. + # We gather these extra comments in stacked_comments, and clear them whenever we + # reach an Entity or a blank line (Whitespace is more than "\n"). + stacked_comments: list[str] = [] + + additions: list[str] = [] + + entry_iter: Iterable[Any] = () + # If the file does not exist in the old branch, don't make any additions. + if old_content is not None: + parser.readUnicode(old_content) + entry_iter = parser.walk(only_localizable=False) + for entry in entry_iter: + if isinstance(entry, Junk): + raise ValueError(f"Unexpected Junk: {entry.all}") + if isinstance(entry, Whitespace): + # Clear stacked comments if more than one empty line. + if entry.all != "\n": + stacked_comments.clear() + continue + if isinstance(entry, Comment): + if isinstance(entry, FluentComment): + # Don't stack Fluent comments. + # Only the comments included in Entity.pre_comment count towards + # that Entity's comment. + if entry.all.startswith("##"): + # A Fluent GroupComment + if entry.all == "##": + # Empty GroupComment. Used to end the scope of a previous + # GroupComment. + # Replace this with our prefix comment. + fluent_group_comment = fluent_group_comment_prefix + else: + # Prefix the group comment. + fluent_group_comment = ( + f"{fluent_group_comment_prefix}{entry.all}\n" + ) + else: + stacked_comments.append(entry.all) + continue + if isinstance(entry, DocumentWrapper): + # Not needed. + continue + + if not isinstance(entry, Entity): + raise ValueError(f"Unexpected type: {entry.__class__.__name__}") + + if entry.key in existing_keys: + # Already included this string in the new translation file. + # Drop the gathered comments for this Entity. + stacked_comments.clear() + continue + + if isinstance(entry, FluentEntity): + if fluent_group_comment is not None: + # We have a found GroupComment which has not been included yet. + # All following Entity's will be under its scope, until the next + # GroupComment. + additions.append(fluent_group_comment) + # Added GroupComment, so don't need to add again. + fluent_group_comment = None + elif isinstance(entry, DTDEntity): + # Include our additional comment before we print the rest for this + # Entity. + additions.append(f"<!-- LOCALIZATION NOTE: {comment_prefix} -->") + elif isinstance(entry, PropertiesEntity): + additions.append(f"# {comment_prefix}") + elif isinstance(entry, AndroidEntity): + additions.append(f"<!-- {comment_prefix} -->") + else: + raise ValueError(f"Unexpected Entity type: {entry.__class__.__name__}") + + # Add any other comment lines that came directly before this Entity. + additions.extend(stacked_comments) + stacked_comments.clear() + additions.append(entry.all) + + content_middle = "" + + if additions: + # New line before and after the additions + additions.insert(0, "") + additions.append("") + if is_android: + content_middle = "\n ".join(additions) + else: + content_middle = "\n".join(additions) + + # Remove " " in otherwise blank lines. + content_middle = re.sub("^ +$", "", content_middle, flags=re.MULTILINE) + + return content_start + content_middle + content_end

===================================== tools/torbrowser/l10n/combine/tests/README ===================================== @@ -0,0 +1,2 @@ +python tests to be run with pytest. +Requires the compare-locales package.

===================================== tools/torbrowser/l10n_migrations/__init__.py → tools/torbrowser/l10n/combine/tests/__init__.py =====================================

===================================== tools/torbrowser/l10n/combine/tests/test_android.py ===================================== @@ -0,0 +1,330 @@ +import textwrap + +from combine import combine_files + + +def wrap_in_xml(content): + if content is None: + return None + # Allow for indents to make the tests more readable. + content = textwrap.dedent(content) + return f"""\ +<?xml version="1.0" encoding="utf-8" standalone="yes"?> +<resources> +{textwrap.indent(content, " ")}</resources> +""" + + +def assert_result(new_content, old_content, expect): + new_content = wrap_in_xml(new_content) + old_content = wrap_in_xml(old_content) + expect = wrap_in_xml(expect) + assert expect == combine_files( + "test_strings.xml", new_content, old_content, "REMOVED STRING" + ) + + +def test_combine_empty(): + assert_result(None, None, None) + + +def test_combine_new_file(): + # New file with no old content. + assert_result( + """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + """, + None, + """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + """, + ) + + +def test_combine_removed_file(): + # Entire file was removed. + assert_result( + None, + """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + """, + """\ + + <!-- REMOVED STRING --> + <string name="string_1">First</string> + <!-- REMOVED STRING --> + <string name="string_2">Second</string> + """, + ) + + +def test_no_change(): + content = """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + """ + assert_result(content, content, content) + + +def test_added_string(): + assert_result( + """\ + <string name="string_1">First</string> + <string name="string_new">NEW</string> + <string name="string_2">Second</string> + """, + """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + """, + """\ + <string name="string_1">First</string> + <string name="string_new">NEW</string> + <string name="string_2">Second</string> + """, + ) + + +def test_removed_string(): + assert_result( + """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + """, + """\ + <string name="string_1">First</string> + <string name="removed">REMOVED</string> + <string name="string_2">Second</string> + """, + """\ + <string name="string_1">First</string> + <string name="string_2">Second</string> + + <!-- REMOVED STRING --> + <string name="removed">REMOVED</string> + """, + ) + + +def test_removed_and_added(): + assert_result( + """\ + <string name="new_1">New string</string> + <string name="string_1">First</string> + <string name="string_2">Second</string> + <string name="new_2">New string 2</string> + """, + """\ + <string name="string_1">First</string> + <string name="removed_1">First removed</string> + <string name="removed_2">Second removed</string> + <string name="string_2">Second</string> + <string name="removed_3">Third removed</string> + """, + """\ + <string name="new_1">New string</string> + <string name="string_1">First</string> + <string name="string_2">Second</string> + <string name="new_2">New string 2</string> + + <!-- REMOVED STRING --> + <string name="removed_1">First removed</string> + <!-- REMOVED STRING --> + <string name="removed_2">Second removed</string> + <!-- REMOVED STRING --> + <string name="removed_3">Third removed</string> + """, + ) + + +def test_updated(): + # String content was updated. + assert_result( + """\ + <string name="changed_string">NEW</string> + """, + """\ + <string name="changed_string">OLD</string> + """, + """\ + <string name="changed_string">NEW</string> + """, + ) + + +def test_updated_comment(): + # String comment was updated. + assert_result( + """\ + <!-- NEW --> + <string name="changed_string">string</string> + """, + """\ + <!-- OLD --> + <string name="changed_string">string</string> + """, + """\ + <!-- NEW --> + <string name="changed_string">string</string> + """, + ) + # Comment added. + assert_result( + """\ + <!-- NEW --> + <string name="changed_string">string</string> + """, + """\ + <string name="changed_string">string</string> + """, + """\ + <!-- NEW --> + <string name="changed_string">string</string> + """, + ) + # Comment removed. + assert_result( + """\ + <string name="changed_string">string</string> + """, + """\ + <!-- OLD --> + <string name="changed_string">string</string> + """, + """\ + <string name="changed_string">string</string> + """, + ) + + # With file comments + assert_result( + """\ + <!-- NEW file comment --> + + <!-- NEW --> + <string name="changed_string">string</string> + """, + """\ + <!-- OLD file comment --> + + <!-- OLD --> + <string name="changed_string">string</string> + """, + """\ + <!-- NEW file comment --> + + <!-- NEW --> + <string name="changed_string">string</string> + """, + ) + + +def test_reordered(): + # String was re_ordered. + assert_result( + """\ + <string name="string_1">value</string> + <string name="moved_string">move</string> + """, + """\ + <string name="moved_string">move</string> + <string name="string_1">value</string> + """, + """\ + <string name="string_1">value</string> + <string name="moved_string">move</string> + """, + ) + + +def test_removed_string_with_comment(): + assert_result( + """\ + <!-- Comment for first. --> + <string name="string_1">First</string> + <string name="string_2">Second</string> + """, + """\ + <!-- Comment for first. --> + <string name="string_1">First</string> + <!-- Comment for removed. --> + <string name="removed">REMOVED</string> + <string name="string_2">Second</string> + """, + """\ + <!-- Comment for first. --> + <string name="string_1">First</string> + <string name="string_2">Second</string> + + <!-- REMOVED STRING --> + <!-- Comment for removed. --> + <string name="removed">REMOVED</string> + """, + ) + + # With file comments and multi-line. + # All comments prior to a removed string are moved with it, until another + # entity or blank line is reached. + assert_result( + """\ + <!-- First File comment --> + + <!-- Comment for first. --> + <!-- Comment 2 for first. --> + <string name="string_1">First</string> + + <!-- Second --> + <!-- File comment --> + + <string name="string_2">Second</string> + """, + """\ + <!-- First File comment --> + + <!-- Comment for first. --> + <!-- Comment 2 for first. --> + <string name="string_1">First</string> + <string name="removed_1">First removed</string> + <!-- Comment for second removed. --> + <string name="removed_2">Second removed</string> + + <!-- Removed file comment --> + + <!-- Comment 1 for third removed --> + <!-- Comment 2 for third removed --> + <string name="removed_3">Third removed</string> + + <!-- Second --> + <!-- File comment --> + + <string name="removed_4">Fourth removed</string> + <string name="string_2">Second</string> + """, + """\ + <!-- First File comment --> + + <!-- Comment for first. --> + <!-- Comment 2 for first. --> + <string name="string_1">First</string> + + <!-- Second --> + <!-- File comment --> + + <string name="string_2">Second</string> + + <!-- REMOVED STRING --> + <string name="removed_1">First removed</string> + <!-- REMOVED STRING --> + <!-- Comment for second removed. --> + <string name="removed_2">Second removed</string> + <!-- REMOVED STRING --> + <!-- Comment 1 for third removed --> + <!-- Comment 2 for third removed --> + <string name="removed_3">Third removed</string> + <!-- REMOVED STRING --> + <string name="removed_4">Fourth removed</string> + """, + )

===================================== tools/torbrowser/l10n/combine/tests/test_dtd.py ===================================== @@ -0,0 +1,325 @@ +import textwrap + +from combine import combine_files + + +def assert_result(new_content, old_content, expect): + # Allow for indents to make the tests more readable. + if new_content is not None: + new_content = textwrap.dedent(new_content) + if old_content is not None: + old_content = textwrap.dedent(old_content) + if expect is not None: + expect = textwrap.dedent(expect) + assert expect == combine_files( + "test.dtd", new_content, old_content, "REMOVED STRING" + ) + + +def test_combine_empty(): + assert_result(None, None, None) + + +def test_combine_new_file(): + # New file with no old content. + assert_result( + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """, + None, + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """, + ) + + +def test_combine_removed_file(): + # Entire file was removed. + assert_result( + None, + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """, + """\ + + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY string.1 "First"> + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY string.2 "Second"> + """, + ) + + +def test_no_change(): + content = """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """ + assert_result(content, content, content) + + +def test_added_string(): + assert_result( + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.new "NEW"> + <!ENTITY string.2 "Second"> + """, + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """, + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.new "NEW"> + <!ENTITY string.2 "Second"> + """, + ) + + +def test_removed_string(): + assert_result( + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """, + """\ + <!ENTITY string.1 "First"> + <!ENTITY removed "REMOVED"> + <!ENTITY string.2 "Second"> + """, + """\ + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY removed "REMOVED"> + """, + ) + + +def test_removed_and_added(): + assert_result( + """\ + <!ENTITY new.1 "New string"> + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + <!ENTITY new.2 "New string 2"> + """, + """\ + <!ENTITY string.1 "First"> + <!ENTITY removed.1 "First removed"> + <!ENTITY removed.2 "Second removed"> + <!ENTITY string.2 "Second"> + <!ENTITY removed.3 "Third removed"> + """, + """\ + <!ENTITY new.1 "New string"> + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + <!ENTITY new.2 "New string 2"> + + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY removed.1 "First removed"> + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY removed.2 "Second removed"> + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY removed.3 "Third removed"> + """, + ) + + +def test_updated(): + # String content was updated. + assert_result( + """\ + <!ENTITY changed.string "NEW"> + """, + """\ + <!ENTITY changed.string "OLD"> + """, + """\ + <!ENTITY changed.string "NEW"> + """, + ) + + +def test_updated_comment(): + # String comment was updated. + assert_result( + """\ + <!-- LOCALIZATION NOTE: NEW --> + <!ENTITY changed.string "string"> + """, + """\ + <!-- LOCALIZATION NOTE: OLD --> + <!ENTITY changed.string "string"> + """, + """\ + <!-- LOCALIZATION NOTE: NEW --> + <!ENTITY changed.string "string"> + """, + ) + # Comment added. + assert_result( + """\ + <!-- LOCALIZATION NOTE: NEW --> + <!ENTITY changed.string "string"> + """, + """\ + <!ENTITY changed.string "string"> + """, + """\ + <!-- LOCALIZATION NOTE: NEW --> + <!ENTITY changed.string "string"> + """, + ) + # Comment removed. + assert_result( + """\ + <!ENTITY changed.string "string"> + """, + """\ + <!-- LOCALIZATION NOTE: OLD --> + <!ENTITY changed.string "string"> + """, + """\ + <!ENTITY changed.string "string"> + """, + ) + + # With multiple comments + assert_result( + """\ + <!-- NEW FILE COMMENT --> + + <!-- LOCALIZATION NOTE: NEW --> + <!ENTITY changed.string "string"> + """, + """\ + <!-- OLD --> + + <!-- LOCALIZATION NOTE: OLD --> + <!ENTITY changed.string "string"> + """, + """\ + <!-- NEW FILE COMMENT --> + + <!-- LOCALIZATION NOTE: NEW --> + <!ENTITY changed.string "string"> + """, + ) + + +def test_reordered(): + # String was re.ordered. + assert_result( + """\ + <!ENTITY string.1 "value"> + <!ENTITY moved.string "move"> + """, + """\ + <!ENTITY moved.string "move"> + <!ENTITY string.1 "value"> + """, + """\ + <!ENTITY string.1 "value"> + <!ENTITY moved.string "move"> + """, + ) + + +def test_removed_string_with_comment(): + assert_result( + """\ + <!-- LOCALIZATION NOTE: Comment for first. --> + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + """, + """\ + <!-- LOCALIZATION NOTE: Comment for first. --> + <!ENTITY string.1 "First"> + <!-- LOCALIZATION NOTE: Comment for removed. --> + <!ENTITY removed "REMOVED"> + <!ENTITY string.2 "Second"> + """, + """\ + <!-- LOCALIZATION NOTE: Comment for first. --> + <!ENTITY string.1 "First"> + <!ENTITY string.2 "Second"> + + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!-- LOCALIZATION NOTE: Comment for removed. --> + <!ENTITY removed "REMOVED"> + """, + ) + + # With multiple lines of comments. + + assert_result( + """\ + <!-- First file comment --> + + <!-- LOCALIZATION NOTE: Comment for first. --> + <!-- LOCALIZATION NOTE: Comment 2 for first. --> + <!ENTITY string.1 "First"> + + <!-- Second + - file + - comment --> + + <!ENTITY string.2 "Second"> + """, + """\ + <!-- First file comment --> + + <!-- LOCALIZATION NOTE: Comment for first. --> + <!ENTITY string.1 "First"> + <!ENTITY removed.1 "First removed"> + <!-- LOCALIZATION NOTE: Comment for second removed. --> + <!ENTITY removed.2 "Second removed"> + + <!-- Removed file comment --> + + <!-- LOCALIZATION NOTE: Comment for third removed. --> + <!-- LOCALIZATION NOTE: Comment 2 for + third removed. --> + <!ENTITY removed.3 "Third removed"> + + <!-- Second + - file + - comment --> + + <!ENTITY removed.4 "Fourth removed"> + <!ENTITY string.2 "Second"> + """, + """\ + <!-- First file comment --> + + <!-- LOCALIZATION NOTE: Comment for first. --> + <!-- LOCALIZATION NOTE: Comment 2 for first. --> + <!ENTITY string.1 "First"> + + <!-- Second + - file + - comment --> + + <!ENTITY string.2 "Second"> + + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY removed.1 "First removed"> + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!-- LOCALIZATION NOTE: Comment for second removed. --> + <!ENTITY removed.2 "Second removed"> + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!-- LOCALIZATION NOTE: Comment for third removed. --> + <!-- LOCALIZATION NOTE: Comment 2 for + third removed. --> + <!ENTITY removed.3 "Third removed"> + <!-- LOCALIZATION NOTE: REMOVED STRING --> + <!ENTITY removed.4 "Fourth removed"> + """, + )

===================================== tools/torbrowser/l10n/combine/tests/test_fluent.py ===================================== @@ -0,0 +1,344 @@ +import textwrap + +from combine import combine_files + + +def assert_result(new_content, old_content, expect): + # Allow for indents to make the tests more readable. + if new_content is not None: + new_content = textwrap.dedent(new_content) + if old_content is not None: + old_content = textwrap.dedent(old_content) + if expect is not None: + expect = textwrap.dedent(expect) + assert expect == combine_files( + "test.ftl", new_content, old_content, "REMOVED STRING" + ) + + +def test_combine_empty(): + assert_result(None, None, None) + + +def test_combine_new_file(): + # New file with no old content. + assert_result( + """\ + string-1 = First + string-2 = Second + """, + None, + """\ + string-1 = First + string-2 = Second + """, + ) + + +def test_combine_removed_file(): + # Entire file was removed. + assert_result( + None, + """\ + string-1 = First + string-2 = Second + """, + """\ + + + ## REMOVED STRING + + string-1 = First + string-2 = Second + """, + ) + + +def test_no_change(): + content = """\ + string-1 = First + string-2 = Second + """ + assert_result(content, content, content) + + +def test_added_string(): + assert_result( + """\ + string-1 = First + string-new = NEW + string-2 = Second + """, + """\ + string-1 = First + string-2 = Second + """, + """\ + string-1 = First + string-new = NEW + string-2 = Second + """, + ) + + +def test_removed_string(): + assert_result( + """\ + string-1 = First + string-2 = Second + """, + """\ + string-1 = First + removed = REMOVED + string-2 = Second + """, + """\ + string-1 = First + string-2 = Second + + + ## REMOVED STRING + + removed = REMOVED + """, + ) + + +def test_removed_and_added(): + assert_result( + """\ + new-1 = New string + string-1 = + .attr = First + string-2 = Second + new-2 = + .title = New string 2 + """, + """\ + string-1 = + .attr = First + removed-1 = First removed + removed-2 = + .attr = Second removed + string-2 = Second + removed-3 = Third removed + """, + """\ + new-1 = New string + string-1 = + .attr = First + string-2 = Second + new-2 = + .title = New string 2 + + + ## REMOVED STRING + + removed-1 = First removed + removed-2 = + .attr = Second removed + removed-3 = Third removed + """, + ) + + +def test_updated(): + # String content was updated. + assert_result( + """\ + changed-string = NEW + """, + """\ + changed-string = OLD + """, + """\ + changed-string = NEW + """, + ) + + +def test_updated_comment(): + # String comment was updated. + assert_result( + """\ + # NEW + changed-string = string + """, + """\ + # OLD + changed-string = string + """, + """\ + # NEW + changed-string = string + """, + ) + # Comment added. + assert_result( + """\ + # NEW + changed-string = string + """, + """\ + changed-string = string + """, + """\ + # NEW + changed-string = string + """, + ) + # Comment removed. + assert_result( + """\ + changed-string = string + """, + """\ + # OLD + changed-string = string + """, + """\ + changed-string = string + """, + ) + + # With group comments. + assert_result( + """\ + ## GROUP NEW + + # NEW + changed-string = string + """, + """\ + ## GROUP OLD + + # OLD + changed-string = string + """, + """\ + ## GROUP NEW + + # NEW + changed-string = string + """, + ) + + +def test_reordered(): + # String was re-ordered. + assert_result( + """\ + string-1 = value + moved-string = move + """, + """\ + moved-string = move + string-1 = value + """, + """\ + string-1 = value + moved-string = move + """, + ) + + +def test_removed_string_with_comment(): + assert_result( + """\ + # Comment for first. + string-1 = First + string-2 = Second + """, + """\ + # Comment for first. + string-1 = First + # Comment for removed. + removed = REMOVED + string-2 = Second + """, + """\ + # Comment for first. + string-1 = First + string-2 = Second + + + ## REMOVED STRING + + # Comment for removed. + removed = REMOVED + """, + ) + + # Group comments are combined with the "REMOVED STRING" comments. + # If strings have no group comment, then a single "REMOVED STRING" is + # included for them. + assert_result( + """\ + ## First Group comment + + # Comment for first. + string-1 = First + + ## + + no-group = No group comment + + ## Second + ## Group comment + + string-2 = Second + """, + """\ + ## First Group comment + + # Comment for first. + string-1 = First + removed-1 = First removed + # Comment for second removed. + removed-2 = Second removed + + ## + + no-group = No group comment + removed-3 = Third removed + + ## Second + ## Group comment + + removed-4 = Fourth removed + string-2 = Second + """, + """\ + ## First Group comment + + # Comment for first. + string-1 = First + + ## + + no-group = No group comment + + ## Second + ## Group comment + + string-2 = Second + + + ## REMOVED STRING + ## First Group comment + + removed-1 = First removed + # Comment for second removed. + removed-2 = Second removed + + ## REMOVED STRING + + removed-3 = Third removed + + ## REMOVED STRING + ## Second + ## Group comment + + removed-4 = Fourth removed + """, + )

===================================== tools/torbrowser/l10n/combine/tests/test_properties.py ===================================== @@ -0,0 +1,322 @@ +import textwrap + +from combine import combine_files + + +def assert_result(new_content, old_content, expect): + # Allow for indents to make the tests more readable. + if new_content is not None: + new_content = textwrap.dedent(new_content) + if old_content is not None: + old_content = textwrap.dedent(old_content) + if expect is not None: + expect = textwrap.dedent(expect) + assert expect == combine_files( + "test.properties", new_content, old_content, "REMOVED STRING" + ) + + +def test_combine_empty(): + assert_result(None, None, None) + + +def test_combine_new_file(): + # New file with no old content. + assert_result( + """\ + string.1 = First + string.2 = Second + """, + None, + """\ + string.1 = First + string.2 = Second + """, + ) + + +def test_combine_removed_file(): + # Entire file was removed. + assert_result( + None, + """\ + string.1 = First + string.2 = Second + """, + """\ + + # REMOVED STRING + string.1 = First + # REMOVED STRING + string.2 = Second + """, + ) + + +def test_no_change(): + content = """\ + string.1 = First + string.2 = Second + """ + assert_result(content, content, content) + + +def test_added_string(): + assert_result( + """\ + string.1 = First + string.new = NEW + string.2 = Second + """, + """\ + string.1 = First + string.2 = Second + """, + """\ + string.1 = First + string.new = NEW + string.2 = Second + """, + ) + + +def test_removed_string(): + assert_result( + """\ + string.1 = First + string.2 = Second + """, + """\ + string.1 = First + removed = REMOVED + string.2 = Second + """, + """\ + string.1 = First + string.2 = Second + + # REMOVED STRING + removed = REMOVED + """, + ) + + +def test_removed_and_added(): + assert_result( + """\ + new.1 = New string + string.1 = First + string.2 = Second + new.2 = New string 2 + """, + """\ + string.1 = First + removed.1 = First removed + removed.2 = Second removed + string.2 = Second + removed.3 = Third removed + """, + """\ + new.1 = New string + string.1 = First + string.2 = Second + new.2 = New string 2 + + # REMOVED STRING + removed.1 = First removed + # REMOVED STRING + removed.2 = Second removed + # REMOVED STRING + removed.3 = Third removed + """, + ) + + +def test_updated(): + # String content was updated. + assert_result( + """\ + changed.string = NEW + """, + """\ + changed.string = OLD + """, + """\ + changed.string = NEW + """, + ) + + +def test_updated_comment(): + # String comment was updated. + assert_result( + """\ + # NEW + changed.string = string + """, + """\ + # OLD + changed.string = string + """, + """\ + # NEW + changed.string = string + """, + ) + # Comment added. + assert_result( + """\ + # NEW + changed.string = string + """, + """\ + changed.string = string + """, + """\ + # NEW + changed.string = string + """, + ) + # Comment removed. + assert_result( + """\ + changed.string = string + """, + """\ + # OLD + changed.string = string + """, + """\ + changed.string = string + """, + ) + + # With file comments + assert_result( + """\ + # NEW file comment + + # NEW + changed.string = string + """, + """\ + # OLD file comment + + # OLD + changed.string = string + """, + """\ + # NEW file comment + + # NEW + changed.string = string + """, + ) + + +def test_reordered(): + # String was re.ordered. + assert_result( + """\ + string.1 = value + moved.string = move + """, + """\ + moved.string = move + string.1 = value + """, + """\ + string.1 = value + moved.string = move + """, + ) + + +def test_removed_string_with_comment(): + assert_result( + """\ + # Comment for first. + string.1 = First + string.2 = Second + """, + """\ + # Comment for first. + string.1 = First + # Comment for removed. + removed = REMOVED + string.2 = Second + """, + """\ + # Comment for first. + string.1 = First + string.2 = Second + + # REMOVED STRING + # Comment for removed. + removed = REMOVED + """, + ) + + # With file comments and multi-line. + # All comments prior to a removed string are moved with it, until another + # entity or blank line is reached. + assert_result( + """\ + # First File comment + + # Comment for first. + # Comment 2 for first. + string.1 = First + + # Second + # File comment + + string.2 = Second + """, + """\ + # First File comment + + # Comment for first. + # Comment 2 for first. + string.1 = First + removed.1 = First removed + # Comment for second removed. + removed.2 = Second removed + + # Removed file comment + + # Comment 1 for third removed + # Comment 2 for third removed + removed.3 = Third removed + + # Second + # File comment + + removed.4 = Fourth removed + string.2 = Second + """, + """\ + # First File comment + + # Comment for first. + # Comment 2 for first. + string.1 = First + + # Second + # File comment + + string.2 = Second + + # REMOVED STRING + removed.1 = First removed + # REMOVED STRING + # Comment for second removed. + removed.2 = Second removed + # REMOVED STRING + # Comment 1 for third removed + # Comment 2 for third removed + removed.3 = Third removed + # REMOVED STRING + removed.4 = Fourth removed + """, + )

===================================== tools/torbrowser/migrate_l10n.py → tools/torbrowser/l10n/migrate.py =====================================

===================================== tools/torbrowser/l10n/migrations/__init__.py =====================================

===================================== tools/torbrowser/l10n_migrations/bug-41333-new-about-tor.py → tools/torbrowser/l10n/migrations/bug-41333-new-about-tor.py =====================================

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/compare/a014a0e...