richard pushed to branch tor-browser-102.12.0esr-12.5-1 at The Tor Project / Applications / Tor Browser

Commits: ae4c538d by Henry Wilkes at 2023-06-05T19:47:48+00:00 Bug 41803 - Add some developer tools for working on tor-browser.

- - - - -

2 changed files:

- + tools/torbrowser/git-rebase-fixup-preprocessor - + tools/torbrowser/tb-dev

Changes:

===================================== tools/torbrowser/git-rebase-fixup-preprocessor ===================================== @@ -0,0 +1,93 @@ +#!/usr/bin/python +""" +Pre-process a git todo file before passing it on to an editor. +""" + +import sys +import os +import subprocess +import re + +EDITOR_ENV_NAME = "GIT_REBASE_FIXUP_PREPROCESSOR_USER_EDITOR" + +try: + editor = os.environ[EDITOR_ENV_NAME] +except KeyError: + print(f"Missing {EDITOR_ENV_NAME} in environment", file=sys.stderr) + exit(1) + +if len(sys.argv) < 2: + print("Missing filename argument", file=sys.stderr) + exit(1) + +filename = sys.argv[1] + + +class TodoLine: + """ + Represents a line in the git todo file. + """ + + _PICK_REGEX = re.compile(r"^pick [a-f0-9]+ (?P<fixups>(fixup! )*)(?P<title>.*)") + + def __init__(self, line): + """ + Create a new line with the given text content. + """ + self._line = line + self._make_fixup = False + + match = self._PICK_REGEX.match(line) + if match: + self._is_pick = True + self._num_fixups = len(match.group("fixups")) / len("fixup! ") + self._title = match.group("title") + else: + self._is_pick = False + self._num_fixups = False + self._title = None + + def add_to_list_try_fixup(self, existing_lines): + """ + Add the TodoLine to the given list of other TodoLine, trying to fix up + one of the existing lines. + """ + if not self._num_fixups: # Not a fixup line. + existing_lines.append(self) + return + + # Search from the end of the list upwards. + for index in reversed(range(len(existing_lines))): + other = existing_lines[index] + if ( + other._is_pick + and self._num_fixups == other._num_fixups + 1 + and other._title == self._title + ): + self._make_fixup = True + existing_lines.insert(index + 1, self) + return + + # No line found to fixup. + existing_lines.append(self) + + def get_text(self): + """ + Get the text for the line to save. + """ + line = self._line + if self._make_fixup: + line = line.replace("pick", "fixup", 1) + return line + + +todo_lines = [] +with open(filename, "r", encoding="utf8") as todo_file: + for line in todo_file: + TodoLine(line).add_to_list_try_fixup(todo_lines) + +with open(filename, "w", encoding="utf8") as todo_file: + for line in todo_lines: + todo_file.write(line.get_text()) + +exit(subprocess.run([editor, *sys.argv[1:]], check=False).returncode)

===================================== tools/torbrowser/tb-dev ===================================== @@ -0,0 +1,715 @@ +#!/usr/bin/python +# PYTHON_ARGCOMPLETE_OK +""" +Useful tools for working on tor-browser repository. +""" + +import sys +import termios +import os +import atexit +import tempfile +import subprocess +import re +import json +import urllib.request +import argparse +import argcomplete + +GIT_PATH = "/usr/bin/git" +UPSTREAM_URLS = [ + "https://gitlab.torproject.org/tpo/applications/tor-browser.git", + "git@gitlab.torproject.org:tpo/applications/tor-browser.git", +] +FIXUP_PREPROCESSOR_EDITOR = "git-rebase-fixup-preprocessor" +USER_EDITOR_ENV_NAME = "GIT_REBASE_FIXUP_PREPROCESSOR_USER_EDITOR" + + +def git_run(args, check=True, env=None): + """ + Run a git command with output sent to stdout. + """ + if env is not None: + tmp_env = dict(os.environ) + for key, value in env.items(): + tmp_env[key] = value + env = tmp_env + subprocess.run([GIT_PATH, *args], check=check, env=env) + + +def git_get(args): + """ + Run a git command with each non-empty line returned in a list. + """ + git_process = subprocess.run( + [GIT_PATH, *args], text=True, stdout=subprocess.PIPE, check=True + ) + return [line for line in git_process.stdout.split("\n") if line] + + +local_root = None + + +def get_local_root(): + """ + Get the path for the tor-browser root directory. + """ + global local_root + if local_root is None: + try: + git_root = git_get(["rev-parse", "--show-toplevel"])[0] + except subprocess.CalledProcessError: + git_root = None + if git_root is None or os.path.basename(git_root) != "tor-browser": + local_root = "" + else: + local_root = git_root + return local_root + + +upstream_name = None + + +def get_upstream_name(): + """ + Get the name of the upstream remote. + """ + global upstream_name + if upstream_name is None: + for remote in git_get(["remote"]): + fetch_url = git_get(["remote", "get-url", remote])[0] + if fetch_url in UPSTREAM_URLS: + upstream_name = remote + break + if upstream_name is None: + raise Exception("No upstream remote found.") + return upstream_name + + +class Reference: + """Represents a git reference to a commit.""" + + def __init__(self, name, commit): + self.name = name + self.commit = commit + + +def get_refs(ref_type, name_start): + """ + Get a list of references that match the given 'ref_type' ("tag" or "remote" + or "head") that starts with the given 'name_start'. + """ + if ref_type == "tag": + # Instead of returning tag hash, return the commit hash it points to. + fstring = "%(*objectname)" + ref_start = "refs/tags/" + elif ref_type == "remote": + fstring = "%(objectname)" + ref_start = "refs/remotes/" + elif ref_type == "head": + fstring = "%(objectname)" + ref_start = "refs/heads/" + else: + raise TypeError(f"Unknown type {ref_type}") + + fstring = f"{fstring},%(refname)" + pattern = f"{ref_start}{name_start}**" + + def line_to_ref(line): + [commit, ref_name] = line.split(",", 1) + return Reference(ref_name.replace(ref_start, "", 1), commit) + + return [ + line_to_ref(line) + for line in git_get(["for-each-ref", f"--format={fstring}", pattern]) + ] + + +def get_nearest_ref(ref_type, name_start, search_from): + """ + Search backwards from the 'search_from' commit to find the first commit + that matches the given 'ref_type' that starts with the given 'name_start'. + """ + ref_list = get_refs(ref_type, name_start) + + for commit in git_get(["rev-list", "-1000", search_from]): + for ref in ref_list: + if commit == ref.commit: + return ref + + raise Exception(f"No {name_start} commit found in the last 1000 commits") + + +def get_firefox_ref(search_from): + """ + Search backwards from the 'search_from' commit to find the commit that comes + from firefox. + """ + return get_nearest_ref("tag", "FIREFOX_", search_from) + + +def get_upstream_commit(search_from): + """ + Get the first common ancestor of search_from that is also in its upstream + branch. + """ + return git_get(["merge-base", search_from, f"{search_from}@{{upstream}}"])[0] + + +def get_changed_files(from_commit, staged=False): + """ + Get a list of filenames relative to the current working directory that have + been changed since 'from_commit' (non-inclusive). + """ + args = ["diff"] + if staged: + args.append("--staged") + args.append("--name-only") + args.append(from_commit) + return [ + os.path.relpath(os.path.join(get_local_root(), filename)) + for filename in git_get(args) + ] + + +def file_contains(filename, regex): + """ + Return whether the file is a utf-8 text file containing the regular + expression given by 'regex'. + """ + with open(filename, "r", encoding="utf-8") as file: + try: + for line in file: + if regex.search(line): + return True + except UnicodeDecodeError: + # Not a text file + pass + return False + + +def get_gitlab_default(): + """ + Get the name of the default branch on gitlab. + """ + query = """ + query { + project(fullPath: "tpo/applications/tor-browser") { + repository { rootRef } + } + } + """ + request_data = {"query": re.sub(r"\s+", "", query)} + gitlab_request = urllib.request.Request( + "https://gitlab.torproject.org/api/graphql", + headers={ + "Content-Type": "application/json", + "User-Agent": "", + }, + data=json.dumps(request_data).encode("ascii"), + ) + + with urllib.request.urlopen(gitlab_request, timeout=20) as response: + branch_name = json.load(response)["data"]["project"]["repository"]["rootRef"] + return f"{get_upstream_name()}/{branch_name}" + + +def within_tor_browser_root(): + """ + Whether we are with the tor browser root. + """ + root = get_local_root() + if not root: + return False + return os.path.commonpath([os.getcwd(), root]) == root + + +# * -------------------- * +# | Methods for commands | +# * -------------------- * + + +def show_firefox_commit(_args): + """ + Print the tag name and commit for the last firefox commit below the current + HEAD. + """ + ref = get_firefox_ref("HEAD") + print(ref.name) + print(ref.commit) + + +def show_upstream_commit(_args): + """ + Print the last upstream commit for the current HEAD. + """ + print(get_upstream_commit("HEAD")) + + +def show_log(args): + """ + Show the git log between the current HEAD and the last firefox commit. + """ + commit = get_firefox_ref("HEAD").commit + git_run(["log", f"{commit}..HEAD", *args.gitargs], check=False) + + +def show_files_containing(args): + """ + List all the files that that have been modified for tor browser, that also + contain a regular expression. + """ + try: + regex = re.compile(args.regex) + except re.error as err: + raise Exception(f"{args.regex} is not a valid python regex") from err + + file_list = get_changed_files(get_firefox_ref("HEAD").commit) + + for filename in file_list: + if not os.path.isfile(filename): + # deleted ofile + continue + if file_contains(filename, regex): + print(filename) + + +def show_changed_files(_args): + """ + List all the files that have been modified relative to upstream. + """ + for filename in get_changed_files(get_upstream_commit("HEAD")): + print(filename) + + +def lint_changed_files(args): + """ + Lint all the files that have been modified relative to upstream. + """ + os.chdir(get_local_root()) + file_list = [ + f + for f in get_changed_files(get_upstream_commit("HEAD")) + if os.path.isfile(f) # Not deleted + ] + command_base = ["./mach", "lint"] + lint_process = subprocess.run( + [*command_base, "--list"], text=True, stdout=subprocess.PIPE, check=True + ) + + linters = [] + for line in lint_process.stdout.split("\n"): + if not line: + continue + if line.startswith("Note that clang-tidy"): + # Note at end + continue + if line == "license": + # don't lint the license + continue + if line.startswith("android-"): + continue + # lint everything else + linters.append("-l") + linters.append(line) + + if not linters: + raise Exception("No linters found") + + if args.fix: + command_base.append("--fix") + # We add --warnings since clang only reports whitespace issues as warnings. + lint_process = subprocess.run( + [*command_base, "--warnings", *linters, *file_list], check=False + ) + + +def prompt_user(prompt, convert): + """ + Ask the user for some input until the given converter returns without + throwing a ValueError. + """ + while True: + # Flush out stdin. + termios.tcflush(sys.stdin, termios.TCIFLUSH) + print(prompt, end="") + sys.stdout.flush() + try: + return convert(sys.stdin.readline().strip()) + except ValueError: + # Continue to prompt. + pass + + +def binary_reply_default_no(value): + """Process a 'y' or 'n' reply, defaulting to 'n' if empty.""" + if value == "": + return False + if value.lower() == "y": + return True + if value.lower() == "n": + return False + raise ValueError() + + +def get_fixup_for_file(filename, firefox_commit): + """Find the commit the given file should fix up.""" + + def parse_log_line(line): + [commit, short_ref, title] = line.split(",", 2) + return {"commit": commit, "short-ref": short_ref, "title": title} + + options = [ + parse_log_line(line) + for line in git_get( + [ + "log", + "--pretty=format:%H,%h,%s", + f"{firefox_commit}..HEAD", + "--", + filename, + ] + ) + ] + if not options: + print(f"No commit found for {filename}") + return None + + def valid_index(val): + if val == "d": + return val + + is_patch = val.startswith("p") + if is_patch: + val = val[1:] + + # May raise a ValueError. + as_index = int(val) + if as_index < 0 or as_index > len(options): + raise ValueError() + + if as_index == 0: + if is_patch: + raise ValueError() + return None + + return (is_patch, options[as_index - 1]["commit"]) + + while True: + print(f"For {filename}:\n") + print(" \x1b[1m0\x1b[0m: None") + for index, opt in enumerate(options): + print( + f" \x1b[1m{index + 1}\x1b[0m: " + + f"\x1b[1;38;5;212m{opt['short-ref']}\x1b[0m " + + opt["title"] + ) + print("") + response = prompt_user( + "Choose an <index> to fixup, or '0' to skip this file, " + "or 'd' to view the pending diff, " + "or 'p<index>' to view the patch for the index: ", + valid_index, + ) + if response is None: + # Skip this file. + return None + + if response == "d": + git_run(["diff", "--", filename]) + continue + + view_patch, commit = response + if view_patch: + git_run(["log", "-p", "-1", commit, "--", filename]) + continue + + return commit + + +def auto_fixup(_args): + """ + Automatically find and fix up commits using the current unstaged changes. + """ + # Only want to search as far back as the firefox commit. + firefox_commit = get_firefox_ref("HEAD").commit + + staged_files = get_changed_files("HEAD", staged=True) + if staged_files: + raise Exception(f"Have already staged files: {staged_files}") + + fixups = {} + for filename in get_changed_files("HEAD"): + commit = get_fixup_for_file(filename, firefox_commit) + if commit is None: + continue + if commit not in fixups: + fixups[commit] = [filename] + else: + fixups[commit].append(filename) + print("") + + for commit, files in fixups.items(): + print("") + git_run(["add", *files]) + git_run(["commit", f"--fixup={commit}"]) + print("") + + if prompt_user( + "Edit fixup commit message? (y/\x1b[4mn\x1b[0m)", binary_reply_default_no + ): + git_run(["commit", "--amend"]) + + +def clean_fixups(_args): + """ + Perform an interactive rebase that automatically applies fixups, similar to + --autosquash but also works on fixups of fixups. + """ + user_editor = git_get(["var", "GIT_SEQUENCE_EDITOR"])[0] + sub_editor = os.path.join( + os.path.dirname(os.path.realpath(__file__)), FIXUP_PREPROCESSOR_EDITOR + ) + + git_run( + ["rebase", "--interactive"], + check=False, + env={"GIT_SEQUENCE_EDITOR": sub_editor, USER_EDITOR_ENV_NAME: user_editor}, + ) + + +def show_default(_args): + """ + Print the default branch name from gitlab. + """ + print(get_gitlab_default()) + + +def branch_from_default(args): + """ + Fetch the default gitlab branch from upstream and create a new local branch. + """ + default_branch = get_gitlab_default() + + git_run(["fetch"], get_upstream_name()) + git_run(["switch", "--create", args.branchname, "--track", default_branch]) + + +def rebase_on_default(_args): + """ + Fetch the default gitlab branch from upstream and rebase the current branch + on top. + """ + try: + branch_name = git_get(["branch", "--show-current"])[0] + except IndexError: + raise Exception("No current branch") + + current_upstream = get_upstream_commit("HEAD") + default_branch = get_gitlab_default() + + git_run(["fetch"], get_upstream_name()) + # We set the new upstream before the rebase in case there are conflicts. + git_run(["branch", f"--set-upstream-to={default_branch}"]) + git_run( + ["rebase", "--onto", default_branch, current_upstream, branch_name], check=False + ) + + +def show_range_diff(args): + """ + Show the range diff between two branches, from their firefox bases. + """ + firefox_commit_1 = get_firefox_ref(args.branch1).commit + firefox_commit_2 = get_firefox_ref(args.branch2).commit + git_run( + [ + "range-diff", + f"{firefox_commit_1}..{args.branch1}", + f"{firefox_commit_2}..{args.branch2}", + ], + check=False, + ) + + +def show_diff_diff(args): + """ + Show the diff between the diffs of two branches, relative to their firefox + bases. + """ + config_res = git_get(["config", "--get", "diff.tool"]) + if not config_res: + raise Exception("No diff.tool configured for git") + diff_tool = config_res[0] + + # Filter out parts of the diff we expect to be different. + index_regex = re.compile(r"index [0-9a-f]{12}..[0-9a-f]{12}") + lines_regex = re.compile(r"@@ -[0-9]+,[0-9]+ +[0-9]+,[0-9]+ @@(?P<rest>.*)") + + def save_diff(branch): + firefox_commit = get_firefox_ref(branch).commit + file_desc, file_name = tempfile.mkstemp( + text=True, prefix=f'{branch.split("/")[-1]}-' + ) + # Register deleting the file at exit. + atexit.register(os.remove, file_name) + + diff_process = subprocess.Popen( + [GIT_PATH, "diff", f"{firefox_commit}..{branch}"], + stdout=subprocess.PIPE, + text=True, + ) + + with os.fdopen(file_desc, "w") as file: + for line in diff_process.stdout: + if index_regex.match(line): + # Fake data that will match. + file.write("index ????????????..????????????\n") + continue + lines_match = lines_regex.match(line) + if lines_match: + # Fake data that will match. + file.write("@@ ?,? ?,? @@" + lines_match.group('rest')) + continue + file.write(line) + + status = diff_process.poll() + if status != 0: + raise Exception(f"git diff exited with status {status}") + + return file_name + + file_1 = save_diff(args.branch1) + file_2 = save_diff(args.branch2) + subprocess.run([diff_tool, file_1, file_2], check=False) + + +# * -------------------- * +# | Command line parsing | +# * -------------------- * + + +def branch_complete(prefix, parsed_args, **kwargs): + """ + Complete the argument with a branch name. + """ + if not within_tor_browser_root(): + return [] + try: + branches = [ref.name for ref in get_refs("head", "")] + branches.extend([ref.name for ref in get_refs("remote", "")]) + branches.append("HEAD") + except Exception: + return [] + return [br for br in branches if br.startswith(prefix)] + + +parser = argparse.ArgumentParser() +subparsers = parser.add_subparsers(required=True) + +for name, details in { + "show-upstream-commit": { + "func": show_upstream_commit, + }, + "changed-files": { + "func": show_changed_files, + }, + "lint-changed-files": { + "func": lint_changed_files, + "args": { + "--fix": { + "help": "whether to fix the files", + "action": "store_true", + }, + }, + }, + "auto-fixup": { + "func": auto_fixup, + }, + "clean-fixups": { + "func": clean_fixups, + }, + "show-default": { + "func": show_default, + }, + "branch-from-default": { + "func": branch_from_default, + "args": { + "branchname": { + "help": "the name for the new local branch", + "metavar": "<branch-name>", + }, + }, + }, + "rebase-on-default": { + "func": rebase_on_default, + }, + "show-firefox-commit": { + "func": show_firefox_commit, + }, + "log": { + "func": show_log, + "args": { + "gitargs": { + "help": "argument to pass to git log", + "metavar": "-- git-log-arg", + "nargs": "*", + }, + }, + }, + "branch-range-diff": { + "func": show_range_diff, + "args": { + "branch1": { + "help": "the first branch to compare", + "metavar": "<branch-1>", + "completer": branch_complete, + }, + "branch2": { + "help": "the second branch to compare", + "metavar": "<branch-2>", + "completer": branch_complete, + }, + }, + }, + "branch-diff-diff": { + "func": show_diff_diff, + "args": { + "branch1": { + "help": "the first branch to compare", + "metavar": "<branch-1>", + "completer": branch_complete, + }, + "branch2": { + "help": "the second branch to compare", + "metavar": "<branch-2>", + "completer": branch_complete, + }, + }, + }, + "files-containing": { + "func": show_files_containing, + "args": { + "regex": {"help": "the regex that the files must contain"}, + }, + }, +}.items(): + help_message = re.sub(r"\s+", " ", details["func"].__doc__).strip() + sub = subparsers.add_parser(name, help=help_message) + sub.set_defaults(func=details["func"]) + for arg, keywords in details.get("args", {}).items(): + completer = None + if "completer" in keywords: + completer = keywords["completer"] + del keywords["completer"] + sub_arg = sub.add_argument(arg, **keywords) + if completer is not None: + sub_arg.completer = completer + +argcomplete.autocomplete(parser) + +if not within_tor_browser_root(): + raise Exception("Must be within a tor-browser directory") +parsed_args = parser.parse_args() + +parsed_args.func(parsed_args)

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/commit/ae4c538d...