Pier Angelo Vendrame pushed to branch main at The Tor Project / Applications / tor-browser-build Commits: 3086604b by Pier Angelo Vendrame at 2026-02-11T15:50:54+00:00 Bug 41713: Triage: find components for bugs found from commits. In the Bugzilla triage script, run a second query for the bugs that were not found in the first query and instead have been found with git commits. - - - - - 2 changed files: - − tools/browser/generate-bugzilla-triage-csv - + tools/browser/generate-bugzilla-triage-csv.py Changes: ===================================== tools/browser/generate-bugzilla-triage-csv deleted ===================================== @@ -1,249 +0,0 @@ -#!/usr/bin/env bash - -# gitlab labels for review tickets -browser_label="Apps::Product::TorBrowser" -esr_label="esr-next" -priority_label="Priority::Medium" -impact_label="Apps::Impact::Unknown" -type_label="Apps::Type::Audit" - -# milestone for the next major Tor Browser release -milestone="Browser 16.0" - -# prints to stderr -function echoerr() { echo "$@" 1>&2; } - -script_dir=$(dirname "${BASH_ARGV0:-$0}") - -# help dialog -if [ "$#" -ne 2 ]; then - echoerr "Usage: $0 <ff-version> <gitlab-audit-issue>" - echoerr - echoerr "ff-version rapid-release Firefox version to audit" - echoerr "gitlab-audit-issue tor-browser GitLab issue number for this audit" - echoerr "" - echoerr "Example:" - echoerr "" - echoerr "$0 129 43303" - exit 1 -fi - -# exit on error -set -e - - -# Ensure various required tools are available -function check_exists() { - local cmd=$1 - if ! which ${cmd} > /dev/null ; then - echoerr "missing ${cmd} dependency" - exit 1 - fi -} - -check_exists git -check_exists jq -check_exists mktemp -check_exists perl -check_exists printf -check_exists sed -check_exists sort -check_exists touch -check_exists uniq -check_exists wget - -# Assign arguments to named variables -firefox_version=$1 -firefox_old_version=$((firefox_version - 1)) -audit_issue=$2 -git_begin="FIREFOX_NIGHTLY_${firefox_old_version}_END" -git_end="FIREFOX_NIGHTLY_${firefox_version}_END" - -# Check valid Firefox version -if ! [[ "${firefox_version}" =~ ^[1-9][0-9]{2}$ ]]; then - echoerr "invalid Firefox version (probably)" - exit 1 -fi - -# Check valid Gitlab issue number -if ! [[ "${audit_issue}" =~ ^[1-9][0-9]{4}$ ]]; then - echoerr "invalid gitlab audit issue number (probably)" - exit 1 -fi - -# -# Encoding/Decoding Functions -# - -# escape " and \ -function json_escape() { - local input="$1" - echo "${input}" | sed 's/["\]/\\"/g' -} - - -# un-escape \" -function jq_unescape() { - local input="$1" - echo "${input}" | sed 's/\\"/"/g' -} - -# change quotes to double-quotes -function csv_escape() { - local input="$1" - echo "${input}" | sed 's/"/""/g' -} - -# we need to urlencode the strings used in the new issue link -function url_encode() { - local input="$1" - echo "${input}" | perl -MURI::Escape -wlne 'print uri_escape $_' -} - - -# -# Create temp json files -# -git_json=$(mktemp -t git-audit-${firefox_version}-XXXXXXXXXXX.json) -bugzilla_json=$(mktemp -t bugzilla-audit-${firefox_version}-XXXXXXXXXXX.json) -union_json=$(mktemp -t union-audit-${firefox_version}-XXXXXXXXXXX.json) -touch "${git_json}" -touch "${bugzilla_json}" -touch "${union_json}" - -function json_cleanup { - rm -f "${git_json}" - rm -f "${bugzilla_json}" - rm -f "${union_json}" - popd > /dev/null -} -pushd "${script_dir}/torbrowser" > /dev/null -trap json_cleanup EXIT - -# -# Generate Git Commit Triage List -# - -# Try and extract bug id and summary from git log -# Mozilla's commits are not always 100% consistently named, so this -# regex is a bit flexible to handle various inputs such as: -# "Bug 1234 -", "Bug 1234:", "Bug Bug 1234 -", "[Bug 1234] -", " bug 1234 -". -sed_extract_id_summary="s/^[[ ]*[bug –-]+ ([1-9][0-9]*)[]:\., –-]*(.*)\$/\\1 \\2/pI" - -# Generate a json array of objects in the same format as bugzilla: {component: string, id: number, summary: string} -printf "[\n" >> "${git_json}" - -first_object=true -git log --format='%s' $git_begin..$git_end \ -| sed -En "${sed_extract_id_summary}" \ -| sort -h \ -| uniq \ -| while IFS= read -r line; do - read -r id summary <<< "${line}" - summary=$(json_escape "${summary}") - - # json does not allow trailing commas - if [[ "${first_object}" = true ]]; then - first_object=false - else - printf ",\n" >> "${git_json}" - fi - - product="Unknown" - # Bugs that include webcompat-reviewers and no other additional -reviewers - # from other teams are *likely* in the Web Compatibility component. But they - # won't show in the bugzilla query because the component doesn't tend to - # have any set milestones. - if [[ "${summary}" =~ r=.*webcompat-reviewers && ! "${summary}" =~ r=.*-reviewers.*-reviewers ]]; then - product="Web Compatibility" - fi - - printf " { \"product\": \"%s\", \"component\": \"Unknown\", \"id\": %s, \"summary\": \"%s\" }" "${product}" ${id} "${summary}" >> "${git_json}" -done -printf "\n]\n" >> "${git_json}" - -# -# Download Bugzilla Triage List -# - -# search for: -# + Product is NOT "Thunderbird,Calander,Chat Core,MailNews Core" (&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core). AND -# + Target Milestone contains "${firefox_version}" (e.g. 115 Branch or Firefox 115) (&f2=target_milestone&o2=substring&v2=${firefox_version}). -# "&limit=0" shows all matching bugs. - -query_tail="&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core&f2=target_milestone&o2=substring&v2=${firefox_version}&limit=0" - -bugzilla_query="https://bugzilla.mozilla.org/buglist.cgi?${query_tail}" -bugzilla_json_query="https://bugzilla.mozilla.org/rest/bug?include_fields=id,product,component,summary${query_tail}" - -wget "${bugzilla_json_query}" -O ${bugzilla_json} - -# -# Create Union of these two sets of issues -# - -# bugzilla array is actually on a root object: { bugs: [...] } -jq -s '[ (.[0].bugs)[], (.[1])[] ] | group_by(.id) | map(.[0])' "${bugzilla_json}" "${git_json}" > "${union_json}" - -# -# Generate Triage CSV -# - -echo "\"Review\",,\"Product :: Component\",\"Bugzilla Bug\"" - -jq '. | sort_by([.product, .component, .id])[] | "\(.id)|\(.product)|\(.component)|\(.summary)"' ${union_json} \ -| while IFS='|' read -r id product component summary; do - - # bugzilla info - id="${id:1}" - component="${component:0}" - summary="${summary:0:-1}" - summary=$(jq_unescape "${summary}") - # short summary for gitlab issue title - [[ ${#summary} -gt 80 ]] && summary_short="${summary:0:77}..." || summary_short="${summary}" - - # filter out some issue types that we never care about - skip_issue=false - - # skip `[wpt-sync] Sync PR` - if [[ "${summary}" =~ ^\[wpt-sync\]\ Sync\ PR.*$ ]]; then - skip_issue=true - # skip `Crash in [@` and variants - elif [[ "${summary}" =~ ^Crash[esin\ ]*\ \[\@.*$ ]]; then - skip_issue=true - # skip `Assertion failuire: ` - elif [[ "${summary}" =~ ^Assertion\ failure:\ .*$ ]]; then - skip_issue=true - # skip `Hit MOZ_CRASH` - elif [[ "${summary}" =~ ^Hit\ MOZ_CRASH.*$ ]]; then - skip_issue=true - fi - - if [[ "${skip_issue}" = true ]]; then - echoerr "Skipped Bugzilla ${id}: ${summary_short}" - else - csv_summary=$(csv_escape "${summary}") - csv_product_component=$(csv_escape "${product} :: ${component}") - - # parent issue - bugzilla_url="https://bugzilla.mozilla.org/show_bug.cgi?id=${id}" - # review issue title - new_issue_title=$(url_encode "Review Mozilla ${id}: ${summary_short}") - # review issue description + labeling (14.0 stable, FF128-esr, Next) - new_issue_description=$(url_encode "### Bugzilla: ${bugzilla_url}")%0A$(url_encode "/label ~\"${browser_label}\" ~\"${esr_label}\" ~\"${priority_label}\" ~\"${impact_label}\" ~\"${type_label}\"")%0A$(url_encode "/milestone %\"${milestone}\"")%0A$(url_encode "/relate tpo/applications/tor-browser#${audit_issue}")%0A%0A$(url_encode "<!-- briefly describe why this issue needs further review -->")%0A - - # url which create's new issue with title and description pre-populated - new_issue_url="https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/new?issue[title]=${new_issue_title}&issue[description]=${new_issue_description}" - - # this link will start the creation of a new gitlab issue to review - create_issue=$(csv_escape "=HYPERLINK(\"${new_issue_url}\", \"New Issue\")") - bugzilla_link=$(csv_escape "=HYPERLINK(\"${bugzilla_url}\", \"Bugzilla ${id}: ${csv_summary}\")") - - echo "FALSE,\"${create_issue}\",\"${csv_product_component}\",\"${bugzilla_link}\"," - fi -done - -echo - -bugzilla_query="=HYPERLINK(\"${bugzilla_query}\", \"Bugzilla query\")" -echo \"$(csv_escape "${bugzilla_query}")\" ===================================== tools/browser/generate-bugzilla-triage-csv.py ===================================== @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +import argparse +import csv +import re +import sys +import urllib.parse +from pathlib import Path + +import requests +from git import Repo + + +GITLAB_LABELS = [ + "Apps::Product::TorBrowser", + "esr-next", + "Priority::Medium", + "Apps::Impact::Unknown", + "Apps::Type::Audit", +] +GITLAB_MILESTONE = "Browser 16.0" + + +def download_bugs(url): + url += "&" if "?" in url else "?" + url += "include_fields=id,product,component,summary" + r = requests.get(url) + r.raise_for_status() + # {"bugs": [ {...}, ... ]} + bugs = r.json().get("bugs", []) + results = {} + for bug in bugs: + product = bug.get("product", "") + component = bug.get("component", "") + results[bug.get("id")] = { + "id": bug.get("id"), + "component": f"{product} :: {component}", + "summary": bug.get("summary", ""), + } + return results + + +def extract_from_git(repo_path, ff_version, bugs): + subj_regex = re.compile( + r"^[\s*[bug –-]+\s*([1-9][0-9]*)[\]:\., –-]*(.*)", re.VERBOSE + ) + # Keep these lowercase + skip = ["no bug", "merge ", "revert "] + + repo = Repo(repo_path) + seen_ids = set() + results = {} + for commit in repo.iter_commits( + f"FIREFOX_NIGHTLY_{ff_version - 1}_END..FIREFOX_NIGHTLY_{ff_version}_END", + reverse=True, + ): + subject = commit.message.splitlines()[0] + subj_low = subject.lower() + m = subj_regex.search(subj_low) + if not m: + if not any(subj_low.startswith(p) for p in skip): + print(f"Could not match {subject}", file=sys.stderr) + continue + bug_id = int(m.group(1)) + if bug_id in bugs or bug_id in seen_ids: + continue + seen_ids.add(bug_id) + summary = m.group(2).strip() + results[bug_id] = { + "id": bug_id, + "component": "(git commit)", + "summary": summary, + } + + # Notice that seen_ids contains only the bugs we have seen in the + # commit range but we did not find earlier. + # So, we expect this number to be quite low and that it will be + # possible to fetch all these bugs in a single pass. + if seen_ids: + url = "https://bugzilla.mozilla.org/rest/bug?id=" + url += ",".join([str(i) for i in seen_ids]) + results.update(download_bugs(url)) + bugs.update(results) + + +def make_hyperlink(url, label): + label = label.replace('"', '""') + return f'=HYPERLINK("{url}", "{label}")' + + +def url_encode(s): + return urllib.parse.quote(s) + + +def generate_csv(bugs, audit_issue, query, output_path): + bugs = sorted(bugs.values(), key=lambda x: (x["component"], x["id"])) + + with output_path.open("w") as f: + writer = csv.writer(f, quoting=csv.QUOTE_ALL) + writer.writerow(["Review", "", "Bugzilla Component", "Bugzilla Bug"]) + + # Keep these lowercase! + skip = [ + "[wpt-sync] sync pr", + "assertion failure: ", + "crash in ", + "crash [@", + "hit moz_crash", + "update android nightly application-services", + "update web-platform-tests", + ] + for bug in bugs: + bug_id = bug["id"] + component = bug["component"] + summary = bug["summary"] + summary_lower = summary.lower() + short = (summary[:77] + "...") if len(summary) > 80 else summary + + if any(summary_lower.startswith(p) for p in skip): + print(f"Skipped Bugzilla {bug_id}: {summary}", file=sys.stderr) + continue + + bugzilla_url = ( + f"https://bugzilla.mozilla.org/show_bug.cgi?id={bug_id}" + ) + new_title = url_encode(f"Review Mozilla {bug_id}: {short}") + new_desc = ( + url_encode(f"### Bugzilla: {bugzilla_url}") + + "%0A" + + url_encode("/label ~" + (" ~".join(GITLAB_LABELS))) + + "%0A" + + url_encode(f'/milestone %"{GITLAB_MILESTONE}"') + + "%0A" + + url_encode( + f"/relate tpo/applications/tor-browser#{audit_issue}" + ) + + "%0A%0A" + + url_encode( + "<!-- briefly describe why this issue needs further review -->" + ) + ) + new_issue_url = f"https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/new?issue[title]={new_title}&issue[description]={new_desc}" + + create_link = make_hyperlink(new_issue_url, "New Issue") + bugzilla_link = make_hyperlink( + bugzilla_url, f"Bugzilla {bug_id}: {summary}" + ) + writer.writerow(["FALSE", create_link, component, bugzilla_link]) + + writer.writerow([]) + writer.writerow([make_hyperlink(query, "Bugzilla query")]) + + +def main(): + parser = argparse.ArgumentParser( + description="Create a triage CSV for a Firefox version." + ) + parser.add_argument( + "-r", + "--repo-path", + help="Path to a tor-browser.git clone", + type=Path, + default=Path(__file__).parent / "torbrowser", + ) + parser.add_argument( + "-o", + "--output-file", + help="Path to the output file", + type=Path, + ) + parser.add_argument( + "ff_version", + help="Firefox rapid-release version (e.g. 150)", + type=int, + ) + parser.add_argument( + "gitlab_audit_issue", + help="Tor-Browser-Spec GitLab issue number", + type=int, + ) + known_args, remaining = parser.parse_known_args() + parser.set_defaults( + output_file=Path(f"triage-{known_args.ff_version}.csv") + ) + args = parser.parse_args() + + if args.ff_version < 140: + print("Wrong Firefox version (probably)!", file=sys.stderr) + sys.exit(1) + if args.gitlab_audit_issue < 40000: + print("Wrong GitLab issue number (probably)!", file=sys.stderr) + sys.exit(1) + + excluded_products = "Thunderbird,Calendar,Chat%20Core,MailNews%20Core" + query = f"https://bugzilla.mozilla.org/rest/bug?f1=product&n1=1&o1=anyexact&v1={excluded_products}&f2=target_milestone&o2=substring&v2={args.ff_version}&limit=0" + bugs = download_bugs(query) + extract_from_git(args.repo_path, args.ff_version, bugs) + + generate_csv( + bugs, + args.gitlab_audit_issue, + query, + args.output_file, + ) + + +if __name__ == "__main__": + main() View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/30... -- View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/30... You're receiving this email because of your account on gitlab.torproject.org.