morgan pushed to branch tor-browser-128.0esr-14.0-1 at The Tor Project / Applications / Tor Browser Commits: a0092907 by Richard Pospesel at 2024-07-22T21:52:45+00:00 Bug 42683: Create script to generate issue triage csv file from bugzilla query and git logs - - - - - 1 changed file: - + tools/torbrowser/generate-bugzilla-triage-csv.sh Changes: ===================================== tools/torbrowser/generate-bugzilla-triage-csv.sh ===================================== @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +# prints to stderr +function echoerr() { echo "$@" 1>&2; } + +# help dialog +if [ "$#" -lt 5 ]; then + echoerr "Usage: $0 ff-version begin-commit end-commit gitlab-audit-issue reviewers..." + echoerr "" + echoerr "Writes a CSV to stdout of Bugzilla issues to triage for a particular Firefox version. This" + echoerr "script performs a union of the labeled Bugzilla issues in Mozilla's issue tracker and the" + echoerr "labeled commits in the provided commit range" + echoerr + echoerr " ff-version rapid-release Firefox version to audit" + echoerr " begin-commit starting gecko-dev commit of this Firefox version" + echoerr " end-commit ending gecko-dev commit of this Firefox version" + echoerr " gitlab-audit-issue tor-browser-spec Gitlab issue number for this audit" + echoerr " reviewers... space-separated list of reviewers responsible for this audit" + echoerr "" + echoerr "Example:" + echoerr "" + echoerr "$0 116 FIREFOX_ESR_115_BASE FIREFOX_116_0_3_RELEASE 40064 richard pierov henry" + exit 1 +fi + +# set -x +set -e + + +# Ensure various required tools are available +function check_exists() { + local cmd=$1 + if ! which ${cmd} > /dev/null ; then + echoerr "missing ${cmd} dependency" + exit 1 + fi +} + +check_exists git +check_exists jq +check_exists mktemp +check_exists perl +check_exists printf +check_exists sed +check_exists sort +check_exists touch +check_exists uniq +check_exists wget + +# Assign arguments to named variables +firefox_version=$1 +git_begin=$2 +git_end=$3 +audit_issue=$4 +reviewers="${@:5}" + +# Check valid Firefox version +if ! [[ "${firefox_version}" =~ ^[1-9][0-9]{2}$ ]]; then + echoerr "invalid Firefox version (probably)" + exit 1 +fi + +# Check valid Gitlab issue number +if ! [[ "${audit_issue}" =~ ^[1-9][0-9]{4}$ ]]; then + echoerr "invalid gitlab audit issue number (probably)" + exit 1 +fi + +# +# Encoding/Decoding Functions +# + +# escape " and \ +function json_escape() { + local input="$1" + echo "${input}" | sed 's/["\]/\\"/g' +} + + +# un-escape \" +function jq_unescape() { + local input="$1" + echo "${input}" | sed 's/\\"/"/g' +} + +# change quotes to double-quotes +function csv_escape() { + local input="$1" + echo "${input}" | sed 's/"/""/g' +} + +# we need to urlencode the strings used in the new issue link +function url_encode() { + local input="$1" + echo "${input}" | perl -MURI::Escape -wlne 'print uri_escape $_' +} + + +# +# Create temp json files +# +git_json=$(mktemp -t git-audit-${firefox_version}-XXXXXXXXXXX.json) +bugzilla_json=$(mktemp -t bugzilla-audit-${firefox_version}-XXXXXXXXXXX.json) +union_json=$(mktemp -t union-audit-${firefox_version}-XXXXXXXXXXX.json) +touch "${git_json}" +touch "${bugzilla_json}" +touch "${union_json}" + +function json_cleanup { + rm -f "${git_json}" + rm -f "${bugzilla_json}" + rm -f "${union_json}" +} +trap json_cleanup EXIT + +# +# Generate Git Commit Triage List +# + +# Try and extract bug id and summary from git log +# Mozilla's commits are not always 100% consistently named, so this +# regex is a bit flexible to handle various inputs such as: +# "Bug 1234 -", "Bug 1234:", "Bug Bug 1234 -", "[Bug 1234] -", " bug 1234 -". +sed_extract_id_summary="s/^[[ ]*[bug –-]+ ([1-9][0-9]*)[]:\., –-]*(.*)\$/\\1 \\2/pI" + +# Generate a json array of objects in the same format as bugzilla: {id: number, summary: string} +printf "[\n" >> "${git_json}" + +first_object=true +git log --format='%s' $git_begin..$git_end \ +| sed -En "${sed_extract_id_summary}" \ +| sort -h \ +| uniq \ +| while IFS= read -r line; do + read -r id summary <<< "${line}" + summary=$(json_escape "${summary}") + + # json does not allow trailing commas + if [[ "${first_object}" = true ]]; then + first_object=false + else + printf ",\n" >> "${git_json}" + fi + + printf " { \"id\": %s, \"summary\": \"%s\" }" ${id} "${summary}" >> "${git_json}" +done +printf "\n]\n" >> "${git_json}" + +# +# Download Bugzilla Triage List +# + +# search for: +# + Product is NOT "Thunderbird,Calander,Chat Core,MailNews Core" (&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core). AND +# + Target Milestone contains "${firefox_version}" (115 Branch or Firefox 115) (&f2=target_milestone&o2=substring&v2=${firefox_version}). +# "&limit=0" shows all matching bugs. + +query_tail="&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core&f2=target_milestone&o2=substring&v2=${firefox_version}&limit=0" + +bugzilla_query="https://bugzilla.mozilla.org/buglist.cgi?${query_tail}" +bugzilla_json_query="https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary${query_tail}" + +wget "${bugzilla_json_query}" -O ${bugzilla_json} + + +# +# Create Union of these two sets of issues +# + +# bugzilla array is actually on a root object: { bugs: [...] } +jq -s '[ (.[0].bugs)[], (.[1])[] ] | group_by(.id) | map(.[0])' "${bugzilla_json}" "${git_json}" > "${union_json}" + +# +# Generate Triage CSV +# + +echo "\"Review\",,\"Bugzilla Bug\"" + +jq '. | sort_by(.id)[] | "\(.id)|\(.summary)"' ${union_json} \ +| while IFS='|' read -r id summary; do + + # bugzilla info + id="${id:1}" + summary="${summary:0:-1}" + summary=$(jq_unescape "${summary}") + # short summary for gitlab issue title + [[ ${#summary} -gt 80 ]] && summary_short="${summary:0:77}..." || summary_short="${summary}" + + # filter out some issue types that we never care about + skip_issue=false + + # skip `[wpt-sync] Sync PR` + if [[ "${summary}" =~ ^\[wpt-sync\]\ Sync\ PR.*$ ]]; then + skip_issue=true + # skip `Crash in [@` and variants + elif [[ "${summary}" =~ ^Crash[esin\ ]*\ \[\@.*$ ]]; then + skip_issue=true + # skip `Assertion failuire: ` + elif [[ "${summary}" =~ ^Assertion\ failure:\ .*$ ]]; then + skip_issue=true + # skip `Hit MOZ_CRASH` + elif [[ "${summary}" =~ ^Hit\ MOZ_CRASH.*$ ]]; then + skip_issue=true + fi + + if [[ "${skip_issue}" = true ]]; then + echoerr "Skipped Bugzilla ${id}: ${summary_short}" + else + csv_summary=$(csv_escape "${summary}") + + # parent issue + bugzilla_url="https://bugzilla.mozilla.org/show_bug.cgi?id=${id}" + # review issue title + new_issue_title=$(url_encode "Review Mozilla ${id}: ${summary_short}") + # review issue description + labeling (14.0 stable, FF128-esr, Next) + new_issue_description=$(url_encode "### Bugzilla: ${bugzilla_url}")%0A$(url_encode "/label ~\"14.0 stable\" ~FF128-esr ~Next")%0A$(url_encode "/relate tpo/applications/tor-browser-spec#${audit_issue}")%0A%0A$(url_encode "<!-- briefly describe why this issue needs further review -->")%0A + # url which create's new issue with title and description pre-populated + new_issue_url="https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/new?issue[title]=${new_issue_title}&issue[description]=${new_issue_description}" + + # this link will start the creation of a new gitlab issue to review + create_issue=$(csv_escape "=HYPERLINK(\"${new_issue_url}\", \"New Issue\")") + bugzilla_link=$(csv_escape "=HYPERLINK(\"${bugzilla_url}\", \"Bugzilla ${id}: ${csv_summary}\")") + + echo "FALSE,\"${create_issue}\",\"${bugzilla_link}\"," + fi +done + +echo +echo "\"Triaged by:\"" +for reviewer in $reviewers; do + reviewer=$(csv_escape "${reviewer}") + echo "\"FALSE\",\"${reviewer}\"" +done +echo + +bugzilla_query="=HYPERLINK(\"${bugzilla_query}\", \"Bugzilla query\")" +echo \"$(csv_escape "${bugzilla_query}")\" View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/commit/a0092907... -- View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser/-/commit/a0092907... You're receiving this email because of your account on gitlab.torproject.org.