morgan pushed to branch main at The Tor Project / Applications / tor-browser-build Commits: 48741e0d by Morgan at 2024-11-19T19:50:42+00:00 Bug 41305: Add bugzilla-audit spreadsheet generation script to tools directory - - - - - 2 changed files: - tools/browser/README.md - + tools/browser/generate-bugzilla-triage-csv Changes: ===================================== tools/browser/README.md ===================================== @@ -1,5 +1,36 @@ # Tools +### generate-bugzilla-triage-csv + +This script generates a csv file (to be hosted on Google Sheets) which enumerates all of the Bugzilla issues resolved for a particular Firefox release as defined by Mozilla's bug-tracker AND the set of patches in a provided commit range. Each entry will include the Bugzilla issue title and a link which creates an issue in [tpo/applications/tor-browser](https://gitlab.torproject.org/tpo/applications/tor-browser) for further review. + +This script should be invoked, the output written to a .csv file, and uploaded to a shared Google Sheets spreadsheet for team triage + review. + +#### Prerequisites + +- The user must create the following soft-links: + - `/tools/browser/torbrowser` -> `/path/to/local/tor-browser.git` + +#### Usage + +``` +Usage: ./tools/browser/generate-bugzilla-triage-csv <ff-version> <begin-commit> <end-commit> <gitlab-audit-issue> <reviewers...> + +ff-version rapid-release Firefox version to audit +begin-commit starting gecko-dev commit of this Firefox version +end-commit ending gecko-dev commit of this Firefox version +gitlab-audit-issue tor-browser GitLab issue number for this audit +reviewers... space-separated list of reviewers responsible for this audit +``` + +#### Examples + +Generates a spreadsheet derived from the Firefox 129 resolved issues and all commits between `FIREFOX_NIGHTLY_128_END` and `FIREFOX_NIGHTLY_129_END`. New issues will automatically link themslves to [tor-browser#43303](https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/43303). The requested reviewers will be morgan, pierov, and henry. + +```bash +./tools/browser/generate-bugzilla-triage-csv 129 FIREFOX_NIGHTLY_128_END FIREFOX_NIGHTLY_129_END 43303 morgan pierov henry +``` + ### sign-tag This script gpg signs a git tag associated with a particular browser commit in the user's tor-browser.git or mullvad-browser.git repo. ===================================== tools/browser/generate-bugzilla-triage-csv ===================================== @@ -0,0 +1,243 @@ +#!/usr/bin/env bash + +# gitlab labels for review tickets +browser_label="15.0 stable" +esr_label="esr-140" + +# prints to stderr +function echoerr() { echo "$@" 1>&2; } + +script_dir=$(dirname "${BASH_ARGV0:-$0}") + +# help dialog +if [ "$#" -lt 5 ]; then + echoerr "Usage: $0 <ff-version> <begin-commit> <end-commit> <gitlab-audit-issue> <reviewers...>" + echoerr + echoerr "ff-version rapid-release Firefox version to audit" + echoerr "begin-commit starting gecko-dev commit of this Firefox version" + echoerr "end-commit ending gecko-dev commit of this Firefox version" + echoerr "gitlab-audit-issue tor-browser GitLab issue number for this audit" + echoerr "reviewers... space-separated list of reviewers responsible for this audit" + echoerr "" + echoerr "Example:" + echoerr "" + echoerr "$0 129 FIREFOX_NIGHTLY_128_END FIREFOX_NIGHTLY_129_END 43303 morgan pierov henry" + exit 1 +fi + +# exit on error +set -e + + +# Ensure various required tools are available +function check_exists() { + local cmd=$1 + if ! which ${cmd} > /dev/null ; then + echoerr "missing ${cmd} dependency" + exit 1 + fi +} + +check_exists git +check_exists jq +check_exists mktemp +check_exists perl +check_exists printf +check_exists sed +check_exists sort +check_exists touch +check_exists uniq +check_exists wget + +# Assign arguments to named variables +firefox_version=$1 +git_begin=$2 +git_end=$3 +audit_issue=$4 +reviewers="${@:5}" + +# Check valid Firefox version +if ! [[ "${firefox_version}" =~ ^[1-9][0-9]{2}$ ]]; then + echoerr "invalid Firefox version (probably)" + exit 1 +fi + +# Check valid Gitlab issue number +if ! [[ "${audit_issue}" =~ ^[1-9][0-9]{4}$ ]]; then + echoerr "invalid gitlab audit issue number (probably)" + exit 1 +fi + +# +# Encoding/Decoding Functions +# + +# escape " and \ +function json_escape() { + local input="$1" + echo "${input}" | sed 's/["\]/\\"/g' +} + + +# un-escape \" +function jq_unescape() { + local input="$1" + echo "${input}" | sed 's/\\"/"/g' +} + +# change quotes to double-quotes +function csv_escape() { + local input="$1" + echo "${input}" | sed 's/"/""/g' +} + +# we need to urlencode the strings used in the new issue link +function url_encode() { + local input="$1" + echo "${input}" | perl -MURI::Escape -wlne 'print uri_escape $_' +} + + +# +# Create temp json files +# +git_json=$(mktemp -t git-audit-${firefox_version}-XXXXXXXXXXX.json) +bugzilla_json=$(mktemp -t bugzilla-audit-${firefox_version}-XXXXXXXXXXX.json) +union_json=$(mktemp -t union-audit-${firefox_version}-XXXXXXXXXXX.json) +touch "${git_json}" +touch "${bugzilla_json}" +touch "${union_json}" + +function json_cleanup { + rm -f "${git_json}" + rm -f "${bugzilla_json}" + rm -f "${union_json}" + popd > /dev/null +} +pushd "${script_dir}/torbrowser" > /dev/null +trap json_cleanup EXIT + +# +# Generate Git Commit Triage List +# + +# Try and extract bug id and summary from git log +# Mozilla's commits are not always 100% consistently named, so this +# regex is a bit flexible to handle various inputs such as: +# "Bug 1234 -", "Bug 1234:", "Bug Bug 1234 -", "[Bug 1234] -", " bug 1234 -". +sed_extract_id_summary="s/^[[ ]*[bug –-]+ ([1-9][0-9]*)[]:\., –-]*(.*)\$/\\1 \\2/pI" + +# Generate a json array of objects in the same format as bugzilla: {component: string, id: number, summary: string} +printf "[\n" >> "${git_json}" + +first_object=true +git log --format='%s' $git_begin..$git_end \ +| sed -En "${sed_extract_id_summary}" \ +| sort -h \ +| uniq \ +| while IFS= read -r line; do + read -r id summary <<< "${line}" + summary=$(json_escape "${summary}") + + # json does not allow trailing commas + if [[ "${first_object}" = true ]]; then + first_object=false + else + printf ",\n" >> "${git_json}" + fi + + printf " { \"component\": \"Unknown\", \"id\": %s, \"summary\": \"%s\" }" ${id} "${summary}" >> "${git_json}" +done +printf "\n]\n" >> "${git_json}" + +# +# Download Bugzilla Triage List +# + +# search for: +# + Product is NOT "Thunderbird,Calander,Chat Core,MailNews Core" (&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core). AND +# + Target Milestone contains "${firefox_version}" (e.g. 115 Branch or Firefox 115) (&f2=target_milestone&o2=substring&v2=${firefox_version}). +# "&limit=0" shows all matching bugs. + +query_tail="&f1=product&n1=1&o1=anyexact&v1=Thunderbird%2CCalendar%2CChat%20Core%2CMailNews%20Core&f2=target_milestone&o2=substring&v2=${firefox_version}&limit=0" + +bugzilla_query="https://bugzilla.mozilla.org/buglist.cgi?${query_tail}" +bugzilla_json_query="https://bugzilla.mozilla.org/rest/bug?include_fields=id,component,summary${query_tail}" + +wget "${bugzilla_json_query}" -O ${bugzilla_json} + + +# +# Create Union of these two sets of issues +# + +# bugzilla array is actually on a root object: { bugs: [...] } +jq -s '[ (.[0].bugs)[], (.[1])[] ] | group_by(.id) | map(.[0])' "${bugzilla_json}" "${git_json}" > "${union_json}" + +# +# Generate Triage CSV +# + +echo "\"Review\",,\"Bugzilla Component\",\"Bugzilla Bug\"" + +jq '. | sort_by([.component, .id])[] | "\(.id)|\(.component)|\(.summary)"' ${union_json} \ +| while IFS='|' read -r id component summary; do + + # bugzilla info + id="${id:1}" + component="${component:0}" + summary="${summary:0:-1}" + summary=$(jq_unescape "${summary}") + # short summary for gitlab issue title + [[ ${#summary} -gt 80 ]] && summary_short="${summary:0:77}..." || summary_short="${summary}" + + # filter out some issue types that we never care about + skip_issue=false + + # skip `[wpt-sync] Sync PR` + if [[ "${summary}" =~ ^\[wpt-sync\]\ Sync\ PR.*$ ]]; then + skip_issue=true + # skip `Crash in [@` and variants + elif [[ "${summary}" =~ ^Crash[esin\ ]*\ \[\@.*$ ]]; then + skip_issue=true + # skip `Assertion failuire: ` + elif [[ "${summary}" =~ ^Assertion\ failure:\ .*$ ]]; then + skip_issue=true + # skip `Hit MOZ_CRASH` + elif [[ "${summary}" =~ ^Hit\ MOZ_CRASH.*$ ]]; then + skip_issue=true + fi + + if [[ "${skip_issue}" = true ]]; then + echoerr "Skipped Bugzilla ${id}: ${summary_short}" + else + csv_summary=$(csv_escape "${summary}") + csv_component=$(csv_escape "${component}") + + # parent issue + bugzilla_url="https://bugzilla.mozilla.org/show_bug.cgi?id=${id}" + # review issue title + new_issue_title=$(url_encode "Review Mozilla ${id}: ${summary_short}") + # review issue description + labeling (14.0 stable, FF128-esr, Next) + new_issue_description=$(url_encode "### Bugzilla: ${bugzilla_url}")%0A$(url_encode "/label ~\"${browser_label}\" ~\"${esr_label}\" ~\"Bugzilla Review\" ~Next")%0A$(url_encode "/relate tpo/applications/tor-browser#${audit_issue}")%0A%0A$(url_encode "<!-- briefly describe why this issue needs further review -->")%0A + # url which create's new issue with title and description pre-populated + new_issue_url="https://gitlab.torproject.org/tpo/applications/tor-browser/-/issues/new?issue[title]=${new_issue_title}&issue[description]=${new_issue_description}" + + # this link will start the creation of a new gitlab issue to review + create_issue=$(csv_escape "=HYPERLINK(\"${new_issue_url}\", \"New Issue\")") + bugzilla_link=$(csv_escape "=HYPERLINK(\"${bugzilla_url}\", \"Bugzilla ${id}: ${csv_summary}\")") + + echo "FALSE,\"${create_issue}\",\"${csv_component}\",\"${bugzilla_link}\"," + fi +done + +echo +echo "\"Triaged by:\"" +for reviewer in $reviewers; do + reviewer=$(csv_escape "${reviewer}") + echo "\"FALSE\",\"${reviewer}\"" +done +echo + +bugzilla_query="=HYPERLINK(\"${bugzilla_query}\", \"Bugzilla query\")" +echo \"$(csv_escape "${bugzilla_query}")\" View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/48... -- View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/commit/48... You're receiving this email because of your account on gitlab.torproject.org.