This is an automated email from the git hooks/post-receive script. richard pushed a commit to branch master in repository tor-browser-spec. The following commit(s) were added to refs/heads/master by this push: new a437cf1 Bug 40028: Create issue template for network audits a437cf1 is described below commit a437cf18f2f7cd239f4b5fb3137e840082756b31 Author: Richard Pospesel <richard@torproject.org> AuthorDate: Mon Apr 4 23:11:17 2022 +0000 Bug 40028: Create issue template for network audits - also update code_audit.sh to also print a commit's message/hash for further investigation, save reports in script directory, a renamed reports to include repo folder name, added cc and hh file extensions to c/cpp filter, changed arg order to make it easer to change filter for same commit range --- .gitlab/issue_templates/Firefox Issue Audit.md | 113 +++++++++++++++++++++++++ audits/code_audit.sh | 70 ++++++++++++--- 2 files changed, 169 insertions(+), 14 deletions(-) diff --git a/.gitlab/issue_templates/Firefox Issue Audit.md b/.gitlab/issue_templates/Firefox Issue Audit.md new file mode 100644 index 0000000..81f781f --- /dev/null +++ b/.gitlab/issue_templates/Firefox Issue Audit.md @@ -0,0 +1,113 @@ +# General + +The audit begins at the commit hash where the previous audit ended. Use code_audit.sh for creating the diff and highlighting potentially problematic code. The audit is scoped to a specific language (currently C/C++, Rust, Java/Kotlin, and Javascript). + +The output includes the entire patch where the new problematic code was introduced. Search for `XXX MATCH XXX` to find the next potential violation. + +`code_audit.sh` contains the list of known problematic APIs. New usage of these functions are documented and analyzed in this audit. + +## Firefox: https://github.com/mozilla/gecko-dev.git + +- Start: `$(FIRST_GIT_HASH)` ( `$(START_TAG)` ) +- End: `$(LAST_GIT_HASH)` ( `$(END_TAG)` ) + +### Languages: +- [ ] java +- [ ] cpp +- [ ] js +- [ ] rust + +Nothing of interest (using `code_audit.sh`) + +**OR** + +### foreach PROBLEMATIC_HASH: +#### $(PROBLEMATIC_HASH) +- Summary +- Review Result: (SAFE|BAD) + +--- + +## Application Services: https://github.com/mozilla/application-services.git + +- Start: `$(FIRST_GIT_HASH)` ( `$(START_TAG)` ) +- End: `$(LAST_GIT_HASH)` ( `$(END_TAG)` ) + +### Languages: +- [ ] java +- [ ] cpp +- [ ] js +- [ ] rust + +Nothing of interest (using `code_audit.sh`) + +**OR** + +### foreach PROBLEMATIC_HASH: +#### $(PROBLEMATIC_HASH) +- Summary +- Review Result: (SAFE|BAD) + +## Android Components: https://github.com/mozilla-mobile/android-components.git + +- Start: `$(FIRST_GIT_HASH)` ( `$(START_TAG)` ) +- End: `$(LAST_GIT_HASH)` ( `$(END_TAG)` ) + +### Languages: +- [ ] java +- [ ] cpp +- [ ] js +- [ ] rust + +Nothing of interest (using `code_audit.sh`) + +**OR** + +### foreach PROBLEMATIC_HASH: +#### $(PROBLEMATIC_HASH) +- Summary +- Review Result: (SAFE|BAD) + +## Fenix: https://github.com/mozilla-mobile/fenix.git + +- Start: `$(FIRST_GIT_HASH)` ( `$(START_TAG)` ) +- End: `$(LAST_GIT_HASH)` ( `$(END_TAG)` ) + +### Languages: +- [ ] java +- [ ] cpp +- [ ] js +- [ ] rust + +Nothing of interest (using `code_audit.sh`) + +**OR** + +### foreach PROBLEMATIC_HASH: +#### $(PROBLEMATIC_HASH) +- Summary +- Review Result: (SAFE|BAD) + +## Ticket Review ## + +Bugzilla Query: `https://bugzilla.mozilla.org/buglist.cgi?query_format=advanced&resolution=FIXED&target_milestone=$(FIREFOX_VERSION)%20Branch&order=priority%2Cbug_severity&limit=0` + +where `$(FIREFOX_VERSION)` is the major Firefox version we are auditing (eg: '91') + +Nothing of interest (manual inspection) + +**OR** (foreach)** + +### foreach PROBLEMATIC_TICKET: +#### $(PROBLEMATIC_TICKET) +- Summary +- Review Result: (SAFE|BAD) + +## Regression/Prior Vuln Review ## + +Review proxy bypass bugs; check for new vectors to look for: + - https://gitlab.torproject.org/groups/tpo/applications/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=Proxy%20Bypass + - Look for new features like these. Especially external app launch vectors + +## Export +- [ ] Export Report and save to `tor-browser-spec/audits` diff --git a/audits/code_audit.sh b/audits/code_audit.sh index d260d15..a8b5868 100755 --- a/audits/code_audit.sh +++ b/audits/code_audit.sh @@ -1,15 +1,17 @@ #!/bin/bash -e +# set -x + if [ $# -ne 4 ]; then - echo "usage: <path/to/repo> <lang> <old commit> <new commit>" + echo "usage: <path/to/repo> <old commit> <new commit> <lang>" exit 1 fi REPO_DIR=$1 -SCOPE=$2 -OLD=$3 -NEW=$4 +OLD=$2 +NEW=$3 +SCOPE=$4 declare -a KEYWORDS @@ -127,7 +129,7 @@ case "${SCOPE}" in initialize_java_symbols ;; "c-cpp" | "c-cxx" | "c" | "cxx" | "cpp" ) - EXT="c cpp h cxx hpp hxx" + EXT="c cpp h cxx hpp hxx cc hh" SCOPE="c-cpp" initialize_cpp_symbols ;; @@ -145,6 +147,7 @@ case "${SCOPE}" in ;; esac +AUDIT_DIR=$(pwd) cd "$REPO_DIR" # Step 2: Generate match pattern based on in-scope keywords @@ -156,19 +159,58 @@ declare -a path for ext in ${EXT}; do path+=("*.${ext}") done -echo "Diffing patches-${OLD}-${NEW}-${SCOPE}.diff from all ${path[*]} files" +PROJECT_NAME=$(basename $(pwd)) +REPORT_FILE="$AUDIT_DIR/$PROJECT_NAME-$SCOPE-${OLD:0:8}-${NEW:0:8}.report" + +echo "Diffing all ${path[*]} files in commit range ${OLD:0:8}..${NEW:0:8}" # Exclude Deleted and Unmerged files from diff DIFF_FILTER=ACMRTXB -git diff --stat --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" "$OLD" "$NEW" -- "${path[@]}" > "patches-${OLD}-${NEW}-${SCOPE}.diff" -# Step 4: Highlight the keyword with an annoying, flashing color +rm -f "${REPORT_FILE}" + +# Step 4: Concat a diff of each commit containing keyword and the commit message/hash +# of said commit + +# Flashing Color constants export GREP_COLOR="05;37;41" -# Capture the entire file and/or overlap with the previous match, add GREP_COLOR highlighting -grep -A10000 -B10000 --color=always -E "${GREP_LINE}" "patches-${OLD}-${NEW}-${SCOPE}.diff" > "keywords-$OLD-$NEW-$SCOPE.diff" -# Add a 'XXX MATCH XXX' at the end of each matched line, easily searchable. -sed -i 's/\(\x1b\[05;37;41.*\)/\1 XXX MATCH XXX/' "keywords-$OLD-$NEW-$SCOPE.diff" +for COMMIT in $(git rev-list --ancestry-path $OLD..$NEW); do + TEMP_DIFF="$(mktemp)" + + echo "Diffing $COMMIT..." + + # Do each diff commit by commit so we can add context from the commit log + # to each diff + git diff --stat --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" $COMMIT~ $COMMIT -- "${path[@]}" > "${TEMP_DIFF}" + if [ -s "${TEMP_DIFF}" ] + then + # Highlight the keyword with an annoying, flashing color + FLASHING_DIFF="$(mktemp)" + grep -A10000 -B10000 --color=always -E "${GREP_LINE}" "${TEMP_DIFF}" > "${FLASHING_DIFF}" + mv "${FLASHING_DIFF}" "${TEMP_DIFF}" + + # Add a 'XXX MATCH XXX' at the end of each matched line, easily searchable. + sed -i 's/\(\x1b\[05;37;41.*\)/\1 XXX MATCH XXX/' "${TEMP_DIFF}" + + # Found some diff, so cat the changelog for commit then the diff + echo "-----------------------------------------------" >> "${REPORT_FILE}" + git log -n 1 $COMMIT >> "${REPORT_FILE}" + echo "-----------------------------------------------" >> "${REPORT_FILE}" + cat "${TEMP_DIFF}" >> "${REPORT_FILE}" + fi + + rm -f "${TEMP_DIFF}" +done # Step 5: Review the code changes -echo "Diff generated. View it with:" -echo " less -R $REPO_DIR/keywords-$OLD-$NEW-$SCOPE.diff" + +if [ -s "${REPORT_FILE}" ] +then + echo "" + echo "Report generated. View it with:" + echo "" + echo "less -R \"$(basename "${REPORT_FILE}")\"" +else + echo "No keywords found. No report generated" +fi + -- To stop receiving notification emails like this one, please contact the administrator of this repository.