commit 68566ac4e7021456026bf057e47a00fa8513d2cf Author: Matthew Finkel sysrqb@torproject.org Date: Thu Jan 21 21:29:22 2021 +0000

Bug 40010: Generalize audit script --- audits/java_audit.sh | 206 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 142 insertions(+), 64 deletions(-)

diff --git a/audits/java_audit.sh b/audits/java_audit.sh old mode 100644 new mode 100755 index b1183eb..3586470 --- a/audits/java_audit.sh +++ b/audits/java_audit.sh @@ -1,90 +1,168 @@ #!/bin/bash -e

-if [ $# -ne 3 ]; then - echo "usage: <path/to/repo> <old commit> <new commit>" +if [ $# -ne 4 ]; then + echo "usage: <path/to/repo> <lang> <old commit> <new commit>" exit 1 fi

REPO_DIR=$1

-OLD=$2 -NEW=$3 - -SCOPE="java" # string: this is the java audit -EXT="java kt" +SCOPE=$2 +OLD=$3 +NEW=$4

declare -a KEYWORDS

#KEYWORDS+=('+++\ ')

-# URL access -KEYWORDS+=(URLConnection) -KEYWORDS+=(UrlConnectionDownloader) - -# Proxy settings -KEYWORDS+=(ProxySelector) - -# Android and java networking and 3rd party libs -KEYWORDS+=("openConnection(") -KEYWORDS+=("java.net") -KEYWORDS+=("javax.net") -KEYWORDS+=(android.net) -KEYWORDS+=(android.webkit) - -# Third Party http libs -KEYWORDS+=(ch.boye.httpclientandroidlib.impl.client) -KEYWORDS+=(okhttp) - -# Intents -KEYWORDS+=(IntentHelper) -KEYWORDS+=(openUriExternal) -KEYWORDS+=(getHandlersForMimeType) -KEYWORDS+=(getHandlersForURL) -KEYWORDS+=(getHandlersForIntent) -# KEYOWRDS+=(android.content.Intent) # Common -KEYWORDS+=(startActivity) -KEYWORDS+=(startActivities) -KEYWORDS+=(startBroadcast) -KEYWORDS+=(sendBroadcast) -KEYWORDS+=(sendOrderedBroadcast) -KEYWORDS+=(startService) -KEYWORDS+=(bindService) -KEYWORDS+=(android.app.PendingIntent) -KEYWORDS+=(ActivityHandlerHelper.startIntentAndCatch) -KEYWORDS+=(AppLinksInterceptor) -KEYWORDS+=(AppLinksUseCases) - -# Rust symbols -KEYWORDS+=("connect(") -KEYWORDS+=("recvmsg(") -KEYWORDS+=("sendmsg(") -KEYWORDS+=("::post(") -KEYWORDS+=("::get(") - -cd $REPO_DIR - -# Step 1: Generate match pattern based on in-scope keywords +initialize_java_symbols() { + # URL access + KEYWORDS+=(URLConnection) + KEYWORDS+=(UrlConnectionDownloader) + + # Proxy settings + KEYWORDS+=(ProxySelector) + + # Android and java networking and 3rd party libs + KEYWORDS+=("openConnection(") + KEYWORDS+=("java.net") + KEYWORDS+=("javax.net") + KEYWORDS+=(android.net) + KEYWORDS+=(android.webkit) + + # Third Party http libs + KEYWORDS+=(ch.boye.httpclientandroidlib.impl.client) + KEYWORDS+=(okhttp) + + # Intents + KEYWORDS+=(IntentHelper) + KEYWORDS+=(openUriExternal) + KEYWORDS+=(getHandlersForMimeType) + KEYWORDS+=(getHandlersForURL) + KEYWORDS+=(getHandlersForIntent) + # KEYOWRDS+=(android.content.Intent) # Common + KEYWORDS+=(startActivity) + KEYWORDS+=(startActivities) + KEYWORDS+=(startBroadcast) + KEYWORDS+=(sendBroadcast) + KEYWORDS+=(sendOrderedBroadcast) + KEYWORDS+=(startService) + KEYWORDS+=(bindService) + KEYWORDS+=(android.app.PendingIntent) + KEYWORDS+=(ActivityHandlerHelper.startIntentAndCatch) + KEYWORDS+=(AppLinksInterceptor) + KEYWORDS+=(AppLinksUseCases) + KEYWORDS+=(ActivityDelegate) +} + +initialize_rust_symbols() { + KEYWORDS+=("connect(") + KEYWORDS+=("recvmsg(") + KEYWORDS+=("sendmsg(") + KEYWORDS+=("::post(") + KEYWORDS+=("::get(") +} + +initialize_cpp_symbols() { + KEYWORDS+=("PR_GetHostByName") + KEYWORDS+=("PR_GetIPNodeByName") + KEYWORDS+=("PR_GetAddrInfoByName") + KEYWORDS+=("PR_StringToNetAddr") + + KEYWORDS+=("MDNS") + KEYWORDS+=("mDNS") + KEYWORDS+=("mdns") + + KEYWORDS+=("TRR") + KEYWORDS+=("trr") + + KEYWORDS+=("AsyncResolve") + KEYWORDS+=("asyncResolve") + KEYWORDS+=("ResolveHost") + KEYWORDS+=("resolveHost") + + KEYWORDS+=("SOCK_") + KEYWORDS+=("SOCKET_") + KEYWORDS+=("_SOCKET") + + KEYWORDS+=("UDPSocket") + KEYWORDS+=("TCPSocket") + + KEYWORDS+=("PR_Socket") + + KEYWORDS+=("SocketProvider") + KEYWORDS+=("udp-socket") + KEYWORDS+=("tcp-socket") + KEYWORDS+=("tcpsocket") + KEYWORDS+=("SOCKET") + KEYWORDS+=("mozilla.org/network") +} + +initialize_js_symbols() { + KEYWORDS+=("AsyncResolve(") + KEYWORDS+=("asyncResolve(") + KEYWORDS+=("ResolveHost(") + KEYWORDS+=("resolveHost(") + + KEYWORDS+=("udp-socket") + KEYWORDS+=("udpsocket") + KEYWORDS+=("tcp-socket") + KEYWORDS+=("tcpsocket") + KEYWORDS+=("SOCKET") + KEYWORDS+=("mozilla.org/network") +} + +# Step 1: Initialize scope of audit +EXT= +case "${SCOPE}" in + "java" | "kt" | "java-kt" ) + EXT="java kt" + SCOPE="java-kt" + initialize_java_symbols + ;; + "c-cpp" | "c-cxx" | "c" | "cxx" | "cpp" ) + EXT="c cpp h cxx hpp hxx" + SCOPE="c-cpp" + initialize_cpp_symbols + ;; + "rust" ) + EXT="rs" + initialize_rust_symbols + ;; + "js" ) + EXT="js jsm" + initialize_js_symbols + ;; + * ) + echo "requested language not recognized" + exit 1 + ;; +esac + +cd "$REPO_DIR" + +# Step 2: Generate match pattern based on in-scope keywords function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; } -GREP_LINE="$(join_by | ${KEYWORDS[@]})" +GREP_LINE="$(join_by | "${KEYWORDS[@]}")"

-# Step 2: Obtain patches for all in-scope files where a keyword is present -echo "Diffing patches-${OLD}-${NEW}-${SCOPE}.diff" -path= +# Step 3: Obtain patches for all in-scope files where a keyword is present +declare -a path for ext in ${EXT}; do - path="${path} *.${ext}" + path+=("*.${ext}") done +echo "Diffing patches-${OLD}-${NEW}-${SCOPE}.diff from all ${path[*]} files" # Exclude Deleted and Unmerged files from diff DIFF_FILTER=ACMRTXB -git diff --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" $OLD $NEW -- ${path} > patches-${OLD}-${NEW}-${SCOPE}.diff +git diff --color=always --color-moved --diff-filter="${DIFF_FILTER}" -U20 -G"${GREP_LINE}" "$OLD" "$NEW" -- "${path[@]}" > "patches-${OLD}-${NEW}-${SCOPE}.diff"

-# Step 3: Highlight the keyword with an annoying, flashing color +# Step 4: Highlight the keyword with an annoying, flashing color export GREP_COLOR="05;37;41" # Capture the entire file and/or overlap with the previous match, add GREP_COLOR highlighting -egrep -A10000 -B10000 --color=always "${GREP_LINE}" patches-${OLD}-${NEW}-${SCOPE}.diff > keywords-$OLD-$NEW-$SCOPE.diff +grep -A10000 -B10000 --color=always -E "${GREP_LINE}" "patches-${OLD}-${NEW}-${SCOPE}.diff" > "keywords-$OLD-$NEW-$SCOPE.diff"

# Add a 'XXX MATCH XXX' at the end of each matched line, easily searchable. -sed -i 's/(\x1b[05;37;41.*)/\1 XXX MATCH XXX/' keywords-$OLD-$NEW-$SCOPE.diff +sed -i 's/(\x1b[05;37;41.*)/\1 XXX MATCH XXX/' "keywords-$OLD-$NEW-$SCOPE.diff"

-# Step 4: Review the code changes +# Step 5: Review the code changes echo "Diff generated. View it with:" echo " less -R $REPO_DIR/keywords-$OLD-$NEW-$SCOPE.diff"