[tor-commits] [chutney/master] Add SUMMARY and IGNORE_EXISTING modes to warnings.sh

[teor at torproject.org]

commit 5d731f3c93e5f3b43c783d0d03d2d0b93179bdcc
Author: teor <teor2345 at gmail.com>
Date:   Thu Feb 23 17:19:55 2017 +1100

    Add SUMMARY and IGNORE_EXISTING modes to warnings.sh
    
    Closes #21371.
---
 tools/ignore.warnings | 40 ++++++++++++++++++++++++++++++++++++++++
 tools/warnings.sh     | 47 +++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 79 insertions(+), 8 deletions(-)

diff --git a/tools/ignore.warnings b/tools/ignore.warnings
new file mode 100644
index 0000000..59c2c5a
--- /dev/null
+++ b/tools/ignore.warnings
@@ -0,0 +1,40 @@
+# these patterns are ignored by warnings.sh when IGNORE_EXPECTED is set to 1
+expands into rules which apply to all private IPv4 and IPv6 addresses
+good signatures from recognized authorities for us to accept it
+Cannot open fingerprint file
+Consensus with empty bandwidth
+Could not open.*sr-state.*No such file or directory
+Currently, sandboxing is only implemented on Linux
+Failed to choose an exit server
+Failed to find node for hop [0-1] of our path
+# Tor Bug 21524
+IPv4 address.*private.*with accept6/reject6 field type in exit policy
+In a future version of Tor, ExitRelay 0 may become the default when no ExitPolicy is given
+No available nodes when trying to choose node
+Not enough info to publish pending .* consensus
+# Necessary to make IPv4 and IPv6 exit templates work
+Option 'ExitPolicyRejectPrivate' used more than once
+Option 'ExitRelay' used more than once
+Received http status code 404 .* from server .* while fetching
+# Tor Bug 21525
+Rejected vote from .*Bad valid-after time
+Rejecting vote from .* with valid-after time of .* we were expecting 1970-01-01 00:00:00
+SR: Unable to read SR state file
+# Tor Bug 21522
+Testing options override the deprecated option ClientDNSRejectInternalAddresses
+TestingTorNetwork is set
+# Older versions might need them, we should remove them at some point in 0.3.*
+The DirAuthority options 'hs' and 'no-hs' are obsolete
+# Chutney Bug 17090
+# Don't ignore it, we want to make sure there are no regressions
+##Tor is running as an exit relay
+# Tor Bug 21525?
+Unable to store signatures posted by .* Valid-After times do not match
+# We ignore consensus failure warnings: we set this low to bootstrap fast
+V3AuthVotingInterval is very low
+We don't have enough votes to generate a consensus
+You have used DirAuthority or AlternateDirAuthority
+Your log may contain sensitive information
+# Tor Bug 21525
+http status 400.*Bad valid-after time.*after uploading vote
+http status 400.*Valid-After times do not match.*after uploading signatures
diff --git a/tools/warnings.sh b/tools/warnings.sh
index 46d3f0a..26eb835 100755
--- a/tools/warnings.sh
+++ b/tools/warnings.sh
@@ -7,17 +7,36 @@
 # the warnings of that node.
 # Examples: tools/warnings.sh
 #           tools/warnings.sh 000a
+# Environmental variables:
+# CHUTNEY_WARNINGS_IGNORE_EXPECTED: set to 1 to filter out expected warnings
+# CHUTNEY_WARNINGS_SUMMARY: set to 1 to merge warnings from all instances
 
 if [ ! -z "$CHUTNEY_PATH" ]; then
     cd "$CHUTNEY_PATH"
 fi
 
 function show_warnings() {
-    echo "${GREEN}Node `basename $1`:${NC}"
+    if [ "$CHUTNEY_WARNINGS_SUMMARY" -ne 0 ]; then
+        echo "${GREEN}All `basename $1`:${NC}"
+        FILE="$1/*/$LOG_FILE"
+    else
+        echo "${GREEN}Node `basename $1`:${NC}"
+        FILE="$1/$LOG_FILE"
+    fi
+    if [ "$CHUTNEY_WARNINGS_IGNORE_EXPECTED" -ne 0 -a -e "$IGNORE_FILE" ]; then
+        CAT="grep -v -f"
+        echo " ${GREEN}(Ignoring expected warnings, run chutney/tools/warnings.sh to see all warnings)${NC}"
+    else
+        CAT=cat
+        IGNORE_FILE=
+    fi
     # Label errs as "Warning:", they're infrequent enough it doesn't matter
-    sed -n -E 's/^.*\[(warn|err)\]//p' $1/info.log | sort | uniq -c | \
+    $CAT $IGNORE_FILE $FILE | \
+    sed -n -E 's/^.*\[(warn|err)\]//p' | sort | uniq -c | \
     sed -e 's/^\s*//' -e "s/ *\([0-9][0-9]*\) *\(.*\)/ ${YELLOW}Warning:${NC} \2${YELLOW} Number: \1${NC}/"
-    echo ""
+    if [ "$CHUTNEY_WARNINGS_SUMMARY" -eq 0 ]; then
+        echo ""
+    fi
 }
 
 function usage() {
@@ -31,19 +50,31 @@ GREEN=$(tput setaf 2)
 CHUTNEY=./chutney
 NAME=$(basename "$0")
 DEST=net/nodes
+LOG_FILE=info.log
+# ignore warnings we expect to get every time chutney runs
+CHUTNEY_WARNINGS_IGNORE_EXPECTED=${CHUTNEY_WARNINGS_IGNORE_EXPECTED:-0}
+# don't put spaces in CHUTNEY_PATH or IGNORE_FILE
+IGNORE_FILE="$CHUTNEY_PATH/tools/ignore.warnings"
+# merge all log files into one before counting entries
+CHUTNEY_WARNINGS_SUMMARY=${CHUTNEY_WARNINGS_SUMMARY:-0}
 
 [ -d net/nodes ] || { echo "$NAME: no logs available"; exit 1; }
 if [ $# -eq 0 ];
 then
-    for dir in $DEST/*;
+    if [ "$CHUTNEY_WARNINGS_SUMMARY" -ne 0 ]; then
+        show_warnings "$DEST"
+        exit 0
+    fi
+    for dir in "$DEST"/*;
     do
-        [ -e ${dir}/info.log ] || continue
-        show_warnings $dir
+        [ -e "${dir}/$LOG_FILE" ] || continue
+        show_warnings "$dir"
     done
 elif [ $# -eq 1 ];
 then
-    [ -e $DEST/$1/info.log ] || { echo "$NAME: no log available"; exit 1; }
-    show_warnings $DEST/$1
+    [ -e "$DEST/$1/$LOG_FILE" ] || \
+        { echo "$NAME: no log available"; exit 1; }
+    show_warnings "$DEST/$1"
 else
     usage
 fi