[tor-commits] [tor/master] Add a script to check for coverage nondeterminism

[dgoulet at torproject.org]

commit f4c76661d1bb49a47829c6999a07c1de63b55cc9
Author: Nick Mathewson <nickm at torproject.org>
Date:   Fri Feb 8 11:21:09 2019 -0500

    Add a script to check for coverage nondeterminism
    
    Closes ticket 29436.
---
 changes/ticket29436                  |  4 +++
 scripts/test/cov-test-determinism.sh | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/changes/ticket29436 b/changes/ticket29436
new file mode 100644
index 000000000..025be619e
--- /dev/null
+++ b/changes/ticket29436
@@ -0,0 +1,4 @@
+  o Minor features (testing):
+    - We now have a script, cov-test-determinism.sh, to identify places
+      where our unit test coverage has become nondeterministic.
+      Closes ticket 29436.
diff --git a/scripts/test/cov-test-determinism.sh b/scripts/test/cov-test-determinism.sh
new file mode 100755
index 000000000..3b4f372e0
--- /dev/null
+++ b/scripts/test/cov-test-determinism.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# To use this script, build Tor with coverage enabled, and then say:
+#  ./scripts/test/cov-test-determinism.sh run
+#
+# Let it run for a long time so it can run the tests over and over.  It
+# will put their coverage outputs in coverage-raw/coverage-*/.
+#
+# Then say:
+#  ./scripts/test/cov-test-determinism.sh check
+#
+# It will diff the other coverage outputs to the first one, and put their
+# diffs in coverage-raw/diff-coverage-*.
+
+run=0
+check=0
+
+if test "$1" = run; then
+    run=1
+elif test "$1" = check; then
+    check=1
+else
+    echo "First use 'run' with this script, then use 'check'."
+    exit 1
+fi
+
+if test "$run" = 1; then
+    while true; do
+        make reset-gcov
+        CD=coverage-raw/coverage-$(date +%s)
+        make -j5 check
+        mkdir -p "$CD"
+        ./scripts/test/coverage "$CD"
+    done
+fi
+
+if test "$check" = 1; then
+    cd coverage-raw || exit 1
+
+    FIRST="$(find . -name "coverage-*" -type d | head -1)"
+    rm -f A
+    ln -sf "$FIRST" A
+    for dir in coverage-*; do
+        rm -f B
+        ln -sf "$dir" B
+        ../scripts/test/cov-diff A B > "diff-$dir"
+    done
+fi