[tor-commits] [tor/master] Add "make test-network-all" to verify multiple test networks

[nickm at torproject.org]

commit 5feae32f46b3f0cc3da93ece80163c1a994c83b9
Author: teor (Tim Wilson-Brown) <teor2345 at gmail.com>
Date:   Tue Sep 8 22:27:59 2015 +1000

    Add "make test-network-all" to verify multiple test networks
    
    make test-network-all is Makefile target which verifies a series
    of test networks generated using test-network.sh and chutney.
    
    It runs IPv6 and mixed version test networks if the prerequisites are
    available.
    
    Each test network reports PASS, FAIL, or SKIP.
    Closes ticket 16953. Patch by "teor".
    
    Also adds "--hs-multi-client 1" option to TEST_NETWORK_FLAGS.
    This resolves #17012.
    
    Larger networks, such as bridges+hs, may fail until #16952 is merged.
---
 Makefile.am         |   38 ++++++++++++++++++++++++++++++++++++--
 src/test/include.am |    7 +++++++
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index b5e71aa..68695ed 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -36,13 +36,16 @@ EXTRA_DIST+= \
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
-TEST_NETWORK_FLAGS="--coverage"
+TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS
-TEST_NETWORK_FLAGS=
+TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
 
+TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
+TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
+
 #install-data-local:
 #	$(INSTALL) -m 755 -d $(LOCALSTATEDIR)/lib/tor
 
@@ -88,6 +91,36 @@ need-chutney-path:
 test-network: need-chutney-path all
 	$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
 
+# Run all available tests using automake's test-driver
+# only run IPv6 tests if we can ping6 ::1 (localhost)
+# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
+# only run mixed tests if we have a tor-stable binary
+# see #17015 for autodetection of different tor versions
+test-network-all: need-chutney-path all test-driver
+	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
+	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
+	if ping6 -q -o ::1 >/dev/null 2>&1; then \
+		echo "ping6 ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+	else \
+		echo "ping6 ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+	fi; \
+	if command -v tor-stable >/dev/null 2>&1; then \
+		echo "tor-stable found, running mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
+		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+	else \
+		echo "tor-stable not found, skipping mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
+		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+	fi; \
+	for f in $$skip_flavors; do \
+		echo "SKIP: $$f"; \
+	done; \
+	for f in $$flavors; do \
+		./test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
+	done
+	@echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."
+
 need-stem-path:
 	@if test ! -d "$$STEM_SOURCE_DIR"; then \
 		echo '$$STEM_SOURCE_DIR was not set.'; echo; \
@@ -173,3 +206,4 @@ mostlyclean-local:
 	rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no}
 	rm -rf $(HTML_COVER_DIR)
 	rm -rf $(top_builddir)/doc/doxygen
+	rm -rf $(TEST_NETWORK_ALL_LOG_DIR)
diff --git a/src/test/include.am b/src/test/include.am
index ef2fd4f..f7c0204 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -8,6 +8,13 @@ endif
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue src/test/test_keygen.sh $(TESTSCRIPTS)
 
+# These flavors are run using automake's test-driver and test-network.sh
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-min bridges+hs
+# only run if we can ping6 ::1 (localhost)
+TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min
+# only run if we can find a stable (or simply another) version of tor
+TEST_CHUTNEY_FLAVORS_MIXED = mixed
+
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.  Below is a kludge to work around.
 ###