[tor-commits] [chutney/master] Make chutney verify retry until success

[teor at torproject.org]

commit 1592d53a91a4b9421ad9eb5567ba3028a0b85fe4
Author: teor (Tim Wilson-Brown) <teor2345 at gmail.com>
Date:   Mon Jul 18 12:51:42 2016 +1000

    Make chutney verify retry until success
    
    Return immediately on success.
    Only return failure after 60 seconds with no successes.
    
    This makes networks that bootstrap quickly verify much faster,
    and networks that bootstrap slowly are much more reliable.
---
 README                          | 10 ++++++++++
 lib/chutney/TorNet.py           |  7 +++++++
 scripts/chutney_tests/verify.py | 16 ++++++++++++++--
 tools/test-network.sh           | 15 +++++++--------
 4 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/README b/README
index f945d91..826e5dd 100644
--- a/README
+++ b/README
@@ -64,6 +64,16 @@ HS Connection Tests:
   # Default behavior is one client connects to each HS
   ./chutney stop networks/hs-025
 
+Waiting for the network to bootstrap:
+
+  Commands like "chutney verify" start immediately, and keep trying for
+  CHUTNEY_BOOTSTRAP_TIME seconds. If they haven't been successful after that
+  time, they fail.
+
+  The tools/test-network.sh script waits 15 seconds before calling chutney
+  verify, because that's the minimum amount of time it takes to bootstrap a
+  consensus containing relays.
+
 Changing the network address:
 
    Chutney defaults to binding to localhost. To change the bind address,
diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py
index 87c5dab..12fc19e 100644
--- a/lib/chutney/TorNet.py
+++ b/lib/chutney/TorNet.py
@@ -700,6 +700,13 @@ DEFAULTS = {
     # to connect to each HS?
     # (Clients choose an exit at random, so this doesn't apply to exits.)
     'hs_multi_client': int(os.environ.get('CHUTNEY_HS_MULTI_CLIENT', 0)),
+    # How long should verify (and similar commands) wait for a successful
+    # outcome? (seconds)
+    # We check BOOTSTRAP_TIME for compatibility with old versions of
+    # test-network.sh
+    'bootstrap_time': int(os.environ.get('CHUTNEY_BOOTSTRAP_TIME',
+                                         os.environ.get('BOOTSTRAP_TIME',
+                                                        60))),
 }
 
 
diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py
index 480cf97..8734b60 100644
--- a/scripts/chutney_tests/verify.py
+++ b/scripts/chutney_tests/verify.py
@@ -3,8 +3,20 @@ import chutney
 
 
 def run_test(network):
-    print("Verifying data transmission:")
-    status = _verify_traffic(network)
+    wait_time = network._dfltEnv['bootstrap_time']
+    start_time = time.time()
+    end_time = start_time + wait_time
+    print("Verifying data transmission: (retrying for up to %d seconds)"
+          % wait_time)
+    status = False
+    # Keep on retrying the verify until it succeeds or times out
+    while not status and time.time() < end_time:
+        # TrafficTester connections time out after ~3 seconds
+        # a TrafficTester times out after ~10 seconds if no data is being sent
+        status = _verify_traffic(network)
+        # Avoid madly spewing output if we fail immediately each time
+        if not status:
+            time.sleep(2)
     print("Transmission: %s" % ("Success" if status else "Failure"))
     if not status:
         # TODO: allow the debug flag to be passed as an argument to
diff --git a/tools/test-network.sh b/tools/test-network.sh
index 6707e2d..38a0477 100755
--- a/tools/test-network.sh
+++ b/tools/test-network.sh
@@ -21,7 +21,7 @@ do
       shift
     ;;
     --delay|--sleep|--bootstrap-time|--time)
-      export BOOTSTRAP_TIME="$2"
+      export CHUTNEY_BOOTSTRAP_TIME="$2"
       shift
     ;;
     # Environmental variables used by chutney verify performance tests
@@ -137,13 +137,12 @@ fi
 cd "$CHUTNEY_PATH"
 ./tools/bootstrap-network.sh $NETWORK_FLAVOUR || exit 2
 
-# Sleep some, waiting for the network to bootstrap.
-# TODO: Add chutney command 'bootstrap-status' and use that instead.
-BOOTSTRAP_TIME=${BOOTSTRAP_TIME:-35}
-$ECHO_N "$myname: sleeping for $BOOTSTRAP_TIME seconds"
-n=$BOOTSTRAP_TIME; while [ $n -gt 0 ]; do
-    sleep 1; n=$(expr $n - 1); $ECHO_N .
-done; echo ""
+# chutney verify starts immediately, and keeps on trying for 60 seconds
+CHUTNEY_BOOTSTRAP_TIME=${CHUTNEY_BOOTSTRAP_TIME:-60}
+# but even the fastest tor networks take 5 seconds for their first consensus
+# and then 10 seconds after that for relays to bootstrap and upload descriptors
+echo "Waiting 15 seconds for a consensus containing relaysto be generated..."
+sleep 15
 ./chutney verify $CHUTNEY_NETWORK
 VERIFY_EXIT_STATUS=$?
 # work around a bug/feature in make -j2 (or more)