commit 95ce144c2615dd9400a41d8374e56683ad9975b6 Author: teor teor@torproject.org Date: Thu Mar 12 13:38:38 2020 +1000
verify: Wait for a consistent amount of time
Wait for a consistent amount of time between verify attempts, regardless of the time elapsed during the attempt.
Try to make at least 2 verify attempts per consensus interval, but also try to limit the number of attempts.
Also increase the default timeout: sometimes 3 seconds is too short on a heavily-loaded machine.
These changes should make verification more reliable.
Part of 33379. --- lib/chutney/Traffic.py | 2 +- scripts/chutney_tests/verify.py | 31 ++++++++++++++++++++++--------- torrc_templates/authority.i | 2 ++ 3 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 9990dcd..d8918ee 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -440,7 +440,7 @@ class TrafficTester(object): while now < end and not self.tests.all_done(): # run only one iteration at a time, with a nice short timeout, so we # can actually detect completion and timeouts. - asyncore.loop(0.2, False, self.socket_map, 1) + asyncore.loop(5.0, False, self.socket_map, 1) now = time.time() if now > dump_at: debug("Test status: %s"%self.tests.status()) diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py index a0f0c3e..2f048e0 100644 --- a/scripts/chutney_tests/verify.py +++ b/scripts/chutney_tests/verify.py @@ -7,6 +7,9 @@ import time import chutney
def run_test(network): + # Try to verify twice each consensus + VERIFY_ATTEMPT_INTERVAL = network.V3_AUTH_VOTING_INTERVAL/2.0 - 1.0 + TIMEOUT_INTERVAL = max(VERIFY_ATTEMPT_INTERVAL - 1.0, 5.0) wait_time = network._dfltEnv['bootstrap_time'] start_time = time.time() end_time = start_time + wait_time @@ -14,20 +17,28 @@ def run_test(network): % wait_time) status = False # Keep on retrying the verify until it succeeds or times out - while not status and time.time() < end_time: + now = start_time + while not status and now < end_time: # TrafficTester connections time out after ~3 seconds - # a TrafficTester times out after ~10 seconds if no data is being sent - status = _verify_traffic(network) + # a TrafficTester times out after ~6 seconds if no data is being sent + last_attempt_time = now + status = _verify_traffic(network, timeout=TIMEOUT_INTERVAL) + now = time.time() + elapsed_attempt_time = now - last_attempt_time # Avoid madly spewing output if we fail immediately each time if not status: - time.sleep(5) + # We want at least 2 verify attempts per consensus interval + sleep_time = VERIFY_ATTEMPT_INTERVAL - elapsed_attempt_time + if sleep_time > 0: + time.sleep(sleep_time) + now = time.time() print("Transmission: %s" % ("Success" if status else "Failure")) if not status: print("Set CHUTNEY_DEBUG to diagnose.") return status
-def _verify_traffic(network): +def _verify_traffic(network, timeout=5.0): """Verify (parts of) the network by sending traffic through it and verify what is received.""" # TODO: IPv6 SOCKSPorts, SOCKSPorts with IPv6Traffic, and IPv6 Exits @@ -43,8 +54,7 @@ def _verify_traffic(network): # and a source-sink pair for a (bridge) client to each hidden service DATALEN = network._dfltEnv['data_bytes'] # Print a dot each time a sink verifies this much data - DOTDATALEN = 5 * 1024 * 1024 # Octets. - TIMEOUT = 3 # Seconds. + DOTDATALEN = 5 * 1024 * 1024 # Octets # Calculate the amount of random data we should use randomlen = _calculate_randomlen(DATALEN) reps = _calculate_reps(DATALEN, randomlen) @@ -65,8 +75,11 @@ def _verify_traffic(network): tmpdata = {} # now make the connections bind_to = (LISTEN_ADDR, LISTEN_PORT) - tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT, reps, - dot_reps) + tt = chutney.Traffic.TrafficTester(bind_to, + data=tmpdata, + timeout=timeout, + repetitions=reps, + dot_repetitions=dot_reps) # _env does not implement get() due to its fallback to parent behaviour client_list = filter(lambda n: n._env['tag'].startswith('c') or diff --git a/torrc_templates/authority.i b/torrc_templates/authority.i index e307bf5..e44b0b6 100644 --- a/torrc_templates/authority.i +++ b/torrc_templates/authority.i @@ -21,6 +21,8 @@ TestingV3AuthInitialVoteDelay 4 TestingV3AuthInitialDistDelay 4 # Vote + Dist must be less than Interval/2, because when there's no consensus, # tor uses Interval/2 as the voting interval +# +# Keep in sync with scripts/chutney_tests/verify.py V3_AUTH_VOTING_INTERVAL V3AuthVotingInterval 20 V3AuthVoteDelay 4 V3AuthDistDelay 4