commit f04f297ad4e85a5345e56b0c3bf24186d80875b3 Author: Nick Mathewson nickm@torproject.org Date: Fri May 10 09:36:51 2019 -0400
Fix up chutney verifier's notion of a test suite
Previously we counted the number of successes and failures, and compared them to the number of expected tests. That's fragile, if we forget to count a test or if we count a success more than once. Now we give every test a name, and we only allow tests to succeed or fail once. --- lib/chutney/Traffic.py | 69 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 21 deletions(-)
diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 50364c6..f877452 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -72,20 +72,30 @@ class TestSuite(object): and how many have succeeded."""
def __init__(self): + self.tests = {} self.not_done = 0 self.successes = 0 self.failures = 0
- def add(self): - self.not_done += 1 - - def success(self): - self.not_done -= 1 - self.successes += 1 - - def failure(self): - self.not_done -= 1 - self.failures += 1 + def add(self, name): + if name not in self.tests: + debug("Registering %s"%name) + self.not_done += 1 + self.tests[name] = 'not done' + + def success(self, name): + if self.tests[name] == 'not done': + debug("Succeeded %s"%name) + self.tests[name] = 'success' + self.not_done -= 1 + self.successes += 1 + + def failure(self, name): + if self.tests[name] == 'not done': + debug("Failed %s"%name) + self.tests[name] = 'failure' + self.not_done -= 1 + self.failures += 1
def failure_count(self): return self.failures @@ -94,7 +104,9 @@ class TestSuite(object): return self.not_done == 0
def status(self): - return('%d/%d/%d' % (self.not_done, self.successes, self.failures)) + return('%s: %d/%d/%d' % (self.tests, self.not_done, self.successes, + self.failures)) +
class Listener(asyncore.dispatcher): "A TCP listener, binding, listening and accepting new connections." @@ -115,6 +127,7 @@ class Listener(asyncore.dispatcher): debug("new client from %s:%s (fd=%d)" % (endpoint[0], endpoint[1], newsock.fileno())) handler = Sink(newsock, self.tt) + self.tt.add(handler)
def fileno(self): return self.socket.fileno() @@ -127,6 +140,10 @@ class Sink(asynchat.async_chat): self.set_terminator(None) self.tt = tt self.repetitions = tt.repetitions + self.testname = "recv-data%s"%id(self) + + def get_test_names(self): + return [ self.testname ]
def collect_incoming_data(self, inp): # shortcut read when we don't ever expect any data @@ -138,7 +155,7 @@ class Sink(asynchat.async_chat): assert(len(self.inbuf) <= len(data) or self.repetitions > 1) if self.inbuf[:len(data)] != data: debug("receive comparison failed (bytes=%d)" % len(data)) - self.tt.failure() + self.tt.failure(self.testname) self.close() # if we're not debugging, print a dot every dot_repetitions reps elif (not debug_flag and self.tt.dot_repetitions > 0 and @@ -154,7 +171,7 @@ class Sink(asynchat.async_chat): if self.repetitions == 0 and len(self.inbuf) == 0: debug("successful verification") self.close() - self.tt.success() + self.tt.success(self.testname) # calculate the actual length of data remaining, including reps debug("receive remaining bytes (bytes=%d)" % (self.repetitions*len(data) - len(self.inbuf))) @@ -168,7 +185,8 @@ class CloseSourceProducer: self.source = source
def more(self): - self.source.tt.success() + self.source.sent_ok() + return b""
class Source(asynchat.async_chat): """A data source, connecting to a TCP server, optionally over a @@ -187,6 +205,8 @@ class Source(asynchat.async_chat): self.repetitions = repetitions self._sent_no_bytes = 0 self.tt = tt + self.testname = "send-data%s"%id(self) + # sanity checks if len(self.data) == 0: self.repetitions = 0 @@ -200,6 +220,12 @@ class Source(asynchat.async_chat): self.state = self.CONNECTING self.connect(dest)
+ def get_test_names(self): + return [ self.testname ] + + def sent_ok(self): + self.tt.success(self.testname) + def handle_connect(self): if self.proxy: self.state = self.CONNECTING_THROUGH_PROXY @@ -264,19 +290,20 @@ class TrafficTester(object): debug("listener fd=%d" % self.listener.fileno())
def add(self, item): - """Register a single item as a test.""" + """Register a single item.""" # We used to hold on to these items for their fds, but now # asyncore manages them for us. + if hasattr(item, "get_test_names"): + for name in item.get_test_names(): + self.tests.add(name)
- self.tests.add() - - def success(self): + def success(self, name): """Declare that a single test has passed.""" - self.tests.success() + self.tests.success(name)
- def failure(self): + def failure(self, name): """Declare that a single test has failed.""" - self.tests.failure() + self.tests.failure(name)
def run(self): start = now = time.time()