June 2019 - tor-commits - lists.torproject.org

[chutney/master] Refactor Source and Sink to not know about their data
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 1bbe2079c40ec6d6517c2125274e9b9510c09197 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 10:12:00 2019 -0400 Refactor Source and Sink to not know about their data Now data is generated by a DataSource type, and consumed by a DataChecker type that compares its incoming data against the data in a DataSource. --- lib/chutney/Traffic.py | 113 +++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 45 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index f877452..72b9ca3 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -107,7 +107,6 @@ class TestSuite(object): return('%s: %d/%d/%d' % (self.tests, self.not_done, self.successes, self.failures)) - class Listener(asyncore.dispatcher): "A TCP listener, binding, listening and accepting new connections." @@ -132,14 +131,69 @@ class Listener(asyncore.dispatcher): def fileno(self): return self.socket.fileno() +class DataSource(object): + """A data source generates some number of bytes of data, and then + returns None. + + For convenience, it conforms to the 'producer' api. + """ + def __init__(self, data, repetitions=1): + self.data = data + self.repetitions = repetitions + self.sent_any = False + + def copy(self): + assert not self.sent_any + return DataSource(self.data, self.repetitions) + + def more(self): + self.sent_any = True + if self.repetitions > 0: + self.repetitions -= 1 + return self.data + + return None + +class DataChecker(object): + """A data checker verifies its input against bytes in a stream.""" + def __init__(self, source): + self.source = source + self.pending = b"" + self.succeeded = False + self.failed = False + + def consume(self, inp): + if self.failed: + return + if self.succeeded and len(inp): + self.succeeded = False + self.failed = True + return + + while len(inp): + n = min(len(inp), len(self.pending)) + if inp[:n] != self.pending[:n]: + self.failed = True + return + inp = inp[n:] + self.pending = self.pending[n:] + if not self.pending: + self.pending = self.source.more() + + if self.pending is None: + if len(inp): + self.failed = True + else: + self.succeeded = True + return + class Sink(asynchat.async_chat): "A data sink, reading from its peer and verifying the data." def __init__(self, sock, tt): asynchat.async_chat.__init__(self, sock) - self.inbuf = b"" self.set_terminator(None) self.tt = tt - self.repetitions = tt.repetitions + self.data_checker = DataChecker(tt.data_source.copy()) self.testname = "recv-data%s"%id(self) def get_test_names(self): @@ -148,33 +202,16 @@ class Sink(asynchat.async_chat): def collect_incoming_data(self, inp): # shortcut read when we don't ever expect any data - self.inbuf += inp - data = self.tt.data - debug("successfully received (bytes=%d)" % len(self.inbuf)) - while len(self.inbuf) >= len(data): - assert(len(self.inbuf) <= len(data) or self.repetitions > 1) - if self.inbuf[:len(data)] != data: - debug("receive comparison failed (bytes=%d)" % len(data)) - self.tt.failure(self.testname) - self.close() - # if we're not debugging, print a dot every dot_repetitions reps - elif (not debug_flag and self.tt.dot_repetitions > 0 and - self.repetitions % self.tt.dot_repetitions == 0): - sys.stdout.write('.') - sys.stdout.flush() - # repeatedly check data against self.inbuf if required - debug("receive comparison success (bytes=%d)" % len(data)) - self.inbuf = self.inbuf[len(data):] - debug("receive leftover bytes (bytes=%d)" % len(self.inbuf)) - self.repetitions -= 1 - debug("receive remaining repetitions (reps=%d)" % self.repetitions) - if self.repetitions == 0 and len(self.inbuf) == 0: + debug("successfully received (bytes=%d)" % len(inp)) + self.data_checker.consume(inp) + if self.data_checker.succeeded: debug("successful verification") self.close() self.tt.success(self.testname) - # calculate the actual length of data remaining, including reps - debug("receive remaining bytes (bytes=%d)" - % (self.repetitions*len(data) - len(self.inbuf))) + elif self.data_checker.failed: + debug("receive comparison failed") + self.tt.failure(self.testname) + self.close() def fileno(self): return self.socket.fileno() @@ -197,22 +234,13 @@ class Source(asynchat.async_chat): def __init__(self, tt, server, buf, proxy=None, repetitions=1): asynchat.async_chat.__init__(self) - self.data = buf - self.outbuf = b'' + self.data_source = DataSource(buf, repetitions) self.inbuf = b'' self.proxy = proxy self.server = server - self.repetitions = repetitions - self._sent_no_bytes = 0 self.tt = tt self.testname = "send-data%s"%id(self) - # sanity checks - if len(self.data) == 0: - self.repetitions = 0 - if self.repetitions == 0: - self.data = b"" - self.set_terminator(None) dest = (self.proxy or self.server) self.create_socket(addr_to_family(dest[0]), socket.SOCK_STREAM) @@ -251,8 +279,7 @@ class Source(asynchat.async_chat): self.close() def push_output(self): - for _ in range(self.repetitions): - self.push_with_producer(asynchat.simple_producer(self.data)) + self.push_with_producer(self.data_source) self.push_with_producer(CloseSourceProducer(self)) self.close_when_done() @@ -279,13 +306,9 @@ class TrafficTester(object): self.pending_close = [] self.timeout = timeout self.tests = TestSuite() - self.data = data - self.repetitions = repetitions + self.data_source = DataSource(data, repetitions) + # sanity checks - if len(self.data) == 0: - self.repetitions = 0 - if self.repetitions == 0: - self.data = b"" self.dot_repetitions = dot_repetitions debug("listener fd=%d" % self.listener.fileno())

1 0

[chutney/master] Store the canonical DataSource in the TrafficTester.
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit b9668e4d5ddbf1a8edcd2f3e768d9b46b5ac4404 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 11:15:21 2019 -0400 Store the canonical DataSource in the TrafficTester. --- lib/chutney/Traffic.py | 6 +++--- scripts/chutney_tests/verify.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 72b9ca3..7ef3b2f 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -232,9 +232,9 @@ class Source(asynchat.async_chat): CONNECTING_THROUGH_PROXY = 2 CONNECTED = 5 - def __init__(self, tt, server, buf, proxy=None, repetitions=1): + def __init__(self, tt, server, proxy=None): asynchat.async_chat.__init__(self) - self.data_source = DataSource(buf, repetitions) + self.data_source = tt.data_source.copy() self.inbuf = b'' self.proxy = proxy self.server = server @@ -355,7 +355,7 @@ def main(): tt = TrafficTester(bind_to, DATA) # Don't use a proxy for self-testing, so that we avoid tor entirely - tt.add(Source(tt, bind_to, DATA)) + tt.add(Source(tt, bind_to)) success = tt.run() if success: diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py index 3e0455d..43100d6 100644 --- a/scripts/chutney_tests/verify.py +++ b/scripts/chutney_tests/verify.py @@ -155,8 +155,7 @@ def _configure_exits(tt, bind_to, tmpdata, reps, client_list, exit_list, 'localhost', op._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(op._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata, proxy, - reps)) + tt.add(chutney.Traffic.Source(tt, bind_to, proxy)) return exit_path_node_count @@ -188,8 +187,8 @@ def _configure_hs(tt, tmpdata, reps, client_list, hs_list, HS_PORT, 'localhost', client._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(client._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, hs_bind_to, tmpdata, - proxy, reps)) + tt.add(chutney.Traffic.Source(tt, hs_bind_to, proxy)) + return hs_path_node_count

1 0

[chutney/master] Don't check whether we can write on a listener
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 0687600c104d69890e4b67cffb73c8ede83ec91e Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 15:22:00 2019 -0400 Don't check whether we can write on a listener --- lib/chutney/Traffic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 2f4cc94..7199f87 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -125,6 +125,9 @@ class Listener(asyncore.dispatcher): self.listen(0) self.tt = tt + def writable(self): + return False + def handle_accept(self): # deprecated in python 3.2 pair = self.accept()

1 0

[chutney/master] Make TrafficTester create sources and sinks
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 58c5a3c69dde2f057e82fb64aa4932acbcdd0840 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 11:24:50 2019 -0400 Make TrafficTester create sources and sinks We'd like to be able to change the client and responder behavior, but we need to keep both parts synchronized as we do so. --- lib/chutney/Traffic.py | 13 ++++++++++--- scripts/chutney_tests/verify.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 7ef3b2f..de21d60 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -125,8 +125,7 @@ class Listener(asyncore.dispatcher): newsock, endpoint = pair debug("new client from %s:%s (fd=%d)" % (endpoint[0], endpoint[1], newsock.fileno())) - handler = Sink(newsock, self.tt) - self.tt.add(handler) + self.tt.add_responder(newsock) def fileno(self): return self.socket.fileno() @@ -320,6 +319,14 @@ class TrafficTester(object): for name in item.get_test_names(): self.tests.add(name) + def add_client(self, server, proxy=None): + source = Source(self, server, proxy) + self.add(source) + + def add_responder(self, socket): + sink = Sink(socket, self) + self.add(sink) + def success(self, name): """Declare that a single test has passed.""" self.tests.success(name) @@ -355,7 +362,7 @@ def main(): tt = TrafficTester(bind_to, DATA) # Don't use a proxy for self-testing, so that we avoid tor entirely - tt.add(Source(tt, bind_to)) + tt.add_client(bind_to) success = tt.run() if success: diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py index 43100d6..e97ff52 100644 --- a/scripts/chutney_tests/verify.py +++ b/scripts/chutney_tests/verify.py @@ -155,7 +155,7 @@ def _configure_exits(tt, bind_to, tmpdata, reps, client_list, exit_list, 'localhost', op._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(op._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, bind_to, proxy)) + tt.add_client(bind_to, proxy) return exit_path_node_count @@ -187,7 +187,7 @@ def _configure_hs(tt, tmpdata, reps, client_list, hs_list, HS_PORT, 'localhost', client._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(client._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, hs_bind_to, proxy)) + tt.add_client(hs_bind_to, proxy) return hs_path_node_count

1 0

[chutney/master] re-add not-connected to handle some failure cases
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 4c0e3f31df3167eea236e0f6c19fc80e757a906d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 13:56:39 2019 -0400 re-add not-connected to handle some failure cases --- lib/chutney/Traffic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 167647b..425d08d 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -227,6 +227,7 @@ class CloseSourceProducer: class Source(asynchat.async_chat): """A data source, connecting to a TCP server, optionally over a SOCKS proxy, sending data.""" + NOT_CONNECTED = 0 CONNECTING = 1 CONNECTING_THROUGH_PROXY = 2 CONNECTED = 5

1 0

[chutney/master] Don't use "ord" to debug a byte; python3 hates it.
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 8c8cbf50dad79eb4233c0962a93f882a09c70b9d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 14:15:09 2019 -0400 Don't use "ord" to debug a byte; python3 hates it. --- lib/chutney/Traffic.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 425d08d..2f4cc94 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -34,6 +34,13 @@ import asynchat from chutney.Debug import debug_flag, debug +if sys.version_info[0] >= 3: + def byte_to_int(b): + return b +else: + def byte_to_int(b): + return ord(b) + def addr_to_family(addr): for family in [socket.AF_INET, socket.AF_INET6]: try: @@ -274,7 +281,7 @@ class Source(asynchat.async_chat): self.push_output() else: debug("proxy handshake failed (0x%x)! (fd=%d)" % - (ord(self.inbuf[1]), self.fileno())) + (byte_to_int(self.inbuf[1]), self.fileno())) self.state = self.NOT_CONNECTED self.close()

1 0

[chutney/master] Make chutney bidirectional by default
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 36a3ee88f19391fb3525454ba527a2479010c85d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 11:44:10 2019 -0400 Make chutney bidirectional by default The default client/responder pair is no longer source/sink, but echoclient/echoserver. --- lib/chutney/Traffic.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index de21d60..167647b 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -286,6 +286,49 @@ class Source(asynchat.async_chat): def fileno(self): return self.socket.fileno() +class EchoServer(asynchat.async_chat): + def __init__(self, sock, tt): + asynchat.async_chat.__init__(self, sock) + self.set_terminator(None) + self.tt = tt + + def collect_incoming_data(self, data): + self.push(data) + + def handle_close(self): + self.close_when_done() + +class EchoClient(Source): + def __init__(self, tt, server, proxy=None): + Source.__init__(self, tt, server, proxy) + self.data_checker = DataChecker(tt.data_source.copy()) + self.testname_check = "check-%s"%id(self) + + def get_test_names(self): + return [ self.testname, self.testname_check ] + + def handle_close(self): + self.close_when_done() + + def collect_incoming_data(self, data): + if self.state == self.CONNECTING_THROUGH_PROXY: + Source.collect_incoming_data(self, data) + if self.state == self.CONNECTING_THROUGH_PROXY: + return + data = self.inbuf + self.inbuf = b"" + + self.data_checker.consume(data) + + if self.data_checker.succeeded: + debug("successful verification") + self.close() + self.tt.success(self.testname_check) + elif self.data_checker.failed: + debug("receive comparison failed") + self.tt.failure(self.testname_check) + self.close() + class TrafficTester(object): """ Hang on select.select() and dispatch to Sources and Sinks. @@ -300,7 +343,15 @@ class TrafficTester(object): data=b"", timeout=3, repetitions=1, - dot_repetitions=0): + dot_repetitions=0, + chat_type="Echo"): + if chat_type == "Echo": + self.client_class = EchoClient + self.responder_class = EchoServer + else: + self.client_class = Source + self.responder_class = Sink + self.listener = Listener(self, endpoint) self.pending_close = [] self.timeout = timeout @@ -320,11 +371,11 @@ class TrafficTester(object): self.tests.add(name) def add_client(self, server, proxy=None): - source = Source(self, server, proxy) + source = self.client_class(self, server, proxy) self.add(source) def add_responder(self, socket): - sink = Sink(socket, self) + sink = self.responder_class(socket, self) self.add(sink) def success(self, name):

1 0

[chutney/master] Log test status less frequently: travis thinks it is too verbose.
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit c5184442a3695d0dad36d7f310d047e5e95ba0c1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 15:24:26 2019 -0400 Log test status less frequently: travis thinks it is too verbose. --- lib/chutney/Traffic.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 7199f87..f2448a8 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -400,12 +400,16 @@ class TrafficTester(object): def run(self): start = now = time.time() end = time.time() + self.timeout + DUMP_TEST_STATUS_INTERVAL=0.5 + dump_at = start+DUMP_TEST_STATUS_INTERVAL while now < end and not self.tests.all_done(): # run only one iteration at a time, with a nice short timeout, so we # can actually detect completion and timeouts. asyncore.loop(0.2, False, None, 1) now = time.time() - debug("Test status: %s"%self.tests.status()) + if now > dump_at: + debug("Test status: %s"%self.tests.status()) + dump_at += DUMP_TEST_STATUS_INTERVAL if not debug_flag: sys.stdout.write('\n')

1 0

[chutney/master] Be a little more verbose about test success/failure
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit eedc7e583362601aac35a2608cdf28254eb229dc Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat May 11 10:50:23 2019 -0400 Be a little more verbose about test success/failure Additionally, give them nicer unique names, and warn if we accidentally double-add a test or something. --- lib/chutney/Traffic.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index ed4ad69..1252840 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -34,6 +34,17 @@ import asynchat from chutney.Debug import debug_flag, debug +def note(s): + sys.stderr.write("NOTE: %s\n"%s) +def warn(s): + sys.stderr.write("WARN: %s\n"%s) + +UNIQ_CTR = 0 +def uniq(s): + global UNIQ_CTR + UNIQ_CTR += 1 + return "%s-%s"%(s,UNIQ_CTR) + if sys.version_info[0] >= 3: def byte_to_int(b): return b @@ -85,24 +96,33 @@ class TestSuite(object): self.failures = 0 def add(self, name): + note("Registering %s"%name) if name not in self.tests: debug("Registering %s"%name) self.not_done += 1 self.tests[name] = 'not done' + else: + warn("... already registered!") def success(self, name): + note("Success for %s"%name) if self.tests[name] == 'not done': debug("Succeeded %s"%name) self.tests[name] = 'success' self.not_done -= 1 self.successes += 1 + else: + warn("... status was %s"%self.tests.get(name)) def failure(self, name): + note("Failure for %s"%name) if self.tests[name] == 'not done': debug("Failed %s"%name) self.tests[name] = 'failure' self.not_done -= 1 self.failures += 1 + else: + warn("... status was %s"%self.tests.get(name)) def failure_count(self): return self.failures @@ -203,7 +223,7 @@ class Sink(asynchat.async_chat): self.set_terminator(None) self.tt = tt self.data_checker = DataChecker(tt.data_source.copy()) - self.testname = "recv-data%s"%id(self) + self.testname = uniq("recv-data") def get_test_names(self): return [ self.testname ] @@ -249,7 +269,7 @@ class Source(asynchat.async_chat): self.proxy = proxy self.server = server self.tt = tt - self.testname = "send-data%s"%id(self) + self.testname = uniq("send-data") self.set_terminator(None) dest = (self.proxy or self.server) @@ -313,7 +333,7 @@ class EchoClient(Source): def __init__(self, tt, server, proxy=None): Source.__init__(self, tt, server, proxy) self.data_checker = DataChecker(tt.data_source.copy()) - self.testname_check = "check-%s"%id(self) + self.testname_check = uniq("check") def get_test_names(self): return [ self.testname, self.testname_check ]

1 0

[chutney/master] Stop trying to use half-open connections: Tor doesn't support them
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 15dcdee6bf4516b7bb017c197c747c3308f3f8b6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun May 12 10:42:45 2019 -0400 Stop trying to use half-open connections: Tor doesn't support them --- lib/chutney/Traffic.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index f403ca0..02cca91 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -322,7 +322,6 @@ class Source(asynchat.async_chat): self.push_with_producer(self.data_source) self.push_with_producer(CloseSourceProducer(self)) - self.close_when_done() def fileno(self): return self.socket.fileno() @@ -332,18 +331,17 @@ class EchoServer(asynchat.async_chat): asynchat.async_chat.__init__(self, sock, map=tt.socket_map) self.set_terminator(None) self.tt = tt + self.am_closing = False def collect_incoming_data(self, data): self.push(data) - def handle_close(self): - self.close_when_done() - class EchoClient(Source): def __init__(self, tt, server, proxy=None): Source.__init__(self, tt, server, proxy) self.data_checker = DataChecker(tt.data_source.copy()) self.testname_check = uniq("check") + self.am_closing = False def enote(self, s): self.tt.tests.note(self.testname_check, s) @@ -351,9 +349,6 @@ class EchoClient(Source): def get_test_names(self): return [ self.testname, self.testname_check ] - def handle_close(self): - self.close_when_done() - def collect_incoming_data(self, data): if self.state == self.CONNECTING_THROUGH_PROXY: Source.collect_incoming_data(self, data) @@ -455,7 +450,7 @@ class TrafficTester(object): %(self.tests.all_done(), self.tests.failure_count())) note("Status:\n%s"%self.tests.teststatus) - + self.listener.close() return self.tests.all_done() and self.tests.failure_count() == 0

1 0