tor-commits June 2019

tor-commits@lists.torproject.org

21 participants
1482 discussions

[chutney/master] Store the canonical DataSource in the TrafficTester.
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit b9668e4d5ddbf1a8edcd2f3e768d9b46b5ac4404 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 11:15:21 2019 -0400 Store the canonical DataSource in the TrafficTester. --- lib/chutney/Traffic.py | 6 +++--- scripts/chutney_tests/verify.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 72b9ca3..7ef3b2f 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -232,9 +232,9 @@ class Source(asynchat.async_chat): CONNECTING_THROUGH_PROXY = 2 CONNECTED = 5 - def __init__(self, tt, server, buf, proxy=None, repetitions=1): + def __init__(self, tt, server, proxy=None): asynchat.async_chat.__init__(self) - self.data_source = DataSource(buf, repetitions) + self.data_source = tt.data_source.copy() self.inbuf = b'' self.proxy = proxy self.server = server @@ -355,7 +355,7 @@ def main(): tt = TrafficTester(bind_to, DATA) # Don't use a proxy for self-testing, so that we avoid tor entirely - tt.add(Source(tt, bind_to, DATA)) + tt.add(Source(tt, bind_to)) success = tt.run() if success: diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py index 3e0455d..43100d6 100644 --- a/scripts/chutney_tests/verify.py +++ b/scripts/chutney_tests/verify.py @@ -155,8 +155,7 @@ def _configure_exits(tt, bind_to, tmpdata, reps, client_list, exit_list, 'localhost', op._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(op._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata, proxy, - reps)) + tt.add(chutney.Traffic.Source(tt, bind_to, proxy)) return exit_path_node_count @@ -188,8 +187,8 @@ def _configure_hs(tt, tmpdata, reps, client_list, hs_list, HS_PORT, 'localhost', client._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(client._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, hs_bind_to, tmpdata, - proxy, reps)) + tt.add(chutney.Traffic.Source(tt, hs_bind_to, proxy)) + return hs_path_node_count

1 0

[chutney/master] Don't check whether we can write on a listener
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 0687600c104d69890e4b67cffb73c8ede83ec91e Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 15:22:00 2019 -0400 Don't check whether we can write on a listener --- lib/chutney/Traffic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 2f4cc94..7199f87 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -125,6 +125,9 @@ class Listener(asyncore.dispatcher): self.listen(0) self.tt = tt + def writable(self): + return False + def handle_accept(self): # deprecated in python 3.2 pair = self.accept()

1 0

[chutney/master] Make TrafficTester create sources and sinks
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 58c5a3c69dde2f057e82fb64aa4932acbcdd0840 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 11:24:50 2019 -0400 Make TrafficTester create sources and sinks We'd like to be able to change the client and responder behavior, but we need to keep both parts synchronized as we do so. --- lib/chutney/Traffic.py | 13 ++++++++++--- scripts/chutney_tests/verify.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 7ef3b2f..de21d60 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -125,8 +125,7 @@ class Listener(asyncore.dispatcher): newsock, endpoint = pair debug("new client from %s:%s (fd=%d)" % (endpoint[0], endpoint[1], newsock.fileno())) - handler = Sink(newsock, self.tt) - self.tt.add(handler) + self.tt.add_responder(newsock) def fileno(self): return self.socket.fileno() @@ -320,6 +319,14 @@ class TrafficTester(object): for name in item.get_test_names(): self.tests.add(name) + def add_client(self, server, proxy=None): + source = Source(self, server, proxy) + self.add(source) + + def add_responder(self, socket): + sink = Sink(socket, self) + self.add(sink) + def success(self, name): """Declare that a single test has passed.""" self.tests.success(name) @@ -355,7 +362,7 @@ def main(): tt = TrafficTester(bind_to, DATA) # Don't use a proxy for self-testing, so that we avoid tor entirely - tt.add(Source(tt, bind_to)) + tt.add_client(bind_to) success = tt.run() if success: diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py index 43100d6..e97ff52 100644 --- a/scripts/chutney_tests/verify.py +++ b/scripts/chutney_tests/verify.py @@ -155,7 +155,7 @@ def _configure_exits(tt, bind_to, tmpdata, reps, client_list, exit_list, 'localhost', op._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(op._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, bind_to, proxy)) + tt.add_client(bind_to, proxy) return exit_path_node_count @@ -187,7 +187,7 @@ def _configure_hs(tt, tmpdata, reps, client_list, hs_list, HS_PORT, 'localhost', client._env['socksport'])) for _ in range(connection_count): proxy = ('localhost', int(client._env['socksport'])) - tt.add(chutney.Traffic.Source(tt, hs_bind_to, proxy)) + tt.add_client(hs_bind_to, proxy) return hs_path_node_count

1 0

[chutney/master] re-add not-connected to handle some failure cases
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 4c0e3f31df3167eea236e0f6c19fc80e757a906d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 13:56:39 2019 -0400 re-add not-connected to handle some failure cases --- lib/chutney/Traffic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 167647b..425d08d 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -227,6 +227,7 @@ class CloseSourceProducer: class Source(asynchat.async_chat): """A data source, connecting to a TCP server, optionally over a SOCKS proxy, sending data.""" + NOT_CONNECTED = 0 CONNECTING = 1 CONNECTING_THROUGH_PROXY = 2 CONNECTED = 5

1 0

[chutney/master] Don't use "ord" to debug a byte; python3 hates it.
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 8c8cbf50dad79eb4233c0962a93f882a09c70b9d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 14:15:09 2019 -0400 Don't use "ord" to debug a byte; python3 hates it. --- lib/chutney/Traffic.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 425d08d..2f4cc94 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -34,6 +34,13 @@ import asynchat from chutney.Debug import debug_flag, debug +if sys.version_info[0] >= 3: + def byte_to_int(b): + return b +else: + def byte_to_int(b): + return ord(b) + def addr_to_family(addr): for family in [socket.AF_INET, socket.AF_INET6]: try: @@ -274,7 +281,7 @@ class Source(asynchat.async_chat): self.push_output() else: debug("proxy handshake failed (0x%x)! (fd=%d)" % - (ord(self.inbuf[1]), self.fileno())) + (byte_to_int(self.inbuf[1]), self.fileno())) self.state = self.NOT_CONNECTED self.close()

1 0

[chutney/master] Make chutney bidirectional by default
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 36a3ee88f19391fb3525454ba527a2479010c85d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 11:44:10 2019 -0400 Make chutney bidirectional by default The default client/responder pair is no longer source/sink, but echoclient/echoserver. --- lib/chutney/Traffic.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index de21d60..167647b 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -286,6 +286,49 @@ class Source(asynchat.async_chat): def fileno(self): return self.socket.fileno() +class EchoServer(asynchat.async_chat): + def __init__(self, sock, tt): + asynchat.async_chat.__init__(self, sock) + self.set_terminator(None) + self.tt = tt + + def collect_incoming_data(self, data): + self.push(data) + + def handle_close(self): + self.close_when_done() + +class EchoClient(Source): + def __init__(self, tt, server, proxy=None): + Source.__init__(self, tt, server, proxy) + self.data_checker = DataChecker(tt.data_source.copy()) + self.testname_check = "check-%s"%id(self) + + def get_test_names(self): + return [ self.testname, self.testname_check ] + + def handle_close(self): + self.close_when_done() + + def collect_incoming_data(self, data): + if self.state == self.CONNECTING_THROUGH_PROXY: + Source.collect_incoming_data(self, data) + if self.state == self.CONNECTING_THROUGH_PROXY: + return + data = self.inbuf + self.inbuf = b"" + + self.data_checker.consume(data) + + if self.data_checker.succeeded: + debug("successful verification") + self.close() + self.tt.success(self.testname_check) + elif self.data_checker.failed: + debug("receive comparison failed") + self.tt.failure(self.testname_check) + self.close() + class TrafficTester(object): """ Hang on select.select() and dispatch to Sources and Sinks. @@ -300,7 +343,15 @@ class TrafficTester(object): data=b"", timeout=3, repetitions=1, - dot_repetitions=0): + dot_repetitions=0, + chat_type="Echo"): + if chat_type == "Echo": + self.client_class = EchoClient + self.responder_class = EchoServer + else: + self.client_class = Source + self.responder_class = Sink + self.listener = Listener(self, endpoint) self.pending_close = [] self.timeout = timeout @@ -320,11 +371,11 @@ class TrafficTester(object): self.tests.add(name) def add_client(self, server, proxy=None): - source = Source(self, server, proxy) + source = self.client_class(self, server, proxy) self.add(source) def add_responder(self, socket): - sink = Sink(socket, self) + sink = self.responder_class(socket, self) self.add(sink) def success(self, name):

1 0

[chutney/master] Log test status less frequently: travis thinks it is too verbose.
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit c5184442a3695d0dad36d7f310d047e5e95ba0c1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 15:24:26 2019 -0400 Log test status less frequently: travis thinks it is too verbose. --- lib/chutney/Traffic.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index 7199f87..f2448a8 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -400,12 +400,16 @@ class TrafficTester(object): def run(self): start = now = time.time() end = time.time() + self.timeout + DUMP_TEST_STATUS_INTERVAL=0.5 + dump_at = start+DUMP_TEST_STATUS_INTERVAL while now < end and not self.tests.all_done(): # run only one iteration at a time, with a nice short timeout, so we # can actually detect completion and timeouts. asyncore.loop(0.2, False, None, 1) now = time.time() - debug("Test status: %s"%self.tests.status()) + if now > dump_at: + debug("Test status: %s"%self.tests.status()) + dump_at += DUMP_TEST_STATUS_INTERVAL if not debug_flag: sys.stdout.write('\n')

1 0

[chutney/master] Be a little more verbose about test success/failure
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit eedc7e583362601aac35a2608cdf28254eb229dc Author: Nick Mathewson <nickm(a)torproject.org> Date: Sat May 11 10:50:23 2019 -0400 Be a little more verbose about test success/failure Additionally, give them nicer unique names, and warn if we accidentally double-add a test or something. --- lib/chutney/Traffic.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index ed4ad69..1252840 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -34,6 +34,17 @@ import asynchat from chutney.Debug import debug_flag, debug +def note(s): + sys.stderr.write("NOTE: %s\n"%s) +def warn(s): + sys.stderr.write("WARN: %s\n"%s) + +UNIQ_CTR = 0 +def uniq(s): + global UNIQ_CTR + UNIQ_CTR += 1 + return "%s-%s"%(s,UNIQ_CTR) + if sys.version_info[0] >= 3: def byte_to_int(b): return b @@ -85,24 +96,33 @@ class TestSuite(object): self.failures = 0 def add(self, name): + note("Registering %s"%name) if name not in self.tests: debug("Registering %s"%name) self.not_done += 1 self.tests[name] = 'not done' + else: + warn("... already registered!") def success(self, name): + note("Success for %s"%name) if self.tests[name] == 'not done': debug("Succeeded %s"%name) self.tests[name] = 'success' self.not_done -= 1 self.successes += 1 + else: + warn("... status was %s"%self.tests.get(name)) def failure(self, name): + note("Failure for %s"%name) if self.tests[name] == 'not done': debug("Failed %s"%name) self.tests[name] = 'failure' self.not_done -= 1 self.failures += 1 + else: + warn("... status was %s"%self.tests.get(name)) def failure_count(self): return self.failures @@ -203,7 +223,7 @@ class Sink(asynchat.async_chat): self.set_terminator(None) self.tt = tt self.data_checker = DataChecker(tt.data_source.copy()) - self.testname = "recv-data%s"%id(self) + self.testname = uniq("recv-data") def get_test_names(self): return [ self.testname ] @@ -249,7 +269,7 @@ class Source(asynchat.async_chat): self.proxy = proxy self.server = server self.tt = tt - self.testname = "send-data%s"%id(self) + self.testname = uniq("send-data") self.set_terminator(None) dest = (self.proxy or self.server) @@ -313,7 +333,7 @@ class EchoClient(Source): def __init__(self, tt, server, proxy=None): Source.__init__(self, tt, server, proxy) self.data_checker = DataChecker(tt.data_source.copy()) - self.testname_check = "check-%s"%id(self) + self.testname_check = uniq("check") def get_test_names(self): return [ self.testname, self.testname_check ]

1 0

[chutney/master] Stop trying to use half-open connections: Tor doesn't support them
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 15dcdee6bf4516b7bb017c197c747c3308f3f8b6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun May 12 10:42:45 2019 -0400 Stop trying to use half-open connections: Tor doesn't support them --- lib/chutney/Traffic.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index f403ca0..02cca91 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -322,7 +322,6 @@ class Source(asynchat.async_chat): self.push_with_producer(self.data_source) self.push_with_producer(CloseSourceProducer(self)) - self.close_when_done() def fileno(self): return self.socket.fileno() @@ -332,18 +331,17 @@ class EchoServer(asynchat.async_chat): asynchat.async_chat.__init__(self, sock, map=tt.socket_map) self.set_terminator(None) self.tt = tt + self.am_closing = False def collect_incoming_data(self, data): self.push(data) - def handle_close(self): - self.close_when_done() - class EchoClient(Source): def __init__(self, tt, server, proxy=None): Source.__init__(self, tt, server, proxy) self.data_checker = DataChecker(tt.data_source.copy()) self.testname_check = uniq("check") + self.am_closing = False def enote(self, s): self.tt.tests.note(self.testname_check, s) @@ -351,9 +349,6 @@ class EchoClient(Source): def get_test_names(self): return [ self.testname, self.testname_check ] - def handle_close(self): - self.close_when_done() - def collect_incoming_data(self, data): if self.state == self.CONNECTING_THROUGH_PROXY: Source.collect_incoming_data(self, data) @@ -455,7 +450,7 @@ class TrafficTester(object): %(self.tests.all_done(), self.tests.failure_count())) note("Status:\n%s"%self.tests.teststatus) - + self.listener.close() return self.tests.all_done() and self.tests.failure_count() == 0

1 0

[chutney/master] Use a separate asyncore socketmap for every TrafficTester
by teor＠torproject.org 20 Jun '19

20 Jun '19

commit 494f3fc1c3c0f53bade89fe58a9dcd6395080a65 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 10 15:56:13 2019 -0400 Use a separate asyncore socketmap for every TrafficTester This should prevent multiple in-process runs from interfering --- lib/chutney/Traffic.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py index f2448a8..ed4ad69 100755 --- a/lib/chutney/Traffic.py +++ b/lib/chutney/Traffic.py @@ -118,7 +118,7 @@ class Listener(asyncore.dispatcher): "A TCP listener, binding, listening and accepting new connections." def __init__(self, tt, endpoint): - asyncore.dispatcher.__init__(self) + asyncore.dispatcher.__init__(self, map=tt.socket_map) self.create_socket(addr_to_family(endpoint[0]), socket.SOCK_STREAM) self.set_reuse_addr() self.bind(endpoint) @@ -199,7 +199,7 @@ class DataChecker(object): class Sink(asynchat.async_chat): "A data sink, reading from its peer and verifying the data." def __init__(self, sock, tt): - asynchat.async_chat.__init__(self, sock) + asynchat.async_chat.__init__(self, sock, map=tt.socket_map) self.set_terminator(None) self.tt = tt self.data_checker = DataChecker(tt.data_source.copy()) @@ -243,7 +243,7 @@ class Source(asynchat.async_chat): CONNECTED = 5 def __init__(self, tt, server, proxy=None): - asynchat.async_chat.__init__(self) + asynchat.async_chat.__init__(self, map=tt.socket_map) self.data_source = tt.data_source.copy() self.inbuf = b'' self.proxy = proxy @@ -299,7 +299,7 @@ class Source(asynchat.async_chat): class EchoServer(asynchat.async_chat): def __init__(self, sock, tt): - asynchat.async_chat.__init__(self, sock) + asynchat.async_chat.__init__(self, sock, map=tt.socket_map) self.set_terminator(None) self.tt = tt @@ -363,6 +363,8 @@ class TrafficTester(object): self.client_class = Source self.responder_class = Sink + self.socket_map = {} + self.listener = Listener(self, endpoint) self.pending_close = [] self.timeout = timeout @@ -405,7 +407,7 @@ class TrafficTester(object): while now < end and not self.tests.all_done(): # run only one iteration at a time, with a nice short timeout, so we # can actually detect completion and timeouts. - asyncore.loop(0.2, False, None, 1) + asyncore.loop(0.2, False, self.socket_map, 1) now = time.time() if now > dump_at: debug("Test status: %s"%self.tests.status())

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits June 2019