December 2017 - tor-commits - lists.torproject.org

[tor/release-0.3.2] sched: Use lower layer cell limit with KISTLite
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit fdfa4a5a140a2d82ebd92e980a577cbcecd2180b Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Dec 20 14:14:02 2017 -0500 sched: Use lower layer cell limit with KISTLite Instead of using INT_MAX as a write limit for KISTLite, use the lower layer limit which is using the specialized num_cells_writeable() of the channel that will down the line check the connection's outbuf and limit it to 32KB (OR_CONN_HIGHWATER). That way we don't take the chance of bloating the connection's outbuf and we keep the cells in the circuit queue which our OOM handler can take care of, not the outbuf. Finally, this commit adds a log_debug() in the update socket information function of KIST so we can get the socket information in debug. Fixes #24671 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/bug24671 | 6 ++++++ src/or/scheduler_kist.c | 22 ++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/changes/bug24671 b/changes/bug24671 new file mode 100644 index 000000000..34d09e704 --- /dev/null +++ b/changes/bug24671 @@ -0,0 +1,6 @@ + o Minor bugfixes (scheduler, KIST): + - Use a sane write limit for KISTLite when writing onto a connection + buffer instead of using INT_MAX and shoving as much as it can. Because + the OOM handler cleans up circuit queues, we are better off at keeping + them in that queue instead of the connection's buffer. Fixes bug 24671; + bugfix on 0.3.2.1-alpha. diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c index 9acd89b37..b7a4ee021 100644 --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@ -298,13 +298,18 @@ update_socket_info_impl, (socket_table_ent_t *ent)) fallback: /* If all of a sudden we don't have kist support, we just zero out all the - * variables for this socket since we don't know what they should be. - * We also effectively allow the socket write as much as it wants to the - * kernel, effectively returning it to vanilla scheduler behavior. Writes - * are still limited by the lower layers of Tor: socket blocking, full - * outbuf, etc. */ + * variables for this socket since we don't know what they should be. We + * also allow the socket to write as much as it can from the estimated + * number of cells the lower layer can accept, effectively returning it to + * Vanilla scheduler behavior. */ ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0; - ent->limit = INT_MAX; + /* This function calls the specialized channel object (currently channeltls) + * and ask how many cells it can write on the outbuf which we then multiply + * by the size of the cells for this channel. The cast is because this + * function requires a non-const channel object, meh. */ + ent->limit = channel_num_cells_writeable((channel_t *) ent->chan) * + (get_cell_network_size(ent->chan->wide_circ_ids) + + TLS_PER_CELL_OVERHEAD); } /* Given a socket that isn't in the table, add it. @@ -398,6 +403,11 @@ update_socket_info(socket_table_t *table, const channel_t *chan) return; // Whelp. Entry didn't exist for some reason so nothing to do. } update_socket_info_impl(ent); + log_debug(LD_SCHED, "chan=%" PRIu64 " updated socket info, limit: %" PRIu64 + ", cwnd: %" PRIu32 ", unacked: %" PRIu32 + ", notsent: %" PRIu32 ", mss: %" PRIu32, + ent->chan->global_identifier, ent->limit, ent->cwnd, ent->unacked, + ent->notsent, ent->mss); } /* Increment the channel's socket written value by the number of bytes. */

1 0

[tor/release-0.3.2] Merge remote-tracking branch 'dgoulet/bug24671_032_01' into maint-0.3.2
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 6cd567d7974e6bf55874f3a6d907feb050144095 Merge: 84adb9fcc fdfa4a5a1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 11:13:33 2017 -0500 Merge remote-tracking branch 'dgoulet/bug24671_032_01' into maint-0.3.2 changes/bug24671 | 6 ++++++ src/or/scheduler_kist.c | 22 ++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-)

1 0

[tor/release-0.3.2] Merge branch 'maint-0.3.2' into release-0.3.2
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 0d1a2e366acd98fcece78c8b4734c5507059c366 Merge: 29e23e62c 6cd567d79 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 11:16:00 2017 -0500 Merge branch 'maint-0.3.2' into release-0.3.2 changes/bug24671 | 6 ++++++ src/or/scheduler_kist.c | 22 ++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-)

1 0

[tor/master] Fix a compilation error in the channel tests.
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 2f0d57db56f87a21e1a927a526c1a82ebd32f9f8 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 11:10:30 2017 -0500 Fix a compilation error in the channel tests. This would only show up on systems like windows where monotime_t and monotime_coarse_t are different types. --- src/test/test_channel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_channel.c b/src/test/test_channel.c index 425c93f42..bdc9d32f7 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -719,7 +719,7 @@ test_channel_inbound_cell(void *arg) tt_assert(!monotime_coarse_is_zero(&chan->timestamp_xfer)); tt_u64_op(chan->timestamp_active, OP_NE, 0); tt_u64_op(chan->timestamp_recv, OP_NE, 0); - tt_assert(monotime_is_zero(&chan->next_padding_time)); + tt_assert(monotime_coarse_is_zero(&chan->next_padding_time)); tt_u64_op(chan->n_cells_recved, OP_EQ, 1); tt_u64_op(chan->n_bytes_recved, OP_EQ, get_cell_network_size(0));

1 0

[tor/release-0.3.1] Merge branch 'maint-0.2.5' into maint-0.2.8
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 7d845976e3897fac8e78a4a26688ac57b660151b Merge: 877dd1d6c 920208776 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 10:43:06 2017 -0500 Merge branch 'maint-0.2.5' into maint-0.2.8 changes/bug24666 | 7 ++++ src/or/circuitmux.c | 34 +++++-------------- src/or/circuitmux.h | 2 +- src/or/or.h | 15 +++++++++ src/or/relay.c | 82 ++++++++++++++++++++++++++++++++++++++++++++-- src/or/relay.h | 8 +++++ src/test/test_circuitmux.c | 12 +++---- 7 files changed, 126 insertions(+), 34 deletions(-) diff --cc src/or/relay.h index e15551ca5,9d160b7b9..c4f98d92f --- a/src/or/relay.h +++ b/src/or/relay.h @@@ -60,9 -63,15 +60,16 @@@ void cell_queue_append_packed_copy(circ void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, cell_t *cell, cell_direction_t direction, streamid_t fromstream); + + void destroy_cell_queue_init(destroy_cell_queue_t *queue); + void destroy_cell_queue_clear(destroy_cell_queue_t *queue); + void destroy_cell_queue_append(destroy_cell_queue_t *queue, + circid_t circid, + uint8_t reason); + void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); -int channel_flush_from_first_active_circuit(channel_t *chan, int max); +MOCK_DECL(int, channel_flush_from_first_active_circuit, + (channel_t *chan, int max)); void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); diff --cc src/test/test_circuitmux.c index 9e8fb5496,d6b658c27..1ffa17247 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@@ -34,11 -33,13 +34,12 @@@ test_cmux_destroy_cell_queue(void *arg circuitmux_t *cmux = NULL; channel_t *ch = NULL; circuit_t *circ = NULL; - cell_queue_t *cq = NULL; + destroy_cell_queue_t *cq = NULL; packed_cell_t *pc = NULL; + destroy_cell_t *dc = NULL; -#ifdef ENABLE_MEMPOOLS - init_cell_pool(); -#endif /* ENABLE_MEMPOOLS */ + scheduler_init(); + (void) arg; cmux = circuitmux_alloc(); @@@ -61,20 -62,23 +62,19 @@@ tt_assert(!circ); tt_assert(cq); - tt_int_op(cq->n, ==, 3); + tt_int_op(cq->n, OP_EQ, 3); - pc = cell_queue_pop(cq); - tt_assert(pc); - tt_mem_op(pc->body, OP_EQ, "\x00\x00\x00\x64\x04\x0a\x00\x00\x00", 9); - packed_cell_free(pc); - pc = NULL; + dc = destroy_cell_queue_pop(cq); + tt_assert(dc); - tt_uint_op(dc->circid, ==, 100); ++ tt_uint_op(dc->circid, OP_EQ, 100); - tt_int_op(circuitmux_num_cells(cmux), ==, 2); + tt_int_op(circuitmux_num_cells(cmux), OP_EQ, 2); done: circuitmux_free(cmux); channel_free(ch); packed_cell_free(pc); + tor_free(dc); - -#ifdef ENABLE_MEMPOOLS - free_cell_pool(); -#endif /* ENABLE_MEMPOOLS */ } struct testcase_t circuitmux_tests[] = {

1 0

[tor/release-0.3.1] Merge branch 'maint-0.2.8' into maint-0.2.9
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 3b08184338fd9b0b3e3bd3e5260684cd078beae0 Merge: c604a76a5 7d845976e Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 10:43:12 2017 -0500 Merge branch 'maint-0.2.8' into maint-0.2.9 changes/bug24666 | 7 ++++ src/or/circuitmux.c | 34 +++++-------------- src/or/circuitmux.h | 2 +- src/or/or.h | 15 +++++++++ src/or/relay.c | 82 ++++++++++++++++++++++++++++++++++++++++++++-- src/or/relay.h | 8 +++++ src/test/test_circuitmux.c | 12 +++---- 7 files changed, 126 insertions(+), 34 deletions(-)

1 0

[tor/release-0.3.1] Move free to end of test function so coverity won't complain.
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit cd1f708a7f44ab305c9fcda0060f55f075b98362 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 10:39:29 2017 -0500 Move free to end of test function so coverity won't complain. --- src/test/test_circuitmux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c index 130be6fc9..d6b658c27 100644 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@ -67,7 +67,6 @@ test_cmux_destroy_cell_queue(void *arg) dc = destroy_cell_queue_pop(cq); tt_assert(dc); tt_uint_op(dc->circid, ==, 100); - tor_free(dc); tt_int_op(circuitmux_num_cells(cmux), ==, 2); @@ -75,6 +74,7 @@ test_cmux_destroy_cell_queue(void *arg) circuitmux_free(cmux); channel_free(ch); packed_cell_free(pc); + tor_free(dc); #ifdef ENABLE_MEMPOOLS free_cell_pool();

1 0

[tor/release-0.3.0] Merge branch 'maint-0.2.9' into maint-0.3.0
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 03b4dd92a4c359fb2f699c579ed6dbcd73981267 Merge: 08ed0d793 79a50afa0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 10:49:40 2017 -0500 Merge branch 'maint-0.2.9' into maint-0.3.0 changes/bug24666 | 7 ++++ src/or/circuitmux.c | 34 ++++++-------------- src/or/circuitmux.h | 2 +- src/or/or.h | 15 +++++++++ src/or/relay.c | 79 ++++++++++++++++++++++++++++++++++++++++++++-- src/or/relay.h | 8 +++++ src/test/test_circuitmux.c | 12 +++---- 7 files changed, 123 insertions(+), 34 deletions(-)

1 0

[tor/release-0.3.0] Merge branch 'maint-0.2.8' into maint-0.2.9
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 3b08184338fd9b0b3e3bd3e5260684cd078beae0 Merge: c604a76a5 7d845976e Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 10:43:12 2017 -0500 Merge branch 'maint-0.2.8' into maint-0.2.9 changes/bug24666 | 7 ++++ src/or/circuitmux.c | 34 +++++-------------- src/or/circuitmux.h | 2 +- src/or/or.h | 15 +++++++++ src/or/relay.c | 82 ++++++++++++++++++++++++++++++++++++++++++++-- src/or/relay.h | 8 +++++ src/test/test_circuitmux.c | 12 +++---- 7 files changed, 126 insertions(+), 34 deletions(-)

1 0

[tor/release-0.3.0] Merge branch 'maint-0.2.5' into maint-0.2.8
by nickm＠torproject.org 21 Dec '17

21 Dec '17

commit 7d845976e3897fac8e78a4a26688ac57b660151b Merge: 877dd1d6c 920208776 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Dec 21 10:43:06 2017 -0500 Merge branch 'maint-0.2.5' into maint-0.2.8 changes/bug24666 | 7 ++++ src/or/circuitmux.c | 34 +++++-------------- src/or/circuitmux.h | 2 +- src/or/or.h | 15 +++++++++ src/or/relay.c | 82 ++++++++++++++++++++++++++++++++++++++++++++-- src/or/relay.h | 8 +++++ src/test/test_circuitmux.c | 12 +++---- 7 files changed, 126 insertions(+), 34 deletions(-) diff --cc src/or/relay.h index e15551ca5,9d160b7b9..c4f98d92f --- a/src/or/relay.h +++ b/src/or/relay.h @@@ -60,9 -63,15 +60,16 @@@ void cell_queue_append_packed_copy(circ void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, cell_t *cell, cell_direction_t direction, streamid_t fromstream); + + void destroy_cell_queue_init(destroy_cell_queue_t *queue); + void destroy_cell_queue_clear(destroy_cell_queue_t *queue); + void destroy_cell_queue_append(destroy_cell_queue_t *queue, + circid_t circid, + uint8_t reason); + void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); -int channel_flush_from_first_active_circuit(channel_t *chan, int max); +MOCK_DECL(int, channel_flush_from_first_active_circuit, + (channel_t *chan, int max)); void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); diff --cc src/test/test_circuitmux.c index 9e8fb5496,d6b658c27..1ffa17247 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@@ -34,11 -33,13 +34,12 @@@ test_cmux_destroy_cell_queue(void *arg circuitmux_t *cmux = NULL; channel_t *ch = NULL; circuit_t *circ = NULL; - cell_queue_t *cq = NULL; + destroy_cell_queue_t *cq = NULL; packed_cell_t *pc = NULL; + destroy_cell_t *dc = NULL; -#ifdef ENABLE_MEMPOOLS - init_cell_pool(); -#endif /* ENABLE_MEMPOOLS */ + scheduler_init(); + (void) arg; cmux = circuitmux_alloc(); @@@ -61,20 -62,23 +62,19 @@@ tt_assert(!circ); tt_assert(cq); - tt_int_op(cq->n, ==, 3); + tt_int_op(cq->n, OP_EQ, 3); - pc = cell_queue_pop(cq); - tt_assert(pc); - tt_mem_op(pc->body, OP_EQ, "\x00\x00\x00\x64\x04\x0a\x00\x00\x00", 9); - packed_cell_free(pc); - pc = NULL; + dc = destroy_cell_queue_pop(cq); + tt_assert(dc); - tt_uint_op(dc->circid, ==, 100); ++ tt_uint_op(dc->circid, OP_EQ, 100); - tt_int_op(circuitmux_num_cells(cmux), ==, 2); + tt_int_op(circuitmux_num_cells(cmux), OP_EQ, 2); done: circuitmux_free(cmux); channel_free(ch); packed_cell_free(pc); + tor_free(dc); - -#ifdef ENABLE_MEMPOOLS - free_cell_pool(); -#endif /* ENABLE_MEMPOOLS */ } struct testcase_t circuitmux_tests[] = {

1 0