February 2018 - tor-commits - lists.torproject.org

[tor-browser-build/master] Bump rbm to latest master
by gk＠torproject.org 02 Feb '18

02 Feb '18

commit 58dd3596db905fc741e4dc945b26b382180b74b8 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Feb 2 07:22:04 2018 +0000 Bump rbm to latest master --- rbm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbm b/rbm index 5c89374..7494edc 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 5c89374df7957238a9565c0b938ee3f30f880438 +Subproject commit 7494edc6d2556c511c213823a6549410ba75f73b

1 0

[rbm/master] Bug 24931: fix the build_log_append option
by gk＠torproject.org 02 Feb '18

02 Feb '18

commit 476391e2c8a808acec3b6c6429dddd9f6280d783 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Jan 18 13:58:20 2018 +0100 Bug 24931: fix the build_log_append option Fix a regression introduced by #24361. --- lib/RBM.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/RBM.pm b/lib/RBM.pm index 73b7d6c..3b19ca9 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -1035,6 +1035,7 @@ sub build_run { if ($build_log ne '-') { my $append = project_config($project, "build_log_append", $options); $build_log = rbm_path($build_log); + unlink $build_log unless $append; make_path(dirname($build_log)); my $now = localtime; path($build_log)->append_utf8("Starting build: $now\n");

1 0

[rbm/master] Merge remote-tracking branch 'boklm/bug_24931'
by gk＠torproject.org 02 Feb '18

02 Feb '18

commit 7494edc6d2556c511c213823a6549410ba75f73b Merge: b83c74c 476391e Author: Georg Koppen <gk(a)torproject.org> Date: Fri Feb 2 07:13:42 2018 +0000 Merge remote-tracking branch 'boklm/bug_24931' lib/RBM.pm | 1 + 1 file changed, 1 insertion(+)

1 0

[tor-browser-build/master] Bug 25111: Don't compile Yasm on our own anymore for Windows Tor Browser
by gk＠torproject.org 02 Feb '18

02 Feb '18

commit 57f544ca8590b00b609b62132aeb2ae7abfa7779 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Feb 1 13:10:08 2018 +0100 Bug 25111: Don't compile Yasm on our own anymore for Windows Tor Browser --- projects/firefox/build | 4 ---- projects/firefox/config | 3 --- projects/yasm/build | 14 -------------- projects/yasm/config | 12 ------------ 4 files changed, 33 deletions(-) diff --git a/projects/firefox/build b/projects/firefox/build index 81d4d1c..051d573 100644 --- a/projects/firefox/build +++ b/projects/firefox/build @@ -12,10 +12,6 @@ # compiler setup so that mingw is first in the PATH. export LD_LIBRARY_PATH=$hgccdir/lib64 export PATH=$hgccdir/bin:$PATH - - # Firefox ESR52 needs Yasm >= 1.2.0 which Ubuntu Precise does not ship. - tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/yasm') %] - export PATH="/var/tmp/dist/yasm/bin:$PATH" [% END -%] [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] distdir=/var/tmp/dist/[% project %] diff --git a/projects/firefox/config b/projects/firefox/config index 4157500..f19b8ac 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -94,8 +94,5 @@ input_files: - project: gcc name: gcc enable: '[% c("var/windows") %]' - - project: yasm - name: yasm - enable: '[% c("var/windows") %]' - filename: STL_win64.patch enable: '[% c("var/windows-x86_64") %]' diff --git a/projects/yasm/build b/projects/yasm/build deleted file mode 100644 index c11924c..0000000 --- a/projects/yasm/build +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -[% c("var/set_default_env") -%] -distdir=/var/tmp/dist/[% project %] -mkdir -p /var/tmp/build -tar -C /var/tmp/build -xf $rootdir/[% c('input_files_by_name/yasm') %] -cd /var/tmp/build/yasm-[% c("version") %] -./configure --prefix="$distdir" -make -j[% c("buildconf/num_procs") %] -make install -cd /var/tmp/dist -[% c('tar', { - tar_src => [ project ], - tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), - }) %] diff --git a/projects/yasm/config b/projects/yasm/config deleted file mode 100644 index 12d009b..0000000 --- a/projects/yasm/config +++ /dev/null @@ -1,12 +0,0 @@ -# vim: filetype=yaml sw=2 -version: 1.2.0 -filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -var: - container: - use_container: 1 - -input_files: - - project: container-image - - URL: 'https://www.tortall.net/projects/yasm/releases/yasm-[% c("version") %].tar.gz' - name: yasm - sha256sum: 768ffab457b90a20a6d895c39749adb547c1b7cb5c108e84b151a838a23ccf31

1 0

[tor/maint-0.3.2] sched: Avoid adding the same channel twice to the KIST pending list
by nickm＠torproject.org 01 Feb '18

01 Feb '18

commit adaf3e9b89f62d68ab631b8f672d9bff996689b9 Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Jan 31 13:46:31 2018 -0500 sched: Avoid adding the same channel twice to the KIST pending list This is the quick fix that is keeping the channel in PENDING state so if we ever try to reschedule the same channel, it won't happened. Fixes #24700 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/bug24700 | 4 ++++ src/or/scheduler_kist.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes/bug24700 b/changes/bug24700 new file mode 100644 index 000000000..74dc581a0 --- /dev/null +++ b/changes/bug24700 @@ -0,0 +1,4 @@ + o Minor bugfixes (scheduler, KIST): + - Avoid adding the same channel twice in the KIST scheduler pending list + wasting CPU cycles at handling the same channel twice. Fixes bug 24700; + bugfix on 0.3.2.1-alpha. diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c index f50f3aa9e..7b5d138be 100644 --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@ -693,7 +693,6 @@ kist_scheduler_run(void) * after the scheduling loop is over. They can hopefully be taken care of * in the next scheduling round. */ - chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; if (!to_readd) { to_readd = smartlist_new(); }

1 0

[tor/maint-0.3.2] sched: Use the sched_heap_idx field to double-check our fix for 24700.
by nickm＠torproject.org 01 Feb '18

01 Feb '18

commit cb5654f300312a8f4c777378d68696667eff427b Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 1 11:05:50 2018 -0500 sched: Use the sched_heap_idx field to double-check our fix for 24700. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/or/channel.c | 3 +++ src/or/scheduler.c | 20 ++++++++++++-------- src/or/scheduler_kist.c | 13 ++++++++++--- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/or/channel.c b/src/or/channel.c index 0b5a7fde9..049a5290d 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -951,6 +951,9 @@ channel_init(channel_t *chan) /* Scheduler state is idle */ chan->scheduler_state = SCHED_CHAN_IDLE; + + /* Channel is not in the scheduler heap. */ + chan->sched_heap_idx = -1; } /** diff --git a/src/or/scheduler.c b/src/or/scheduler.c index cbf51447b..26f927fd8 100644 --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@ -538,10 +538,12 @@ scheduler_channel_has_waiting_cells,(channel_t *chan)) * channels_pending. */ chan->scheduler_state = SCHED_CHAN_PENDING; - smartlist_pqueue_add(channels_pending, - scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), - chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), + chan); + } log_debug(LD_SCHED, "Channel " U64_FORMAT " at %p went from waiting_for_cells " "to pending", @@ -665,10 +667,12 @@ scheduler_channel_wants_writes(channel_t *chan) */ log_debug(LD_SCHED, "chan=%" PRIu64 " became pending", chan->global_identifier); - smartlist_pqueue_add(channels_pending, - scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), - chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), + chan); + } chan->scheduler_state = SCHED_CHAN_PENDING; log_debug(LD_SCHED, "Channel " U64_FORMAT " at %p went from waiting_to_write " diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c index 7b5d138be..d2878437c 100644 --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@ -704,8 +704,10 @@ kist_scheduler_run(void) /* Case 4: cells to send, and still open for writes */ chan->scheduler_state = SCHED_CHAN_PENDING; - smartlist_pqueue_add(cp, scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(cp, scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), chan); + } } } /* End of main scheduling loop */ @@ -725,8 +727,13 @@ kist_scheduler_run(void) SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) { readd_chan->scheduler_state = SCHED_CHAN_PENDING; if (!smartlist_contains(cp, readd_chan)) { - smartlist_pqueue_add(cp, scheduler_compare_channels, + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + /* XXXX Note that the check above is in theory redundant with + * the smartlist_contains check. But let's make sure we're + * not messing anything up, and leave them both for now. */ + smartlist_pqueue_add(cp, scheduler_compare_channels, offsetof(channel_t, sched_heap_idx), readd_chan); + } } } SMARTLIST_FOREACH_END(readd_chan); smartlist_free(to_readd);

1 0

[tor/master] Merge branch 'bug24700_032_01' into bug24700_033_01
by nickm＠torproject.org 01 Feb '18

01 Feb '18

commit e1a40535eaac0cd7306770660da798a0ee247549 Merge: 5516d22a2 cb5654f30 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 1 16:39:04 2018 -0500 Merge branch 'bug24700_032_01' into bug24700_033_01 changes/bug24700 | 4 ++++ src/or/channel.c | 3 +++ src/or/scheduler.c | 20 ++++++++++++-------- src/or/scheduler_kist.c | 14 ++++++++++---- 4 files changed, 29 insertions(+), 12 deletions(-) diff --cc src/or/scheduler.c index 749df5a24,26f927fd8..931122f8d --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@@ -559,11 -537,17 +559,13 @@@ scheduler_channel_has_waiting_cells,(ch * the other lists. It has waiting cells now, so it goes to * channels_pending. */ - chan->scheduler_state = SCHED_CHAN_PENDING; + scheduler_set_channel_state(chan, SCHED_CHAN_PENDING); - smartlist_pqueue_add(channels_pending, - scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), - chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), + chan); + } - log_debug(LD_SCHED, - "Channel " U64_FORMAT " at %p went from waiting_for_cells " - "to pending", - U64_PRINTF_ARG(chan->global_identifier), chan); /* If we made a channel pending, we potentially have scheduling work to * do. */ the_scheduler->schedule(); @@@ -678,11 -665,19 +680,13 @@@ scheduler_channel_wants_writes(channel_ /* * It can write now, so it goes to channels_pending. */ - smartlist_pqueue_add(channels_pending, - scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), - chan); - log_debug(LD_SCHED, "chan=%" PRIu64 " became pending", - chan->global_identifier); + scheduler_set_channel_state(chan, SCHED_CHAN_PENDING); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), + chan); + } - chan->scheduler_state = SCHED_CHAN_PENDING; - log_debug(LD_SCHED, - "Channel " U64_FORMAT " at %p went from waiting_to_write " - "to pending", - U64_PRINTF_ARG(chan->global_identifier), chan); /* We just made a channel pending, we have scheduling work to do. */ the_scheduler->schedule(); } else { diff --cc src/or/scheduler_kist.c index 424ef0590,d2878437c..43831b72b --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@@ -694,9 -703,11 +693,11 @@@ kist_scheduler_run(void /* Case 4: cells to send, and still open for writes */ - chan->scheduler_state = SCHED_CHAN_PENDING; + scheduler_set_channel_state(chan, SCHED_CHAN_PENDING); - smartlist_pqueue_add(cp, scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(cp, scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), chan); + } } } /* End of main scheduling loop */ @@@ -714,10 -725,15 +715,15 @@@ /* Re-add any channels we need to */ if (to_readd) { SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) { - readd_chan->scheduler_state = SCHED_CHAN_PENDING; + scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING); if (!smartlist_contains(cp, readd_chan)) { - smartlist_pqueue_add(cp, scheduler_compare_channels, + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + /* XXXX Note that the check above is in theory redundant with + * the smartlist_contains check. But let's make sure we're + * not messing anything up, and leave them both for now. */ + smartlist_pqueue_add(cp, scheduler_compare_channels, offsetof(channel_t, sched_heap_idx), readd_chan); + } } } SMARTLIST_FOREACH_END(readd_chan); smartlist_free(to_readd);

1 0

[tor/master] sched: Use the sched_heap_idx field to double-check our fix for 24700.
by nickm＠torproject.org 01 Feb '18

01 Feb '18

commit cb5654f300312a8f4c777378d68696667eff427b Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 1 11:05:50 2018 -0500 sched: Use the sched_heap_idx field to double-check our fix for 24700. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/or/channel.c | 3 +++ src/or/scheduler.c | 20 ++++++++++++-------- src/or/scheduler_kist.c | 13 ++++++++++--- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/or/channel.c b/src/or/channel.c index 0b5a7fde9..049a5290d 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -951,6 +951,9 @@ channel_init(channel_t *chan) /* Scheduler state is idle */ chan->scheduler_state = SCHED_CHAN_IDLE; + + /* Channel is not in the scheduler heap. */ + chan->sched_heap_idx = -1; } /** diff --git a/src/or/scheduler.c b/src/or/scheduler.c index cbf51447b..26f927fd8 100644 --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@ -538,10 +538,12 @@ scheduler_channel_has_waiting_cells,(channel_t *chan)) * channels_pending. */ chan->scheduler_state = SCHED_CHAN_PENDING; - smartlist_pqueue_add(channels_pending, - scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), - chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), + chan); + } log_debug(LD_SCHED, "Channel " U64_FORMAT " at %p went from waiting_for_cells " "to pending", @@ -665,10 +667,12 @@ scheduler_channel_wants_writes(channel_t *chan) */ log_debug(LD_SCHED, "chan=%" PRIu64 " became pending", chan->global_identifier); - smartlist_pqueue_add(channels_pending, - scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), - chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), + chan); + } chan->scheduler_state = SCHED_CHAN_PENDING; log_debug(LD_SCHED, "Channel " U64_FORMAT " at %p went from waiting_to_write " diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c index 7b5d138be..d2878437c 100644 --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@ -704,8 +704,10 @@ kist_scheduler_run(void) /* Case 4: cells to send, and still open for writes */ chan->scheduler_state = SCHED_CHAN_PENDING; - smartlist_pqueue_add(cp, scheduler_compare_channels, - offsetof(channel_t, sched_heap_idx), chan); + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + smartlist_pqueue_add(cp, scheduler_compare_channels, + offsetof(channel_t, sched_heap_idx), chan); + } } } /* End of main scheduling loop */ @@ -725,8 +727,13 @@ kist_scheduler_run(void) SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) { readd_chan->scheduler_state = SCHED_CHAN_PENDING; if (!smartlist_contains(cp, readd_chan)) { - smartlist_pqueue_add(cp, scheduler_compare_channels, + if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + /* XXXX Note that the check above is in theory redundant with + * the smartlist_contains check. But let's make sure we're + * not messing anything up, and leave them both for now. */ + smartlist_pqueue_add(cp, scheduler_compare_channels, offsetof(channel_t, sched_heap_idx), readd_chan); + } } } SMARTLIST_FOREACH_END(readd_chan); smartlist_free(to_readd);

1 0

[tor/maint-0.3.2] Merge remote-tracking branch 'dgoulet/bug24700_032_01' into maint-0.3.2
by nickm＠torproject.org 01 Feb '18

01 Feb '18

commit 77634795b0258f04a31a51d827d45eab40557f65 Merge: 31542cc30 cb5654f30 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Feb 1 16:57:57 2018 -0500 Merge remote-tracking branch 'dgoulet/bug24700_032_01' into maint-0.3.2 changes/bug24700 | 4 ++++ src/or/channel.c | 3 +++ src/or/scheduler.c | 20 ++++++++++++-------- src/or/scheduler_kist.c | 14 ++++++++++---- 4 files changed, 29 insertions(+), 12 deletions(-)

1 0

[tor/master] test: KIST Scheduler unit tests to test the pending list state
by nickm＠torproject.org 01 Feb '18

01 Feb '18

commit d40a4e46b6e4b0ab2f544fe844f6b3523203b550 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 1 15:48:28 2018 -0500 test: KIST Scheduler unit tests to test the pending list state This tests many cases of the KIST scheduler with the pending list state by calling entry point in the scheduler while channels are scheduled or not. Also, it adds a test for the bug #24700. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/test/test_scheduler.c | 267 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index 26b412f89..eac66dd27 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -1059,6 +1059,271 @@ test_scheduler_ns_changed(void *arg) return; } +/* + * Mocked functions for the kist_pending_list test. + */ + +static int mock_flush_some_cells_num = 1; +static int mock_more_to_flush = 0; +static int mock_update_socket_info_limit = 0; + +static ssize_t +channel_flush_some_cells_mock_var(channel_t *chan, ssize_t num_cells) +{ + (void) chan; + (void) num_cells; + return mock_flush_some_cells_num; +} + +/* Because when we flush cells, it is possible that the connection outbuf gets + * fully drained, the wants to write scheduler event is fired back while we + * are in the scheduler loop so this mock function does it for us. + * Furthermore, the socket limit is set to 0 so once this is triggered, it + * informs the scheduler that it can't write on the socket anymore. */ +static void +channel_write_to_kernel_mock_trigger_24700(channel_t *chan) +{ + static int chan_id_seen[2] = {0}; + if (++chan_id_seen[chan->global_identifier - 1] > 1) { + tt_assert(0); + } + + scheduler_channel_wants_writes(chan); + + done: + return; +} + +static int +channel_more_to_flush_mock_var(channel_t *chan) +{ + (void) chan; + return mock_more_to_flush; +} + +static void +update_socket_info_impl_mock_var(socket_table_ent_t *ent) +{ + ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0; + ent->limit = mock_update_socket_info_limit; +} + +static void +test_scheduler_kist_pending_list(void *arg) +{ + (void) arg; + +#ifndef HAVE_KIST_SUPPORT + return; +#endif + + /* This is for testing the channel flow with the pending list that is + * depending on the channel state, what will be the expected behavior of the + * scheduler with that list. + * + * For instance, we want to catch double channel add or removing a channel + * that doesn't exists, or putting a channel in the list in a wrong state. + * Essentially, this will articifically test cases of the KIST main loop and + * entry point in the channel subsystem. + * + * In part, this is to also catch things like #24700 and provide a test bed + * for more testing in the future like so. */ + + /* Mocking a series of scheduler function to control the flow of the + * scheduler loop to test every use cases and assess the pending list. */ + MOCK(get_options, mock_get_options); + MOCK(channel_flush_some_cells, channel_flush_some_cells_mock_var); + MOCK(channel_more_to_flush, channel_more_to_flush_mock_var); + MOCK(update_socket_info_impl, update_socket_info_impl_mock_var); + MOCK(channel_write_to_kernel, channel_write_to_kernel_mock); + MOCK(channel_should_write_to_kernel, channel_should_write_to_kernel_mock); + + /* Setup options so we're sure about what sched we are running */ + mocked_options.KISTSchedRunInterval = 10; + set_scheduler_options(SCHEDULER_KIST); + + /* Init scheduler. */ + scheduler_init(); + + /* Initialize a channel. We'll need a second channel for the #24700 bug + * test. */ + channel_t *chan1 = new_fake_channel(); + channel_t *chan2 = new_fake_channel(); + tt_assert(chan1); + tt_assert(chan2); + chan1->magic = chan2->magic = TLS_CHAN_MAGIC; + channel_register(chan1); + channel_register(chan2); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE); + tt_int_op(chan1->sched_heap_idx, OP_EQ, -1); + tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE); + tt_int_op(chan2->sched_heap_idx, OP_EQ, -1); + + /* Once a channel becomes OPEN, it always have at least one cell in it so + * the scheduler is notified that the channel wants to write so this is the + * first step. Might not make sense to you but it is the way it is. */ + scheduler_channel_wants_writes(chan1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + + /* Signal the scheduler that it has waiting cells which means the channel + * will get scheduled. */ + scheduler_channel_has_waiting_cells(chan1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + /* Subsequent call should not add it more times. It is possible we add many + * cells in rapid succession before the channel is scheduled. */ + scheduler_channel_has_waiting_cells(chan1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + scheduler_channel_has_waiting_cells(chan1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + + /* We'll flush one cell and make it that the socket can write but no more to + * flush else we end up in an infinite loop. We expect the channel to be put + * in waiting for cells state and the pending list empty. */ + mock_update_socket_info_limit = INT_MAX; + mock_more_to_flush = 0; + the_scheduler->run(); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS); + + /* Lets make believe that a cell is now in the channel but this time the + * channel can't write so obviously it has more to flush. We expect the + * channel to be back in the pending list. */ + scheduler_channel_has_waiting_cells(chan1); + mock_update_socket_info_limit = 0; + mock_more_to_flush = 1; + the_scheduler->run(); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + + /* Channel is in the pending list now, during that time, we'll trigger a + * wants to write event because maybe the channel buffers were emptied in + * the meantime. This is possible because once the connection outbuf is + * flushed down the low watermark, the scheduler is notified. + * + * We expect the channel to NOT be added in the pending list again and stay + * in PENDING state. */ + scheduler_channel_wants_writes(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + + /* Make it that the channel can write now but has nothing else to flush. We + * expect that it is removed from the pending list and waiting for cells. */ + mock_update_socket_info_limit = INT_MAX; + mock_more_to_flush = 0; + the_scheduler->run(); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS); + + /* While waiting for cells, lets say we were able to write more things on + * the connection outbuf (unlikely that this can happen but let say it + * does). We expect the channel to stay in waiting for cells. */ + scheduler_channel_wants_writes(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS); + + /* We'll not put it in the pending list and make the flush cell fail with 0 + * cell flushed. We expect that it is put back in waiting for cells. */ + scheduler_channel_has_waiting_cells(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + mock_flush_some_cells_num = 0; + the_scheduler->run(); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS); + + /* Set the channel to a state where it doesn't want to write more. We expect + * that the channel becomes idle. */ + scheduler_channel_doesnt_want_writes(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE); + + /* Some cells arrive on the channel now. We expect it to go back in waiting + * to write. You might wonder why it is not put in the pending list? Because + * once the channel becomes OPEN again (the doesn't want to write event only + * occurs if the channel goes in MAINT mode), if there are cells in the + * channel, the wants to write event is triggered thus putting the channel + * in pending mode. + * + * Else, if no cells, it stays IDLE and then once a cell comes in, it should + * go in waiting to write which is a BUG itself because the channel can't be + * scheduled until a second cell comes in. Hopefully, #24554 will fix that + * for KIST. */ + scheduler_channel_has_waiting_cells(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE); + + /* Second cell comes in, unfortunately, it won't get scheduled until a wants + * to write event occurs like described above. */ + scheduler_channel_has_waiting_cells(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE); + + /* Unblock everything putting the channel in the pending list. */ + scheduler_channel_wants_writes(chan1); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + + /* Testing bug #24700 which is the situation where we have at least two + * different channels in the pending list. The first one gets flushed and + * bytes are written on the wire which triggers a wants to write event + * because the outbuf is below the low watermark. The bug was that this + * exact channel was added back in the pending list because its state wasn't + * PENDING. + * + * The following does some ninja-tsu to try to make it happen. We need two + * different channels so we create a second one and add it to the pending + * list. Then, we have a custom function when we write to kernel that does + * two important things: + * + * 1) Calls scheduler_channel_wants_writes(chan) on the channel. + * 2) Keeps track of how many times it sees the channel going through. If + * that limit goes > 1, it means we've added the channel twice in the + * pending list. + * + * In the end, we expect both channels to be in the pending list after this + * scheduler run. */ + + /* Put the second channel in the pending list. */ + scheduler_channel_wants_writes(chan2); + scheduler_channel_has_waiting_cells(chan2); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 2); + tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + + /* This makes it that the first pass on socket_can_write() will be true but + * then when a single cell is flushed (514 + 29 bytes), the second call to + * socket_can_write() will be false. If it wasn't sending back false on the + * second run, we end up in an infinite loop of the scheduler. */ + mock_update_socket_info_limit = 600; + /* We want to hit "Case 3:" of the scheduler so channel_more_to_flush() is + * true but socket_can_write() has to be false on the second check on the + * channel. */ + mock_more_to_flush = 1; + mock_flush_some_cells_num = 1; + MOCK(channel_write_to_kernel, channel_write_to_kernel_mock_trigger_24700); + the_scheduler->run(); + tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 2); + tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING); + + done: + chan1->state = chan2->state = CHANNEL_STATE_CLOSED; + chan1->registered = chan2->registered = 0; + channel_free(chan1); + channel_free(chan2); + scheduler_free_all(); + + UNMOCK(get_options); + UNMOCK(channel_flush_some_cells); + UNMOCK(channel_more_to_flush); + UNMOCK(update_socket_info_impl); + UNMOCK(channel_write_to_kernel); + UNMOCK(channel_should_write_to_kernel); +} + struct testcase_t scheduler_tests[] = { { "compare_channels", test_scheduler_compare_channels, TT_FORK, NULL, NULL }, @@ -1068,6 +1333,8 @@ struct testcase_t scheduler_tests[] = { { "loop_kist", test_scheduler_loop_kist, TT_FORK, NULL, NULL }, { "ns_changed", test_scheduler_ns_changed, TT_FORK, NULL, NULL}, { "should_use_kist", test_scheduler_can_use_kist, TT_FORK, NULL, NULL }, + { "kist_pending_list", test_scheduler_kist_pending_list, TT_FORK, + NULL, NULL }, END_OF_TESTCASES };

1 0