August 2021 - tor-commits - lists.torproject.org

[tor/main] Merge branch 'maint-0.3.5' into maint-0.4.5
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit c48d1c3f7c41c6d5c35aeda540fad013dbf83031 Merge: e7a8c3d127 fe5a9db1e6 Author: Alexander Færøy <ahf(a)torproject.org> Date: Wed Aug 11 13:14:26 2021 +0000 Merge branch 'maint-0.3.5' into maint-0.4.5 changes/bug40078 | 3 +++ src/lib/crypt_ops/crypto_ed25519.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-)

1 0

[tor/main] Merge branch 'maint-0.4.5' into maint-0.4.6
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit e3c2179f25aa3b2a310efb0b2d88214e79173eb5 Merge: a3a5e26212 c48d1c3f7c Author: Alexander Færøy <ahf(a)torproject.org> Date: Wed Aug 11 13:14:26 2021 +0000 Merge branch 'maint-0.4.5' into maint-0.4.6 changes/bug40078 | 3 +++ src/lib/crypt_ops/crypto_ed25519.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-)

1 0

[tor/maint-0.4.5] Disable ed25519-donna's batch verification.
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit fe5a9db1e6d32f6d706140a6ddda6b40db80434a Author: George Kadianakis <desnacked(a)riseup.net> Date: Fri Jul 30 16:35:48 2021 +0300 Disable ed25519-donna's batch verification. Fixes bug 40078. As reported by hdevalence our batch verification logic can cause an assert crash. The assert happens because when the batch verification of ed25519-donna fails, the code in `ed25519_checksig_batch()` falls back to doing a single verification for each signature. The crash occurs because batch verification failed, but then all signatures individually verified just fine. That's because batch verification and single verification use a different equation which means that there are sigs that can pass single verification but fail batch verification. Fixing this would require modding ed25519-donna which is not in scope for this ticket, and will be soon deprecated in favor of arti and ed25519-dalek, so my branch instead removes batch verification. --- changes/bug40078 | 3 +++ src/lib/crypt_ops/crypto_ed25519.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes/bug40078 b/changes/bug40078 new file mode 100644 index 0000000000..717309e076 --- /dev/null +++ b/changes/bug40078 @@ -0,0 +1,3 @@ + o Minor bugfix (crypto): + - Disable the unused batch verification feature of ed25519-donna. Fixes + bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry de Valence. \ No newline at end of file diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c index 0a442bb739..98beaf75f8 100644 --- a/src/lib/crypt_ops/crypto_ed25519.c +++ b/src/lib/crypt_ops/crypto_ed25519.c @@ -102,7 +102,7 @@ static const ed25519_impl_t impl_donna = { ed25519_donna_open, ed25519_donna_sign, - ed25519_sign_open_batch_donna, + NULL, /* Don't use donna's batching code because of #40078 */ ed25519_donna_blind_secret_key, ed25519_donna_blind_public_key,

1 0

[tor/maint-0.4.6] Merge branch 'maint-0.3.5' into maint-0.4.5
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit c48d1c3f7c41c6d5c35aeda540fad013dbf83031 Merge: e7a8c3d127 fe5a9db1e6 Author: Alexander Færøy <ahf(a)torproject.org> Date: Wed Aug 11 13:14:26 2021 +0000 Merge branch 'maint-0.3.5' into maint-0.4.5 changes/bug40078 | 3 +++ src/lib/crypt_ops/crypto_ed25519.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-)

1 0

[tor/maint-0.4.5] Merge branch 'maint-0.3.5' into maint-0.4.5
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit c48d1c3f7c41c6d5c35aeda540fad013dbf83031 Merge: e7a8c3d127 fe5a9db1e6 Author: Alexander Færøy <ahf(a)torproject.org> Date: Wed Aug 11 13:14:26 2021 +0000 Merge branch 'maint-0.3.5' into maint-0.4.5 changes/bug40078 | 3 +++ src/lib/crypt_ops/crypto_ed25519.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-)

1 0

[tor/release-0.4.5] relay: Reduce streaming compression ratio from HIGH to LOW
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit 399518da022ea94bd0644082e29012a51aec5697 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 18 12:49:33 2021 -0500 relay: Reduce streaming compression ratio from HIGH to LOW Fixes #40301 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 35 +++++++++++++++++++---------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | 4 ++-- src/test/test_dir.c | 26 -------------------------- 5 files changed, 26 insertions(+), 45 deletions(-) diff --git a/changes/ticket40301 b/changes/ticket40301 new file mode 100644 index 0000000000..c1fd821e3f --- /dev/null +++ b/changes/ticket40301 @@ -0,0 +1,4 @@ + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index e8cb284165..2f8a281cd1 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -294,19 +294,22 @@ client_likes_consensus(const struct consensus_cache_entry_t *ent, /** Return the compression level we should use for sending a compressed * response of size n_bytes. */ STATIC compression_level_t -choose_compression_level(ssize_t n_bytes) +choose_compression_level(void) { - if (! have_been_under_memory_pressure()) { - return HIGH_COMPRESSION; /* we have plenty of RAM. */ - } else if (n_bytes < 0) { - return HIGH_COMPRESSION; /* unknown; might be big. */ - } else if (n_bytes < 1024) { - return LOW_COMPRESSION; - } else if (n_bytes < 2048) { - return MEDIUM_COMPRESSION; - } else { - return HIGH_COMPRESSION; - } + /* This is the compression level choice for a stream. + * + * We always return LOW because this compression is done in the main thread + * thus we save CPU time as much as possible, and it is also done more than + * background compression for document we serve pre-compressed. + * + * GZip highest compression level (9) gives us a ratio of 49.72% + * Zstd lowest compression level (1) gives us a ratio of 47.38% + * + * Thus, as the network moves more and more to use Zstd when requesting + * directory documents that are not pre-cached, even at the + * lowest level, we still gain over GZip and thus help with load and CPU + * time on the network. */ + return LOW_COMPRESSION; } /** Information passed to handle a GET request. */ @@ -1044,7 +1047,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(estimated_len)); + choose_compression_level()); SMARTLIST_FOREACH(items, const char *, c, connection_buf_add_compress(c, strlen(c), conn, 0)); connection_buf_add_compress("", 0, conn, 1); @@ -1109,7 +1112,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); const int initial_flush_result = connection_dirserv_flushed_some(conn); tor_assert_nonfatal(initial_flush_result == 0); @@ -1204,7 +1207,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compress_method, cache_lifetime); if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); clear_spool = 0; /* Prime the connection with some data. */ int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -1301,7 +1304,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) 60*60); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len)); + choose_compression_level()); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_buf_add_compress( c->cache_info.signed_descriptor_body, diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index 236ea649ef..44e40d108c 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -26,7 +26,7 @@ MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, STATIC int handle_post_hs_descriptor(const char *url, const char *body); enum compression_level_t; -STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC enum compression_level_t choose_compression_level(void); struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 45d0d4d602..f0bb22b517 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -49,8 +49,8 @@ memory_level(compression_level_t level) default: case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; - case MEDIUM_COMPRESSION: return 8; - case LOW_COMPRESSION: return 7; + case MEDIUM_COMPRESSION: return 3; + case LOW_COMPRESSION: return 1; } } #endif /* defined(HAVE_ZSTD) */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0e44c47f3f..4c074d0c3e 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -4859,31 +4859,6 @@ NS(directory_initiate_request)(directory_request_t *req) CALLED(directory_initiate_request)++; } -static void -test_dir_choose_compression_level(void* data) -{ - (void)data; - - /* It starts under_memory_pressure */ - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 1); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(LOW_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(MEDIUM_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - /* Reset under_memory_pressure timer */ - cell_queues_check_size(); - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 0); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - done: ; -} - /* * Mock check_private_dir(), and always succeed - no need to actually * look at or create anything on the filesystem. @@ -6397,7 +6372,6 @@ struct testcase_t dir_tests[] = { DIR(should_not_init_request_to_ourselves, TT_FORK), DIR(should_not_init_request_to_dir_auths_without_v3_info, 0), DIR(should_init_request_to_dir_auths, 0), - DIR(choose_compression_level, 0), DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0),

1 0

[tor/release-0.3.5] Merge branch 'maint-0.3.5' into release-0.3.5
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit 9c52846fe5ab6f0d2ef99f665844037f910a62cc Merge: f863870398 399518da02 Author: Alexander Færøy <ahf(a)torproject.org> Date: Wed Aug 11 12:52:08 2021 +0000 Merge branch 'maint-0.3.5' into release-0.3.5 changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 35 +++++++++++++++++++---------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | 4 ++-- src/test/test_dir.c | 26 -------------------------- 5 files changed, 26 insertions(+), 45 deletions(-)

1 0

[tor/release-0.3.5] relay: Reduce streaming compression ratio from HIGH to LOW
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit 399518da022ea94bd0644082e29012a51aec5697 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 18 12:49:33 2021 -0500 relay: Reduce streaming compression ratio from HIGH to LOW Fixes #40301 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 35 +++++++++++++++++++---------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | 4 ++-- src/test/test_dir.c | 26 -------------------------- 5 files changed, 26 insertions(+), 45 deletions(-) diff --git a/changes/ticket40301 b/changes/ticket40301 new file mode 100644 index 0000000000..c1fd821e3f --- /dev/null +++ b/changes/ticket40301 @@ -0,0 +1,4 @@ + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index e8cb284165..2f8a281cd1 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -294,19 +294,22 @@ client_likes_consensus(const struct consensus_cache_entry_t *ent, /** Return the compression level we should use for sending a compressed * response of size n_bytes. */ STATIC compression_level_t -choose_compression_level(ssize_t n_bytes) +choose_compression_level(void) { - if (! have_been_under_memory_pressure()) { - return HIGH_COMPRESSION; /* we have plenty of RAM. */ - } else if (n_bytes < 0) { - return HIGH_COMPRESSION; /* unknown; might be big. */ - } else if (n_bytes < 1024) { - return LOW_COMPRESSION; - } else if (n_bytes < 2048) { - return MEDIUM_COMPRESSION; - } else { - return HIGH_COMPRESSION; - } + /* This is the compression level choice for a stream. + * + * We always return LOW because this compression is done in the main thread + * thus we save CPU time as much as possible, and it is also done more than + * background compression for document we serve pre-compressed. + * + * GZip highest compression level (9) gives us a ratio of 49.72% + * Zstd lowest compression level (1) gives us a ratio of 47.38% + * + * Thus, as the network moves more and more to use Zstd when requesting + * directory documents that are not pre-cached, even at the + * lowest level, we still gain over GZip and thus help with load and CPU + * time on the network. */ + return LOW_COMPRESSION; } /** Information passed to handle a GET request. */ @@ -1044,7 +1047,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(estimated_len)); + choose_compression_level()); SMARTLIST_FOREACH(items, const char *, c, connection_buf_add_compress(c, strlen(c), conn, 0)); connection_buf_add_compress("", 0, conn, 1); @@ -1109,7 +1112,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); const int initial_flush_result = connection_dirserv_flushed_some(conn); tor_assert_nonfatal(initial_flush_result == 0); @@ -1204,7 +1207,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compress_method, cache_lifetime); if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); clear_spool = 0; /* Prime the connection with some data. */ int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -1301,7 +1304,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) 60*60); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len)); + choose_compression_level()); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_buf_add_compress( c->cache_info.signed_descriptor_body, diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index 236ea649ef..44e40d108c 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -26,7 +26,7 @@ MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, STATIC int handle_post_hs_descriptor(const char *url, const char *body); enum compression_level_t; -STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC enum compression_level_t choose_compression_level(void); struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 45d0d4d602..f0bb22b517 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -49,8 +49,8 @@ memory_level(compression_level_t level) default: case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; - case MEDIUM_COMPRESSION: return 8; - case LOW_COMPRESSION: return 7; + case MEDIUM_COMPRESSION: return 3; + case LOW_COMPRESSION: return 1; } } #endif /* defined(HAVE_ZSTD) */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0e44c47f3f..4c074d0c3e 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -4859,31 +4859,6 @@ NS(directory_initiate_request)(directory_request_t *req) CALLED(directory_initiate_request)++; } -static void -test_dir_choose_compression_level(void* data) -{ - (void)data; - - /* It starts under_memory_pressure */ - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 1); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(LOW_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(MEDIUM_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - /* Reset under_memory_pressure timer */ - cell_queues_check_size(); - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 0); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - done: ; -} - /* * Mock check_private_dir(), and always succeed - no need to actually * look at or create anything on the filesystem. @@ -6397,7 +6372,6 @@ struct testcase_t dir_tests[] = { DIR(should_not_init_request_to_ourselves, TT_FORK), DIR(should_not_init_request_to_dir_auths_without_v3_info, 0), DIR(should_init_request_to_dir_auths, 0), - DIR(choose_compression_level, 0), DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0),

1 0

[tor/release-0.4.5] relay: Reduce streaming compression ratio from HIGH to LOW
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit 70d8fb3eabd0a67ef298a2d36992c8d638a3d0b8 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 18 12:49:33 2021 -0500 relay: Reduce streaming compression ratio from HIGH to LOW Fixes #40301 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 37 ++++++++++++++++++++----------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | 4 ++-- src/test/test_dir.c | 26 -------------------------- 5 files changed, 27 insertions(+), 46 deletions(-) diff --git a/changes/ticket40301 b/changes/ticket40301 new file mode 100644 index 0000000000..c1fd821e3f --- /dev/null +++ b/changes/ticket40301 @@ -0,0 +1,4 @@ + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 00bb0abf23..84bb7c220c 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -296,19 +296,22 @@ client_likes_consensus(const struct consensus_cache_entry_t *ent, /** Return the compression level we should use for sending a compressed * response of size n_bytes. */ STATIC compression_level_t -choose_compression_level(ssize_t n_bytes) +choose_compression_level(void) { - if (! have_been_under_memory_pressure()) { - return HIGH_COMPRESSION; /* we have plenty of RAM. */ - } else if (n_bytes < 0) { - return HIGH_COMPRESSION; /* unknown; might be big. */ - } else if (n_bytes < 1024) { - return LOW_COMPRESSION; - } else if (n_bytes < 2048) { - return MEDIUM_COMPRESSION; - } else { - return HIGH_COMPRESSION; - } + /* This is the compression level choice for a stream. + * + * We always return LOW because this compression is done in the main thread + * thus we save CPU time as much as possible, and it is also done more than + * background compression for document we serve pre-compressed. + * + * GZip highest compression level (9) gives us a ratio of 49.72% + * Zstd lowest compression level (1) gives us a ratio of 47.38% + * + * Thus, as the network moves more and more to use Zstd when requesting + * directory documents that are not pre-cached, even at the + * lowest level, we still gain over GZip and thus help with load and CPU + * time on the network. */ + return LOW_COMPRESSION; } /** Information passed to handle a GET request. */ @@ -1078,7 +1081,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(estimated_len)); + choose_compression_level()); } SMARTLIST_FOREACH(items, const char *, c, @@ -1141,7 +1144,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); const int initial_flush_result = connection_dirserv_flushed_some(conn); tor_assert_nonfatal(initial_flush_result == 0); @@ -1236,7 +1239,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compress_method, cache_lifetime); if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); clear_spool = 0; /* Prime the connection with some data. */ int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -1332,7 +1335,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) 60*60); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len)); + choose_compression_level()); } SMARTLIST_FOREACH(certs, authority_cert_t *, c, @@ -1484,7 +1487,7 @@ handle_get_next_bandwidth(dir_connection_t *conn, compress_method, BANDWIDTH_CACHE_LIFETIME); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len/2)); + choose_compression_level()); log_debug(LD_DIR, "Compressing bandwidth file."); } else { log_debug(LD_DIR, "Not compressing bandwidth file."); diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index d6392e2d42..8e0945125d 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -26,7 +26,7 @@ MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, STATIC int handle_post_hs_descriptor(const char *url, const char *body); enum compression_level_t; -STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC enum compression_level_t choose_compression_level(void); struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 5913d823e1..61c684068e 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -49,8 +49,8 @@ memory_level(compression_level_t level) default: case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; - case MEDIUM_COMPRESSION: return 8; - case LOW_COMPRESSION: return 7; + case MEDIUM_COMPRESSION: return 3; + case LOW_COMPRESSION: return 1; } } #endif /* defined(HAVE_ZSTD) */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 467b740d71..63cc621964 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -5495,31 +5495,6 @@ dir_tests_directory_initiate_request(directory_request_t *req) dir_tests_directory_initiate_request_called++; } -static void -test_dir_choose_compression_level(void* data) -{ - (void)data; - - /* It starts under_memory_pressure */ - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 1); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(LOW_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(MEDIUM_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - /* Reset under_memory_pressure timer */ - cell_queues_check_size(); - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 0); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - done: ; -} - /* * Mock check_private_dir(), and always succeed - no need to actually * look at or create anything on the filesystem. @@ -7325,7 +7300,6 @@ struct testcase_t dir_tests[] = { DIR(should_not_init_request_to_ourselves, TT_FORK), DIR(should_not_init_request_to_dir_auths_without_v3_info, 0), DIR(should_init_request_to_dir_auths, 0), - DIR(choose_compression_level, 0), DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0),

1 0

[tor/release-0.4.6] relay: Reduce streaming compression ratio from HIGH to LOW
by ahf＠torproject.org 11 Aug '21

11 Aug '21

commit 70d8fb3eabd0a67ef298a2d36992c8d638a3d0b8 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 18 12:49:33 2021 -0500 relay: Reduce streaming compression ratio from HIGH to LOW Fixes #40301 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 37 ++++++++++++++++++++----------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | 4 ++-- src/test/test_dir.c | 26 -------------------------- 5 files changed, 27 insertions(+), 46 deletions(-) diff --git a/changes/ticket40301 b/changes/ticket40301 new file mode 100644 index 0000000000..c1fd821e3f --- /dev/null +++ b/changes/ticket40301 @@ -0,0 +1,4 @@ + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 00bb0abf23..84bb7c220c 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -296,19 +296,22 @@ client_likes_consensus(const struct consensus_cache_entry_t *ent, /** Return the compression level we should use for sending a compressed * response of size n_bytes. */ STATIC compression_level_t -choose_compression_level(ssize_t n_bytes) +choose_compression_level(void) { - if (! have_been_under_memory_pressure()) { - return HIGH_COMPRESSION; /* we have plenty of RAM. */ - } else if (n_bytes < 0) { - return HIGH_COMPRESSION; /* unknown; might be big. */ - } else if (n_bytes < 1024) { - return LOW_COMPRESSION; - } else if (n_bytes < 2048) { - return MEDIUM_COMPRESSION; - } else { - return HIGH_COMPRESSION; - } + /* This is the compression level choice for a stream. + * + * We always return LOW because this compression is done in the main thread + * thus we save CPU time as much as possible, and it is also done more than + * background compression for document we serve pre-compressed. + * + * GZip highest compression level (9) gives us a ratio of 49.72% + * Zstd lowest compression level (1) gives us a ratio of 47.38% + * + * Thus, as the network moves more and more to use Zstd when requesting + * directory documents that are not pre-cached, even at the + * lowest level, we still gain over GZip and thus help with load and CPU + * time on the network. */ + return LOW_COMPRESSION; } /** Information passed to handle a GET request. */ @@ -1078,7 +1081,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(estimated_len)); + choose_compression_level()); } SMARTLIST_FOREACH(items, const char *, c, @@ -1141,7 +1144,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); const int initial_flush_result = connection_dirserv_flushed_some(conn); tor_assert_nonfatal(initial_flush_result == 0); @@ -1236,7 +1239,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compress_method, cache_lifetime); if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); clear_spool = 0; /* Prime the connection with some data. */ int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -1332,7 +1335,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) 60*60); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len)); + choose_compression_level()); } SMARTLIST_FOREACH(certs, authority_cert_t *, c, @@ -1484,7 +1487,7 @@ handle_get_next_bandwidth(dir_connection_t *conn, compress_method, BANDWIDTH_CACHE_LIFETIME); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len/2)); + choose_compression_level()); log_debug(LD_DIR, "Compressing bandwidth file."); } else { log_debug(LD_DIR, "Not compressing bandwidth file."); diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index d6392e2d42..8e0945125d 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -26,7 +26,7 @@ MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, STATIC int handle_post_hs_descriptor(const char *url, const char *body); enum compression_level_t; -STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC enum compression_level_t choose_compression_level(void); struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 5913d823e1..61c684068e 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -49,8 +49,8 @@ memory_level(compression_level_t level) default: case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; - case MEDIUM_COMPRESSION: return 8; - case LOW_COMPRESSION: return 7; + case MEDIUM_COMPRESSION: return 3; + case LOW_COMPRESSION: return 1; } } #endif /* defined(HAVE_ZSTD) */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 467b740d71..63cc621964 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -5495,31 +5495,6 @@ dir_tests_directory_initiate_request(directory_request_t *req) dir_tests_directory_initiate_request_called++; } -static void -test_dir_choose_compression_level(void* data) -{ - (void)data; - - /* It starts under_memory_pressure */ - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 1); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(LOW_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(MEDIUM_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - /* Reset under_memory_pressure timer */ - cell_queues_check_size(); - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 0); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - done: ; -} - /* * Mock check_private_dir(), and always succeed - no need to actually * look at or create anything on the filesystem. @@ -7325,7 +7300,6 @@ struct testcase_t dir_tests[] = { DIR(should_not_init_request_to_ourselves, TT_FORK), DIR(should_not_init_request_to_dir_auths_without_v3_info, 0), DIR(should_init_request_to_dir_auths, 0), - DIR(choose_compression_level, 0), DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0),

1 0