February 2021 - tor-commits - lists.torproject.org

[chutney/master] Add truncate-logs.sh to tools
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit f8c9489dfe0fce9e0331024ef64439f6c010b97f Author: gabor.g.somogyi <gabor.g.somogyi(a)gmail.com> Date: Fri Feb 19 14:56:42 2021 +0100 Add truncate-logs.sh to tools --- tools/truncate-logs.sh | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tools/truncate-logs.sh b/tools/truncate-logs.sh new file mode 100644 index 0000000..59e586d --- /dev/null +++ b/tools/truncate-logs.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# +# Usage: +# … [View More]tools/truncate-logs.sh [node] +# +# Output: +# for each node, truncate the logs +# +# If the argument "node" is specified, only truncates the logs of that +# node. +# +# Examples: +# tools/truncate-logs.sh +# tools/truncate-logs.sh 000a + +set -o errexit +set -o nounset + +# Set some default values if the variables are not already set +: "${CHUTNEY_DATA_DIR:=}" + +if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then + # looks like a broken path: use the path to this tool instead + TOOLS_PATH=$(dirname "$0") + CHUTNEY_PATH=$(dirname "$TOOLS_PATH") + export CHUTNEY_PATH +fi +if [ -d "$PWD/$CHUTNEY_PATH" ] && [ -x "$PWD/$CHUTNEY_PATH/chutney" ]; then + # looks like a relative path: make chutney path absolute + export CHUTNEY_PATH="$PWD/$CHUTNEY_PATH" +fi + +# Get a working net path +case "$CHUTNEY_DATA_DIR" in + /*) + # if an absolute path, then leave as-is + # chutney will make this directory automatically if needed + ;; + *) + # if a relative path + if [ ! -d "$CHUTNEY_DATA_DIR" ]; then + # looks like a broken path: use the chutney path as a base + export CHUTNEY_DATA_DIR="$CHUTNEY_PATH/net" + fi + if [ -d "$PWD/$CHUTNEY_DATA_DIR" ]; then + # looks like a relative path: make chutney path absolute + export CHUTNEY_DATA_DIR="$PWD/$CHUTNEY_DATA_DIR" + fi + ;; +esac + +# Truncate the logs for node $1 +truncate_logs() { + echo "Truncating log: $1" + truncate -s 0 "$1" +} + +# Show the usage message for this script +usage() { + echo "Usage: $NAME [node]" + exit 1 +} + +NAME=$(basename "$0") +DEST="$CHUTNEY_DATA_DIR/nodes" +LOG_FILE=*.log + +[ -d "$DEST" ] || { echo "$NAME: no logs available in '$DEST'"; exit 1; } +if [ $# -eq 0 ]; +then + for log in "$DEST"/*/$LOG_FILE; + do + [ -e "${log}" ] || continue + truncate_logs "$log" + done +elif [ $# -eq 1 ]; +then + for log in "$DEST"/$1/$LOG_FILE; + do + [ -e "${log}" ] || continue + truncate_logs "$log" + done +else + usage +fi [View Less]

1 0

[chutney/master] Add execute flag to truncate-logs.sh
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit d368cb5fff82ef0855c550f9399301130f262987 Author: gaborgsomogyi <gabor.g.somogyi(a)gmail.com> Date: Fri Feb 19 15:13:20 2021 +0100 Add execute flag to truncate-logs.sh --- tools/truncate-logs.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tools/truncate-logs.sh b/tools/truncate-logs.sh old mode 100644 new mode 100755

1 0

[tor/master] relay: Reduce streaming compression ratio from HIGH to LOW
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit d98c77b78e9ce945a7a0de151d5f5cf44061edd5 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Feb 18 12:49:33 2021 -0500 relay: Reduce streaming compression ratio from HIGH to LOW Fixes #40301 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 37 ++++++++++++++++++++----------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | … [View More] 4 ++-- src/test/test_dir.c | 26 -------------------------- 5 files changed, 27 insertions(+), 46 deletions(-) diff --git a/changes/ticket40301 b/changes/ticket40301 new file mode 100644 index 0000000000..c1fd821e3f --- /dev/null +++ b/changes/ticket40301 @@ -0,0 +1,4 @@ + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 00bb0abf23..84bb7c220c 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -296,19 +296,22 @@ client_likes_consensus(const struct consensus_cache_entry_t *ent, /** Return the compression level we should use for sending a compressed * response of size <b>n_bytes</b>. */ STATIC compression_level_t -choose_compression_level(ssize_t n_bytes) +choose_compression_level(void) { - if (! have_been_under_memory_pressure()) { - return HIGH_COMPRESSION; /* we have plenty of RAM. */ - } else if (n_bytes < 0) { - return HIGH_COMPRESSION; /* unknown; might be big. */ - } else if (n_bytes < 1024) { - return LOW_COMPRESSION; - } else if (n_bytes < 2048) { - return MEDIUM_COMPRESSION; - } else { - return HIGH_COMPRESSION; - } + /* This is the compression level choice for a stream. + * + * We always return LOW because this compression is done in the main thread + * thus we save CPU time as much as possible, and it is also done more than + * background compression for document we serve pre-compressed. + * + * GZip highest compression level (9) gives us a ratio of 49.72% + * Zstd lowest compression level (1) gives us a ratio of 47.38% + * + * Thus, as the network moves more and more to use Zstd when requesting + * directory documents that are not pre-cached, even at the + * lowest level, we still gain over GZip and thus help with load and CPU + * time on the network. */ + return LOW_COMPRESSION; } /** Information passed to handle a GET request. */ @@ -1078,7 +1081,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(estimated_len)); + choose_compression_level()); } SMARTLIST_FOREACH(items, const char *, c, @@ -1141,7 +1144,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); const int initial_flush_result = connection_dirserv_flushed_some(conn); tor_assert_nonfatal(initial_flush_result == 0); @@ -1236,7 +1239,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compress_method, cache_lifetime); if (compress_method != NO_METHOD) conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); + choose_compression_level()); clear_spool = 0; /* Prime the connection with some data. */ int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -1332,7 +1335,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) 60*60); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len)); + choose_compression_level()); } SMARTLIST_FOREACH(certs, authority_cert_t *, c, @@ -1484,7 +1487,7 @@ handle_get_next_bandwidth(dir_connection_t *conn, compress_method, BANDWIDTH_CACHE_LIFETIME); if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len/2)); + choose_compression_level()); log_debug(LD_DIR, "Compressing bandwidth file."); } else { log_debug(LD_DIR, "Not compressing bandwidth file."); diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index d6392e2d42..8e0945125d 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -26,7 +26,7 @@ MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, STATIC int handle_post_hs_descriptor(const char *url, const char *body); enum compression_level_t; -STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC enum compression_level_t choose_compression_level(void); struct get_handler_args_t; STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 5913d823e1..61c684068e 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -49,8 +49,8 @@ memory_level(compression_level_t level) default: case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; - case MEDIUM_COMPRESSION: return 8; - case LOW_COMPRESSION: return 7; + case MEDIUM_COMPRESSION: return 3; + case LOW_COMPRESSION: return 1; } } #endif /* defined(HAVE_ZSTD) */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index d62dd3fb9e..b7400be1f7 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -5495,31 +5495,6 @@ dir_tests_directory_initiate_request(directory_request_t *req) dir_tests_directory_initiate_request_called++; } -static void -test_dir_choose_compression_level(void* data) -{ - (void)data; - - /* It starts under_memory_pressure */ - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 1); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(LOW_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(MEDIUM_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - /* Reset under_memory_pressure timer */ - cell_queues_check_size(); - tt_int_op(have_been_under_memory_pressure(), OP_EQ, 0); - - tt_assert(HIGH_COMPRESSION == choose_compression_level(-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(1024-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048-1)); - tt_assert(HIGH_COMPRESSION == choose_compression_level(2048)); - - done: ; -} - /* * Mock check_private_dir(), and always succeed - no need to actually * look at or create anything on the filesystem. @@ -7325,7 +7300,6 @@ struct testcase_t dir_tests[] = { DIR(should_not_init_request_to_ourselves, TT_FORK), DIR(should_not_init_request_to_dir_auths_without_v3_info, 0), DIR(should_init_request_to_dir_auths, 0), - DIR(choose_compression_level, 0), DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0), [View Less]

1 0

[tor/master] Merge remote-tracking branch 'tor-gitlab/mr/319'
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit 890780054975e0c5eb95b097deb6fac53640f66d Merge: 6e3a7c410f d98c77b78e Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Feb 22 15:39:30 2021 -0500 Merge remote-tracking branch 'tor-gitlab/mr/319' changes/ticket40301 | 4 ++++ src/feature/dircache/dircache.c | 37 ++++++++++++++++++++----------------- src/feature/dircache/dircache.h | 2 +- src/lib/compress/compress_zstd.c | 4 ++-- src/test/test_dir.c | 26 -------------------------- … [View More]

1 0

[tor-browser-build/master] Pick up build2
by sysrqb＠torproject.org 22 Feb '21

22 Feb '21

commit 2c9dce8d109dfa12a9ed7f5faa5cc48d14abc8bf Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Mon Feb 22 20:37:05 2021 +0000 Pick up build2 --- rbm.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rbm.conf b/rbm.conf index 288e0f0..14a55a6 100644 --- a/rbm.conf +++ b/rbm.conf @@ -25,9 +25,9 @@ buildconf: var: torbrowser_version: '10.5a11' - torbrowser_build: 'build1' + torbrowser_build: 'build2' torbrowser_incremental_from: - - 10.… [View More]

1 0

[tor/maint-0.4.5] Merge remote-tracking branch 'tor-gitlab/mr/316' into maint-0.4.5
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit bc21ed32903459c53599ee03605e8d23bf42ffe9 Merge: 26c2e843f9 4d7f31b964 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Feb 22 15:37:31 2021 -0500 Merge remote-tracking branch 'tor-gitlab/mr/316' into maint-0.4.5 changes/ticket40300 | 5 +++++ src/feature/relay/relay_find_addr.c | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'tor-gitlab/mr/316' into maint-0.4.5
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit bc21ed32903459c53599ee03605e8d23bf42ffe9 Merge: 26c2e843f9 4d7f31b964 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Feb 22 15:37:31 2021 -0500 Merge remote-tracking branch 'tor-gitlab/mr/316' into maint-0.4.5 changes/ticket40300 | 5 +++++ src/feature/relay/relay_find_addr.c | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-)

1 0

[tor/maint-0.4.5] relay: Move log notice after suggested address lookup
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit 4d7f31b9645d360e7074844711f10565e5d25b7b Author: David Goulet <dgoulet(a)torproject.org> Date: Mon Feb 22 09:13:54 2021 -0500 relay: Move log notice after suggested address lookup When trying to find our address to publish, we would log notice if we couldn't find it from the cache but then we would look at the suggested cache (which contains the address from the authorities) in which we might actually have the address. Thus that log notice was … [View More]misplaced. Move it down after the suggested address cache lookup. Closes #40300 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40300 | 5 +++++ src/feature/relay/relay_find_addr.c | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/changes/ticket40300 b/changes/ticket40300 new file mode 100644 index 0000000000..aef01b4c64 --- /dev/null +++ b/changes/ticket40300 @@ -0,0 +1,5 @@ + o Minor bugfixes (relay): + - Remove a spammy log notice that should not have been indicating the + operator that its IPv4/v6 was missing but it was not. Fixes bug 40300; + bugfix on 0.4.5.1-alpha. + diff --git a/src/feature/relay/relay_find_addr.c b/src/feature/relay/relay_find_addr.c index 39e1cc6a19..2a3f602438 100644 --- a/src/feature/relay/relay_find_addr.c +++ b/src/feature/relay/relay_find_addr.c @@ -144,17 +144,6 @@ relay_find_addr_to_publish, (const or_options_t *options, int family, if (find_my_address(options, family, LOG_INFO, addr_out, NULL, NULL)) { goto found; } - /* No publishable address was found even though we have an ORPort thus - * print a notice log so operator can notice. We'll do that every hour so - * it is not too spammy but enough so operators address the issue. */ - static ratelim_t rlim = RATELIM_INIT(3600); - log_fn_ratelim(&rlim, LOG_NOTICE, LD_CONFIG, - "Unable to find %s address for ORPort %u. " - "You might want to specify %sOnly to it or set an " - "explicit address or set Address.", - fmt_af_family(family), - routerconf_find_or_port(options, family), - fmt_af_family(family)); } /* Third, consider address from our suggestion cache. */ @@ -163,7 +152,19 @@ relay_find_addr_to_publish, (const or_options_t *options, int family, goto found; } - /* No publishable address was found. */ + /* No publishable address was found even though we have an ORPort thus + * print a notice log so operator can notice. We'll do that every hour so + * it is not too spammy but enough so operators address the issue. */ + static ratelim_t rlim = RATELIM_INIT(3600); + log_fn_ratelim(&rlim, LOG_NOTICE, LD_CONFIG, + "Unable to find %s address for ORPort %u. " + "You might want to specify %sOnly to it or set an " + "explicit address or set Address.", + fmt_af_family(family), + routerconf_find_or_port(options, family), + fmt_af_family(family)); + + /* Not found. */ return false; found: [View Less]

1 0

[tor/master] relay: Move log notice after suggested address lookup
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit 4d7f31b9645d360e7074844711f10565e5d25b7b Author: David Goulet <dgoulet(a)torproject.org> Date: Mon Feb 22 09:13:54 2021 -0500 relay: Move log notice after suggested address lookup When trying to find our address to publish, we would log notice if we couldn't find it from the cache but then we would look at the suggested cache (which contains the address from the authorities) in which we might actually have the address. Thus that log notice was … [View More]misplaced. Move it down after the suggested address cache lookup. Closes #40300 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket40300 | 5 +++++ src/feature/relay/relay_find_addr.c | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/changes/ticket40300 b/changes/ticket40300 new file mode 100644 index 0000000000..aef01b4c64 --- /dev/null +++ b/changes/ticket40300 @@ -0,0 +1,5 @@ + o Minor bugfixes (relay): + - Remove a spammy log notice that should not have been indicating the + operator that its IPv4/v6 was missing but it was not. Fixes bug 40300; + bugfix on 0.4.5.1-alpha. + diff --git a/src/feature/relay/relay_find_addr.c b/src/feature/relay/relay_find_addr.c index 39e1cc6a19..2a3f602438 100644 --- a/src/feature/relay/relay_find_addr.c +++ b/src/feature/relay/relay_find_addr.c @@ -144,17 +144,6 @@ relay_find_addr_to_publish, (const or_options_t *options, int family, if (find_my_address(options, family, LOG_INFO, addr_out, NULL, NULL)) { goto found; } - /* No publishable address was found even though we have an ORPort thus - * print a notice log so operator can notice. We'll do that every hour so - * it is not too spammy but enough so operators address the issue. */ - static ratelim_t rlim = RATELIM_INIT(3600); - log_fn_ratelim(&rlim, LOG_NOTICE, LD_CONFIG, - "Unable to find %s address for ORPort %u. " - "You might want to specify %sOnly to it or set an " - "explicit address or set Address.", - fmt_af_family(family), - routerconf_find_or_port(options, family), - fmt_af_family(family)); } /* Third, consider address from our suggestion cache. */ @@ -163,7 +152,19 @@ relay_find_addr_to_publish, (const or_options_t *options, int family, goto found; } - /* No publishable address was found. */ + /* No publishable address was found even though we have an ORPort thus + * print a notice log so operator can notice. We'll do that every hour so + * it is not too spammy but enough so operators address the issue. */ + static ratelim_t rlim = RATELIM_INIT(3600); + log_fn_ratelim(&rlim, LOG_NOTICE, LD_CONFIG, + "Unable to find %s address for ORPort %u. " + "You might want to specify %sOnly to it or set an " + "explicit address or set Address.", + fmt_af_family(family), + routerconf_find_or_port(options, family), + fmt_af_family(family)); + + /* Not found. */ return false; found: [View Less]

1 0

[tor/master] Merge branch 'maint-0.4.5'
by nickm＠torproject.org 22 Feb '21

22 Feb '21

commit 6e3a7c410f2c0cfd2f705862cc4d32acd0a88096 Merge: a4df1e8ea4 bc21ed3290 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Feb 22 15:37:39 2021 -0500 Merge branch 'maint-0.4.5' changes/ticket40300 | 5 +++++ src/feature/relay/relay_find_addr.c | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-)

1 0