commit 7c06707750f549fc22b74bdba7b9743d7b536e19 Merge: 6c61011781 0812ecd517 Author: David Goulet <dgoulet@torproject.org> Date: Tue Nov 17 10:36:05 2020 -0500 Merge branch 'tor-gitlab/mr/182' into master changes/bug23126 | 4 + src/app/main/main.c | 2 + src/core/mainloop/mainloop.c | 6 +- src/core/or/command.c | 29 ++- src/core/or/or_circuit_st.h | 6 + src/feature/hs/hs_cache.c | 6 +- src/feature/hs_common/shared_random_client.c | 21 +- src/feature/relay/router.c | 5 + src/feature/rend/rendcache.c | 2 +- src/feature/stats/rephist.c | 346 +++++++++++++++++++-------- src/feature/stats/rephist.h | 53 +++- src/test/hs_test_helpers.c | 19 +- src/test/hs_test_helpers.h | 4 + src/test/test_stats.c | 129 ++++++++++ 14 files changed, 513 insertions(+), 119 deletions(-) diff --cc src/feature/stats/rephist.c index 3c22fda3b8,1501e46b14..59f38fe603 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@@ -1765,71 -1851,110 +1851,110 @@@ hs_v3_stats_free_(hs_v3_stats_t *victim /** Clear history of hidden service statistics and set the measurement * interval start to <b>now</b>. */ static void - rep_hist_reset_hs_stats(time_t now) + rep_hist_reset_hs_v3_stats(time_t now) { - if (!hs_stats) { - hs_stats = hs_stats_new(); + if (!hs_v3_stats) { + hs_v3_stats = hs_v3_stats_new(); } - hs_stats->rp_relay_cells_seen = 0; + digest256map_free(hs_v3_stats->v3_onions_seen_this_period, NULL); + hs_v3_stats->v3_onions_seen_this_period = digest256map_new(); - digestmap_free(hs_stats->onions_seen_this_period, NULL); - hs_stats->onions_seen_this_period = digestmap_new(); + hs_v3_stats->rp_v3_relay_cells_seen = 0; - start_of_hs_stats_interval = now; + start_of_hs_v3_stats_interval = now; } - /** Stop collecting hidden service stats in a way that we can re-start - * doing so in rep_hist_buffer_stats_init(). */ - void - rep_hist_hs_stats_term(void) + /** Return true if it's a good time to collect v3 stats. + * + * v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC + * on the real network). We don't want to collect statistics if (for example) + * we just booted and it's 03:00UTC; we will wait until 12:00UTC before we + * start collecting statistics to make sure that the final result represents + * the whole collection period. This behavior is controlled by + * rep_hist_hs_stats_init(). + */ + MOCK_IMPL(STATIC bool, + should_collect_v3_stats,(void)) { - rep_hist_reset_hs_stats(0); + return start_of_hs_v3_stats_interval <= approx_time(); } - /** We saw a new HS relay cell, Count it! */ + /** We just received a new descriptor with <b>blinded_key</b>. See if we've + * seen this blinded key before, and if not add it to the stats. */ void - rep_hist_seen_new_rp_cell(void) + rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key) { - if (!hs_stats) { - return; // We're not collecting stats + /* Return early if we don't collect HSv3 stats, or if it's not yet the time + * to collect them. */ + if (!hs_v3_stats || !should_collect_v3_stats()) { + return; } - hs_stats->rp_relay_cells_seen++; + bool seen_before = + !!digest256map_get(hs_v3_stats->v3_onions_seen_this_period, + blinded_key); + + log_info(LD_GENERAL, "Considering v3 descriptor with %s (%sseen before)", + safe_str(hex_str((char*)blinded_key, 32)), + seen_before ? "" : "not "); + + /* Count it if we haven't seen it before. */ + if (!seen_before) { + digest256map_set(hs_v3_stats->v3_onions_seen_this_period, + blinded_key, (void*)(uintptr_t)1); + } } - /** As HSDirs, we saw another hidden service with public key - * <b>pubkey</b>. Check whether we have counted it before, if not - * count it now! */ + /** We saw a new HS relay cell: count it! + * If <b>is_v2</b> is set then it's a v2 RP cell, otherwise it's a v3. */ void - rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey) + rep_hist_seen_new_rp_cell(bool is_v2) { - char pubkey_hash[DIGEST_LEN]; + log_debug(LD_GENERAL, "New RP cell (%d)", is_v2); - if (!hs_stats) { - return; // We're not collecting stats + if (is_v2 && hs_v2_stats) { + hs_v2_stats->rp_v2_relay_cells_seen++; + } else if (!is_v2 && hs_v3_stats && should_collect_v3_stats()) { + hs_v3_stats->rp_v3_relay_cells_seen++; } + } - /* Get the digest of the pubkey which will be used to detect whether - we've seen this hidden service before or not. */ - if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) { - /* This fail should not happen; key has been validated by - descriptor parsing code first. */ - return; + /** Generic HS stats code */ + + /** Initialize v2 and v3 hidden service statistics. */ + void + rep_hist_hs_stats_init(time_t now) + { + if (!hs_v2_stats) { + hs_v2_stats = hs_v2_stats_new(); } - /* Check if this is the first time we've seen this hidden - service. If it is, count it as new. */ - if (!digestmap_get(hs_stats->onions_seen_this_period, - pubkey_hash)) { - digestmap_set(hs_stats->onions_seen_this_period, - pubkey_hash, (void*)(uintptr_t)1); + /* Start collecting v2 stats straight away */ + start_of_hs_v2_stats_interval = now; + + if (!hs_v3_stats) { + hs_v3_stats = hs_v3_stats_new(); } + + /* Start collecting v3 stats at the next 12:00 UTC */ + start_of_hs_v3_stats_interval = hs_get_start_time_of_next_time_period(now); + } + + /** Stop collecting hidden service stats in a way that we can re-start + * doing so in rep_hist_buffer_stats_init(). */ + void + rep_hist_hs_stats_term(void) + { + rep_hist_reset_hs_v2_stats(0); + rep_hist_reset_hs_v3_stats(0); } + /** Stats reporting code */ + /* The number of cells that are supposed to be hidden from the adversary * by adding noise from the Laplace distribution. This value, divided by - * EPSILON, is Laplace parameter b. It must be greather than 0. */ + * EPSILON, is Laplace parameter b. It must be greater than 0. */ #define REND_CELLS_DELTA_F 2048 /* Security parameter for obfuscating number of cells with a value between * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same