commit bd28551763bd008be5a6f676410d9b6b1d011bf6 Author: George Kadianakis <desnacked@riseup.net> Date: Wed Oct 21 14:32:30 2020 +0300 Introduce v3_stats_t structure and some of its methods. --- src/app/main/main.c | 2 + src/core/mainloop/mainloop.c | 4 ++ src/feature/hs/hs_cache.c | 1 + src/feature/relay/router.c | 5 +++ src/feature/stats/rephist.c | 98 ++++++++++++++++++++++++++++++++++++++++++++ src/feature/stats/rephist.h | 8 +++- 6 files changed, 117 insertions(+), 1 deletion(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index ff530c0ad0..c2ced30851 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1070,6 +1070,7 @@ sandbox_init_filter(void) OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp"); OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp"); OPEN_DATADIR2_SUFFIX("stats", "hidserv-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "hidserv-v3-stats", ".tmp"); OPEN_DATADIR("approved-routers"); OPEN_DATADIR_SUFFIX("fingerprint", ".tmp"); @@ -1095,6 +1096,7 @@ sandbox_init_filter(void) RENAME_SUFFIX2("stats", "buffer-stats", ".tmp"); RENAME_SUFFIX2("stats", "conn-stats", ".tmp"); RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp"); + RENAME_SUFFIX2("stats", "hidserv-v3-stats", ".tmp"); RENAME_SUFFIX("hashed-fingerprint", ".tmp"); RENAME_SUFFIX("router-stability", ".tmp"); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 4b4ac5c4a7..25555a3f22 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1940,6 +1940,10 @@ write_stats_file_callback(time_t now, const or_options_t *options) time_t next_write = rep_hist_hs_v2_stats_write(now); if (next_write && next_write < next_time_to_write_stats_files) next_time_to_write_stats_files = next_write; + + next_write = rep_hist_hs_v3_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; } if (options->ExitPortStatistics) { time_t next_write = rep_hist_exit_stats_write(now); diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c index 44cd2505fd..0688d7765d 100644 --- a/src/feature/hs/hs_cache.c +++ b/src/feature/hs/hs_cache.c @@ -19,6 +19,7 @@ #include "feature/hs/hs_descriptor.h" #include "feature/nodelist/networkstatus.h" #include "feature/rend/rendcache.h" +#include "feature/stats/rephist.h" #include "feature/hs/hs_cache.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 5ca21964b6..ea631e18fe 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -3288,6 +3288,11 @@ extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, "hidserv-stats-end", now, &contents) > 0) { smartlist_add(chunks, contents); } + if (options->HiddenServiceStatistics && + load_stats_file("stats"PATH_SEPARATOR"hidserv-v3-stats", + "hidserv-v3-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } if (options->EntryStatistics && load_stats_file("stats"PATH_SEPARATOR"entry-stats", "entry-stats-end", now, &contents) > 0) { diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index bde65ea9d9..daf9db074c 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -1818,6 +1818,96 @@ rep_hist_hsdir_stored_maybe_new_v2_onion(const crypto_pk_t *pubkey) } } +/*** HSv3 stats ******/ + +/** Start of the current hidden service stats interval or 0 if we're not + * collecting hidden service statistics. + * + * This is particularly important for v3 statistics since this variable + * controls the start time of initial v3 stats collection. It's initialized by + * rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and + * should_collect_v3_stats() ensures that functions that collect v3 stats do + * not do so sooner than that. + * + * Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3 + * stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded + * key of each onion service as its double-counting index. Onion services + * rotate their descriptor at around 00:00UTC which means that their blinded + * key also changes around that time. However the precise time that onion + * services rotate their descriptors is actually when they fetch a new + * 00:00UTC consensus and that happens at a random time (e.g. it can even + * happen at 02:00UTC). This means that if we started keeping v3 stats at + * around 00:00UTC we wouldn't be able to tell when onion services change + * their blinded key and hence we would double count an unpredictable amount + * of them (for example, if an onion service fetches the 00:00UTC consensus at + * 01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a + * different HSDir at 01:50UTC). + * + * For this reason, we start collecting statistics at 12:00UTC. This way we + * know that by the time we stop collecting statistics for that time period 24 + * hours later, all the onion services have switched to their new blinded + * key. This way we can predict much better how much double counting has been + * performed. + */ +static time_t start_of_hs_v3_stats_interval; + +/** Our v3 statistics structure singleton. */ +static hs_v3_stats_t *hs_v3_stats = NULL; + +/** Allocate, initialize and return an hs_v3_stats_t structure. */ +static hs_v3_stats_t * +hs_v3_stats_new(void) +{ + hs_v3_stats_t *new_hs_v3_stats = tor_malloc_zero(sizeof(hs_v3_stats_t)); + new_hs_v3_stats->v3_onions_seen_this_period = digestmap_new(); + + return new_hs_v3_stats; +} + +#define hs_v3_stats_free(val) \ + FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val)) + +/** Free an hs_v3_stats_t structure. */ +static void +hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats) +{ + if (!victim_hs_v3_stats) { + return; + } + + digestmap_free(victim_hs_v3_stats->v3_onions_seen_this_period, NULL); + tor_free(victim_hs_v3_stats); +} + +/** Clear history of hidden service statistics and set the measurement + * interval start to <b>now</b>. */ +static void +rep_hist_reset_hs_v3_stats(time_t now) +{ + if (!hs_v3_stats) { + hs_v3_stats = hs_v3_stats_new(); + } + + digestmap_free(hs_v3_stats->v3_onions_seen_this_period, NULL); + hs_v3_stats->v3_onions_seen_this_period = digestmap_new(); + + hs_v3_stats->rp_v3_relay_cells_seen = 0; + + start_of_hs_v3_stats_interval = now; +} + +/** Return true if it's a good time to collect v3 stats. + * + * v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC + * on the real network) and hence we don't want to collect statistics if it's + * not yet the time to do so. + */ +static bool +should_collect_v3_stats(void) +{ + return start_of_hs_v3_stats_interval <= approx_time(); +} + /* The number of cells that are supposed to be hidden from the adversary * by adding noise from the Laplace distribution. This value, divided by * EPSILON, is Laplace parameter b. It must be greather than 0. */ @@ -2125,6 +2215,7 @@ void rep_hist_free_all(void) { hs_v2_stats_free(hs_v2_stats); + hs_v3_stats_free(hs_v3_stats); digestmap_free(history_map, free_or_history); tor_free(exit_bytes_read); @@ -2154,3 +2245,10 @@ rep_hist_get_hs_v2_stats(void) return hs_v2_stats; } +/* only exists for unit tests: get HSv2 stats object */ +const hs_v3_stats_t * +rep_hist_get_hs_v3_stats(void) +{ + return hs_v3_stats; +} +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index d5ad21e228..e6c1509498 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -69,6 +69,9 @@ char *rep_hist_get_hs_v2_stats_string(void); void rep_hist_seen_new_rp_cell(void); void rep_hist_hsdir_stored_maybe_new_v2_onion(const crypto_pk_t *pubkey); +time_t rep_hist_hs_v3_stats_write(time_t now); +char *rep_hist_get_hs_v3_stats_string(void); + void rep_hist_free_all(void); void rep_hist_note_negotiated_link_proto(unsigned link_proto, @@ -111,7 +114,8 @@ typedef struct hs_v3_stats_t { digestmap_t *v3_onions_seen_this_period; } hs_v3_stats_t; -STATIC char *rep_hist_format_hs_v2_stats(time_t now, bool is_v3); +STATIC char *rep_hist_format_hs_v2_stats(time_t now); +STATIC char *rep_hist_format_hs_v3_stats(time_t now); #endif /* defined(REPHIST_PRIVATE) */ /** @@ -142,6 +146,8 @@ void rep_hist_padding_count_timers(uint64_t num_timers); #ifdef TOR_UNIT_TESTS typedef struct hs_v2_stats_t hs_v2_stats_t; const hs_v2_stats_t *rep_hist_get_hs_v2_stats(void); +typedef struct hs_v3_stats_t hs_v3_stats_t; +const hs_v3_stats_t *rep_hist_get_hs_v3_stats(void); #endif #endif /* !defined(TOR_REPHIST_H) */