July 2020 - tor-commits - lists.torproject.org

[tor/master] trace: Add four generic circuit tracepoints
by nickm＠torproject.org 10 Jul '20

10 Jul '20

commit d36a44ffa965e69a943068ab457a6528ef204b00 Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Jan 15 12:36:18 2020 -0500 trace: Add four generic circuit tracepoints Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/core/or/circuitlist.c | 9 +++++++ src/core/or/circuituse.c | 1 + src/core/or/trace_probes_circuit.h | 54 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index f4d6cd3c1..10fba498a 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -65,6 +65,7 @@ #include "core/or/circuitpadding.h" #include "core/or/crypt_path.h" #include "core/or/extendinfo.h" +#include "core/or/trace_probes_circuit.h" #include "core/mainloop/connection.h" #include "app/config/config.h" #include "core/or/connection_edge.h" @@ -99,6 +100,7 @@ #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/buf/buffers.h" +#include "lib/trace/events.h" #include "core/or/ocirc_event.h" @@ -565,6 +567,8 @@ circuit_set_state(circuit_t *circ, uint8_t state) } if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); + + tor_trace(circuit, change_state, circ, circ->state, state); circ->state = state; if (CIRCUIT_IS_ORIGIN(circ)) circuit_state_publish(circ); @@ -1253,6 +1257,10 @@ circuit_free_(circuit_t *circ) /* Clear all dangling handle references. */ circuit_handles_clear(circ); + /* Tracepoint. Data within the circuit object is recorded so do this before + * the actual memory free. */ + tor_trace(circuit, free, circ); + if (should_free) { memwipe(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); @@ -2275,6 +2283,7 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line, CIRCUIT_IS_ORIGIN(circ) ? TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, file, line, orig_reason, reason); + tor_trace(circuit, mark_for_close, circ); } /** Called immediately before freeing a marked circuit circ from diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index b9c15c155..ac03b76d5 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -3144,6 +3144,7 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) old_purpose = circ->purpose; circ->purpose = new_purpose; + tor_trace(circuit, change_purpose, circ, old_purpose, new_purpose); if (CIRCUIT_IS_ORIGIN(circ)) { control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ), diff --git a/src/core/or/trace_probes_circuit.h b/src/core/or/trace_probes_circuit.h index c0dbd1a50..e306a7886 100644 --- a/src/core/or/trace_probes_circuit.h +++ b/src/core/or/trace_probes_circuit.h @@ -163,6 +163,60 @@ TRACEPOINT_EVENT_INSTANCE(tor_circuit, origin_circuit_t_class, idle_timeout, TP_ARGS(const origin_circuit_t *, circ) ) +/* + * General circuit events. + */ + +TRACEPOINT_EVENT(tor_circuit, free, + TP_ARGS(const circuit_t *, circ), + TP_FIELDS( + ctf_integer(uint32_t, circ_id, + (CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0)) + ctf_enum(tor_circuit, purpose, int, purpose, circ->purpose) + ctf_enum(tor_circuit, state, int, state, circ->state) + ) +) + +TRACEPOINT_EVENT(tor_circuit, mark_for_close, + TP_ARGS(const circuit_t *, circ), + TP_FIELDS( + ctf_integer(uint32_t, circ_id, + (CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0)) + ctf_enum(tor_circuit, purpose, int, purpose, circ->purpose) + ctf_enum(tor_circuit, state, int, state, circ->state) + ctf_enum(tor_circuit, end_reason, int, close_reason, + circ->marked_for_close_reason) + ctf_enum(tor_circuit, end_reason, int, orig_close_reason, + circ->marked_for_close_orig_reason) + ) +) + +TRACEPOINT_EVENT(tor_circuit, change_purpose, + TP_ARGS(const circuit_t *, circ, int, old_purpose, int, new_purpose), + TP_FIELDS( + ctf_integer(uint32_t, circ_id, + (CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0)) + ctf_enum(tor_circuit, state, int, state, circ->state) + ctf_enum(tor_circuit, purpose, int, purpose, old_purpose) + ctf_enum(tor_circuit, purpose, int, new, new_purpose) + ) +) + +TRACEPOINT_EVENT(tor_circuit, change_state, + TP_ARGS(const circuit_t *, circ, int, old_state, int, new_state), + TP_FIELDS( + ctf_integer(uint32_t, circ_id, + (CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0)) + ctf_enum(tor_circuit, purpose, int, purpose, circ->purpose) + ctf_enum(tor_circuit, state, int, old, old_state) + ctf_enum(tor_circuit, state, int, new, new_state) + ) +) + #endif /* TOR_TRACE_PROBES_CIRCUIT_H */ /* Must be include after the probes declaration. */

1 0

[tor/master] doc: Add a WARNING section to Tracing.md
by nickm＠torproject.org 10 Jul '20

10 Jul '20

commit 942ecfa835bc50ed11912df034abf5f184d46033 Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Mar 11 12:26:10 2020 -0400 doc: Add a WARNING section to Tracing.md Explain what is safe or not with tracing data. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- doc/HACKING/Tracing.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/HACKING/Tracing.md b/doc/HACKING/Tracing.md index 8cf68321a..d898bee17 100644 --- a/doc/HACKING/Tracing.md +++ b/doc/HACKING/Tracing.md @@ -4,7 +4,25 @@ This document describes how the event tracing subsystem works in tor so developers can add events to the code base but also hook them to an event tracing framework (i.e. tracer). -## Basics +## WARNING ## + +Tracing the tor daemon **always** generates sensitive data if used in +production (on the public network). + +It **is** ethical for researchers to use tracing for their own tor client (for +example: building paths, timings, or performance). + +It is **NOT** ethical to archive, publish or keep data containing other users' +activity such as relay data or anything that handles users' traffic. This +of course includes any logs below notice level. + +Publishing analysis of tracing data containing user traffic is **NOT** safe +either. + +In other words, tracing data that contains other users's activity is **NOT** +safe to publish in any form. + +## Basics ### Tracing is separated in two different concepts. The tracing API and the tracing probes.

1 0

[tor/master] changes: Add changes file for #32910
by nickm＠torproject.org 10 Jul '20

10 Jul '20

commit dfaa0a82acdfbd65dde0a30fa8fd598304a86816 Author: David Goulet <dgoulet(a)torproject.org> Date: Fri Jul 10 12:12:26 2020 -0400 changes: Add changes file for #32910 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket32910 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changes/ticket32910 b/changes/ticket32910 new file mode 100644 index 000000000..e3d64d433 --- /dev/null +++ b/changes/ticket32910 @@ -0,0 +1,5 @@ + o Major feature (tracing): + - Add a tracing library with USDT and LTTng-UST support. Few tracepoints + were added in the circuit subsystem. More will come incrementally. This + feature is compiled out by default. It needs to be enabled at configure + time. See documentation in doc/HACKING/Tracing.md. Closes ticket 32910.

1 0

[tor/master] Merge remote-tracking branch 'tor-gitlab/mr/29'
by nickm＠torproject.org 10 Jul '20

10 Jul '20

1 0

[tor/master] Split bandwidth history functions into a separate C file.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 8390df917b7e63696c70037765737037cd9162a0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 07:50:17 2020 -0400 Split bandwidth history functions into a separate C file. These are logically independent from the rest of rephist, and make more sense in isolation. The next patch will rename them too. --- src/app/config/statefile.c | 2 +- src/app/main/main.c | 2 + src/app/main/shutdown.c | 2 + src/core/mainloop/connection.c | 1 + src/core/or/circuitlist.c | 1 + src/feature/relay/router.c | 1 + src/feature/stats/bwhist.c | 588 +++++++++++++++++++++++++++++++++++++++++ src/feature/stats/bwhist.h | 40 +++ src/feature/stats/include.am | 2 + src/feature/stats/rephist.c | 577 ---------------------------------------- src/feature/stats/rephist.h | 17 -- src/test/test_relay.c | 4 +- src/test/test_router.c | 2 +- src/test/testing_common.c | 2 + 14 files changed, 643 insertions(+), 598 deletions(-) diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index dcc55f189..63205007c 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -40,7 +40,7 @@ #include "feature/control/control_events.h" #include "feature/client/entrynodes.h" #include "feature/hibernate/hibernate.h" -#include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "feature/relay/router.h" #include "feature/relay/routermode.h" #include "lib/sandbox/sandbox.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index 89ba78742..7b6a665b7 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -53,6 +53,7 @@ #include "feature/rend/rendcache.h" #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" #include "lib/buf/buffers.h" @@ -549,6 +550,7 @@ tor_init(int argc, char *argv[]) /* Initialize the history structures. */ rep_hist_init(); + bwhist_init(); /* Initialize the service cache. */ rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index aac15246b..4a556333d 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -47,6 +47,7 @@ #include "feature/relay/relay_config.h" #include "feature/rend/rendcache.h" #include "feature/rend/rendclient.h" +#include "feature/stats/bwhist.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/evloop/compat_libevent.h" @@ -121,6 +122,7 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); + bwhist_free_all(); circuit_free_all(); circpad_machines_free(); entry_guards_free_all(); diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index af823335a..7f1b99311 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -106,6 +106,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/geoip/geoip.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index f4d6cd3c1..a6949d943 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -90,6 +90,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/stats/predict_ports.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 57da735e8..a55b30067 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -44,6 +44,7 @@ #include "feature/relay/selftest.h" #include "lib/geoip/geoip.h" #include "feature/stats/geoip_stats.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_format.h" diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c new file mode 100644 index 000000000..a3b971a25 --- /dev/null +++ b/src/feature/stats/bwhist.c @@ -0,0 +1,588 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file bwhist.c + * @brief Tracking for relay bandwidth history + * + * This module handles bandwidth usage history, used by relays to + * self-report how much bandwidth they've used for different + * purposes over last day or so, in order to generate the + * {dirreq-,}{read,write}-history lines in that they publish. + **/ + +#define BWHIST_PRIVATE +#include "orconfig.h" +#include "core/or/or.h" +#include "feature/stats/bwhist.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "feature/relay/routermode.h" + +#include "app/config/or_state_st.h" +#include "app/config/or_options_st.h" + +/** For how many seconds do we keep track of individual per-second bandwidth + * totals? */ +#define NUM_SECS_ROLLING_MEASURE 10 +/** How large are the intervals for which we track and report bandwidth use? */ +#define NUM_SECS_BW_SUM_INTERVAL (24*60*60) +/** How far in the past do we remember and publish bandwidth use? */ +#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60) +/** How many bandwidth usage intervals do we remember? (derived) */ +#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL) + +/** Structure to track bandwidth use, and remember the maxima for a given + * time period. + */ +struct bw_array_t { + /** Observation array: Total number of bytes transferred in each of the last + * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */ + uint64_t obs[NUM_SECS_ROLLING_MEASURE]; + int cur_obs_idx; /**< Current position in obs. */ + time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */ + uint64_t total_obs; /**< Total for all members of obs except + * obs[cur_obs_idx] */ + uint64_t max_total; /**< Largest value that total_obs has taken on in the + * current period. */ + uint64_t total_in_period; /**< Total bytes transferred in the current + * period. */ + + /** When does the next period begin? */ + time_t next_period; + /** Where in 'maxima' should the maximum bandwidth usage for the current + * period be stored? */ + int next_max_idx; + /** How many values in maxima/totals have been set ever? */ + int num_maxes_set; + /** Circular array of the maximum + * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last + * NUM_TOTALS periods */ + uint64_t maxima[NUM_TOTALS]; + /** Circular array of the total bandwidth usage for the last NUM_TOTALS + * periods */ + uint64_t totals[NUM_TOTALS]; +}; + +/** Shift the current period of b forward by one. */ +STATIC void +commit_max(bw_array_t *b) +{ + /* Store total from current period. */ + b->totals[b->next_max_idx] = b->total_in_period; + /* Store maximum from current period. */ + b->maxima[b->next_max_idx++] = b->max_total; + /* Advance next_period and next_max_idx */ + b->next_period += NUM_SECS_BW_SUM_INTERVAL; + if (b->next_max_idx == NUM_TOTALS) + b->next_max_idx = 0; + if (b->num_maxes_set < NUM_TOTALS) + ++b->num_maxes_set; + /* Reset max_total. */ + b->max_total = 0; + /* Reset total_in_period. */ + b->total_in_period = 0; +} + +/** Shift the current observation time of b forward by one second. */ +STATIC void +advance_obs(bw_array_t *b) +{ + int nextidx; + uint64_t total; + + /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE + * seconds; adjust max_total as needed.*/ + total = b->total_obs + b->obs[b->cur_obs_idx]; + if (total > b->max_total) + b->max_total = total; + + nextidx = b->cur_obs_idx+1; + if (nextidx == NUM_SECS_ROLLING_MEASURE) + nextidx = 0; + + b->total_obs = total - b->obs[nextidx]; + b->obs[nextidx]=0; + b->cur_obs_idx = nextidx; + + if (++b->cur_obs_time >= b->next_period) + commit_max(b); +} + +/** Add n bytes to the number of bytes in b for second + * when. */ +static inline void +add_obs(bw_array_t *b, time_t when, uint64_t n) +{ + if (when < b->cur_obs_time) + return; /* Don't record data in the past. */ + + /* If we're currently adding observations for an earlier second than + * 'when', advance b->cur_obs_time and b->cur_obs_idx by an + * appropriate number of seconds, and do all the other housekeeping. */ + while (when > b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ + advance_obs(b); + } + + b->obs[b->cur_obs_idx] += n; + b->total_in_period += n; +} + +/** Allocate, initialize, and return a new bw_array. */ +static bw_array_t * +bw_array_new(void) +{ + bw_array_t *b; + time_t start; + b = tor_malloc_zero(sizeof(bw_array_t)); + start = time(NULL); + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + return b; +} + +#define bw_array_free(val) \ + FREE_AND_NULL(bw_array_t, bw_array_free_, (val)) + +/** Free storage held by bandwidth array b. */ +static void +bw_array_free_(bw_array_t *b) +{ + if (!b) { + return; + } + + tor_free(b); +} + +/** Recent history of bandwidth observations for read operations. */ +static bw_array_t *read_array = NULL; +/** Recent history of bandwidth observations for write operations. */ +STATIC bw_array_t *write_array = NULL; +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up structures for bandwidth history, clearing them if they already + * exist. */ +void +bwhist_init(void) +{ + bw_array_free(read_array); + bw_array_free(write_array); + bw_array_free(dir_read_array); + bw_array_free(dir_write_array); + + read_array = bw_array_new(); + write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); +} + +/** Remember that we read num_bytes bytes in second when. + * + * Add num_bytes to the current running total for when. + * + * when can go back to time, but it's safe to ignore calls + * earlier than the latest when you've heard of. + */ +void +rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) +{ +/* Maybe a circular array for recent seconds, and step to a new point + * every time a new second shows up. Or simpler is to just to have + * a normal array and push down each item every second; it's short. + */ +/* When a new second has rolled over, compute the sum of the bytes we've + * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it + * somewhere. See rep_hist_bandwidth_assess() below. + */ + add_obs(write_array, when, num_bytes); +} + +/** Remember that we wrote num_bytes bytes in second when. + * (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_bytes_read(uint64_t num_bytes, time_t when) +{ +/* if we're smart, we can make this func and the one above share code */ + add_obs(read_array, when, num_bytes); +} + +/** Remember that we wrote num_bytes directory bytes in second + * when. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when) +{ + add_obs(dir_write_array, when, num_bytes); +} + +/** Remember that we read num_bytes directory bytes in second + * when. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when) +{ + add_obs(dir_read_array, when, num_bytes); +} + +/** Helper: Return the largest value in b->maxima. (This is equal to the + * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last + * NUM_SECS_BW_SUM_IS_VALID seconds.) + */ +STATIC uint64_t +find_largest_max(bw_array_t *b) +{ + int i; + uint64_t max; + max=0; + for (i=0; i<NUM_TOTALS; ++i) { + if (b->maxima[i]>max) + max = b->maxima[i]; + } + return max; +} + +/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly) + * seconds. Find one sum for reading and one for writing. They don't have + * to be at the same time. + * + * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. + */ +MOCK_IMPL(int, +rep_hist_bandwidth_assess,(void)) +{ + uint64_t w,r; + r = find_largest_max(read_array); + w = find_largest_max(write_array); + if (r>w) + return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); + else + return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE); +} + +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. + * + * It returns the number of bytes written. + */ +static size_t +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) +{ + char *cp = buf; + int i, n; + const or_options_t *options = get_options(); + uint64_t cutoff; + + if (b->num_maxes_set <= b->next_max_idx) { + /* We haven't been through the circular array yet; time starts at i=0.*/ + i = 0; + } else { + /* We've been around the array at least once. The next i to be + overwritten is the oldest. */ + i = b->next_max_idx; + } + + if (options->RelayBandwidthRate) { + /* We don't want to report that we used more bandwidth than the max we're + * willing to relay; otherwise everybody will know how much traffic + * we used ourself. */ + cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL; + } else { + cutoff = UINT64_MAX; + } + + for (n=0; n<b->num_maxes_set; ++n,++i) { + uint64_t total; + if (i >= NUM_TOTALS) + i -= NUM_TOTALS; + tor_assert(i < NUM_TOTALS); + /* Round the bandwidth used down to the nearest 1k. */ + total = b->totals[i] & ~0x3ff; + if (total > cutoff) + total = cutoff; + + if (n==(b->num_maxes_set-1)) + tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total)); + else + tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total)); + cp += strlen(cp); + } + return cp-buf; +} + +/** Allocate and return lines for representing this server's bandwidth + * history in its descriptor. We publish these lines in our extra-info + * descriptor. + */ +char * +rep_hist_get_bandwidth_lines(void) +{ + char *buf, *cp; + char t[ISO_TIME_LEN+1]; + int r; + bw_array_t *b = NULL; + const char *desc = NULL; + size_t len; + + /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ +/* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) + len = (67+MAX_HIST_VALUE_LEN)*4; + buf = tor_malloc_zero(len); + cp = buf; + for (r=0;r<4;++r) { + char tmp[MAX_HIST_VALUE_LEN]; + size_t slen; + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } + tor_assert(b); + slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + continue; + format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); + tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", + desc, t, NUM_SECS_BW_SUM_INTERVAL); + cp += strlen(cp); + strlcat(cp, tmp, len-(cp-buf)); + cp += slen; + strlcat(cp, "\n", len-(cp-buf)); + ++cp; + } + return buf; +} + +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. Done before writing out a new state file. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + int i,j; + uint64_t maxval; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_new(); + *s_maxima = smartlist_new(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_new(); + *s_maxima = smartlist_new(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + if (i >= NUM_TOTALS) + i = 0; + smartlist_add_asprintf(*s_values, "%"PRIu64, + (b->totals[i] & ~0x3ff)); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + smartlist_add_asprintf(*s_maxima, "%"PRIu64, + (maxval & ~0x3ff)); + } + smartlist_add_asprintf(*s_values, "%"PRIu64, + (b->total_in_period & ~0x3ff)); + maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; + smartlist_add_asprintf(*s_maxima, "%"PRIu64, + (maxval & ~0x3ff)); +} + +/** Update state with the newest bandwidth history. Done before + * writing out a new state file. */ +void +rep_hist_update_state(or_state_t *state) +{ +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) + + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); + + if (server_mode(get_options())) { + or_state_mark_dirty(state, time(NULL)+(2*3600)); + } +#undef UPDATE +} + +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. Done while reading the state file. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) +{ + time_t now = time(NULL); + int retval = 0; + time_t start; + + uint64_t v, mv; + int i,ok,ok_m = 0; + int have_maxima = s_maxima && s_values && + (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; + v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; + } + if (!ok) { + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); + } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + + if (start < now) { + time_t cur_start = start; + time_t actual_interval_len = s_interval; + uint64_t cur_val = 0; + /* Calculate the average per second. This is the best we can do + * because our state file doesn't have per-second resolution. */ + if (start + s_interval > now) + actual_interval_len = now - start; + cur_val = v / actual_interval_len; + /* This is potentially inefficient, but since we don't do it very + * often it should be ok. */ + while (cur_start < start + actual_interval_len) { + add_obs(b, cur_start, cur_val); + ++cur_start; + } + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += actual_interval_len; + } + } SMARTLIST_FOREACH_END(cp); + } + + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; + } + b->total_obs = 0; + + return retval; +} + +/** Set bandwidth history from the state file we just loaded. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD + if (!all_ok) { + *err = tor_strdup("Parsing of bandwidth history values failed"); + /* and create fresh arrays */ + bwhist_init(); + return -1; + } + return 0; +} + +void +bwhist_free_all(void) +{ + bw_array_free(read_array); + bw_array_free(write_array); + bw_array_free(dir_read_array); + bw_array_free(dir_write_array); +} diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h new file mode 100644 index 000000000..afe0d9f4e --- /dev/null +++ b/src/feature/stats/bwhist.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file bwhist.h + * @brief Header for feature/stats/bwhist.c + **/ + +#ifndef TOR_FEATURE_STATS_BWHIST_H +#define TOR_FEATURE_STATS_BWHIST_H + +void bwhist_init(void); +void bwhist_free_all(void); + +void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when); +void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when); + +MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); +char *rep_hist_get_bandwidth_lines(void); +struct or_state_t; +void rep_hist_update_state(struct or_state_t *state); +int rep_hist_load_state(struct or_state_t *state, char **err); + +#ifdef BWHIST_PRIVATE +typedef struct bw_array_t bw_array_t; +STATIC uint64_t find_largest_max(bw_array_t *b); +STATIC void commit_max(bw_array_t *b); +STATIC void advance_obs(bw_array_t *b); +#endif /* defined(REPHIST_PRIVATE) */ + +#ifdef TOR_UNIT_TESTS +extern struct bw_array_t *write_array; +#endif + +#endif /* !defined(TOR_FEATURE_STATS_BWHIST_H) */ diff --git a/src/feature/stats/include.am b/src/feature/stats/include.am index 8789bc3d9..32e760a08 100644 --- a/src/feature/stats/include.am +++ b/src/feature/stats/include.am @@ -1,12 +1,14 @@ # ADD_C_FILE: INSERT SOURCES HERE. LIBTOR_APP_A_SOURCES += \ + src/feature/stats/bwhist.c \ src/feature/stats/geoip_stats.c \ src/feature/stats/rephist.c \ src/feature/stats/predict_ports.c # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ + src/feature/stats/bwhist.h \ src/feature/stats/geoip_stats.h \ src/feature/stats/rephist.h \ src/feature/stats/predict_ports.h diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index d4826cd1a..6f409a46f 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -18,11 +18,6 @@ * stability information about various relays, including "uptime", * "weighted fractional uptime" and "mean time between failures". * - * <li>Bandwidth usage history, used by relays to self-report how much - * bandwidth they've used for different purposes over last day or so, - * in order to generate the {dirreq-,}{read,write}-history lines in - * that they publish. - * * <li>Predicted ports, used by clients to remember how long it's been * since they opened an exit connection to each given target * port. Clients use this information in order to try to keep circuits @@ -77,13 +72,11 @@ #define REPHIST_PRIVATE #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/statefile.h" #include "core/or/circuitlist.h" #include "core/or/connection_or.h" #include "feature/dirauth/authmode.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" -#include "feature/relay/routermode.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/container/order.h" @@ -92,14 +85,11 @@ #include "feature/nodelist/networkstatus_st.h" #include "core/or/or_circuit_st.h" -#include "app/config/or_state_st.h" #ifdef HAVE_FCNTL_H #include <fcntl.h> #endif -static void bw_arrays_init(void); - /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; /** Number of or_history_t objects currently allocated. */ @@ -232,7 +222,6 @@ void rep_hist_init(void) { history_map = digestmap_new(); - bw_arrays_init(); } /** We have just decided that this router with identity digest id is @@ -973,560 +962,6 @@ rep_hist_load_mtbf_data(time_t now) return r; } -/** For how many seconds do we keep track of individual per-second bandwidth - * totals? */ -#define NUM_SECS_ROLLING_MEASURE 10 -/** How large are the intervals for which we track and report bandwidth use? */ -#define NUM_SECS_BW_SUM_INTERVAL (24*60*60) -/** How far in the past do we remember and publish bandwidth use? */ -#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60) -/** How many bandwidth usage intervals do we remember? (derived) */ -#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL) - -/** Structure to track bandwidth use, and remember the maxima for a given - * time period. - */ -struct bw_array_t { - /** Observation array: Total number of bytes transferred in each of the last - * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */ - uint64_t obs[NUM_SECS_ROLLING_MEASURE]; - int cur_obs_idx; /**< Current position in obs. */ - time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */ - uint64_t total_obs; /**< Total for all members of obs except - * obs[cur_obs_idx] */ - uint64_t max_total; /**< Largest value that total_obs has taken on in the - * current period. */ - uint64_t total_in_period; /**< Total bytes transferred in the current - * period. */ - - /** When does the next period begin? */ - time_t next_period; - /** Where in 'maxima' should the maximum bandwidth usage for the current - * period be stored? */ - int next_max_idx; - /** How many values in maxima/totals have been set ever? */ - int num_maxes_set; - /** Circular array of the maximum - * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last - * NUM_TOTALS periods */ - uint64_t maxima[NUM_TOTALS]; - /** Circular array of the total bandwidth usage for the last NUM_TOTALS - * periods */ - uint64_t totals[NUM_TOTALS]; -}; - -/** Shift the current period of b forward by one. */ -STATIC void -commit_max(bw_array_t *b) -{ - /* Store total from current period. */ - b->totals[b->next_max_idx] = b->total_in_period; - /* Store maximum from current period. */ - b->maxima[b->next_max_idx++] = b->max_total; - /* Advance next_period and next_max_idx */ - b->next_period += NUM_SECS_BW_SUM_INTERVAL; - if (b->next_max_idx == NUM_TOTALS) - b->next_max_idx = 0; - if (b->num_maxes_set < NUM_TOTALS) - ++b->num_maxes_set; - /* Reset max_total. */ - b->max_total = 0; - /* Reset total_in_period. */ - b->total_in_period = 0; -} - -/** Shift the current observation time of b forward by one second. */ -STATIC void -advance_obs(bw_array_t *b) -{ - int nextidx; - uint64_t total; - - /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE - * seconds; adjust max_total as needed.*/ - total = b->total_obs + b->obs[b->cur_obs_idx]; - if (total > b->max_total) - b->max_total = total; - - nextidx = b->cur_obs_idx+1; - if (nextidx == NUM_SECS_ROLLING_MEASURE) - nextidx = 0; - - b->total_obs = total - b->obs[nextidx]; - b->obs[nextidx]=0; - b->cur_obs_idx = nextidx; - - if (++b->cur_obs_time >= b->next_period) - commit_max(b); -} - -/** Add n bytes to the number of bytes in b for second - * when. */ -static inline void -add_obs(bw_array_t *b, time_t when, uint64_t n) -{ - if (when < b->cur_obs_time) - return; /* Don't record data in the past. */ - - /* If we're currently adding observations for an earlier second than - * 'when', advance b->cur_obs_time and b->cur_obs_idx by an - * appropriate number of seconds, and do all the other housekeeping. */ - while (when > b->cur_obs_time) { - /* Doing this one second at a time is potentially inefficient, if we start - with a state file that is very old. Fortunately, it doesn't seem to - show up in profiles, so we can just ignore it for now. */ - advance_obs(b); - } - - b->obs[b->cur_obs_idx] += n; - b->total_in_period += n; -} - -/** Allocate, initialize, and return a new bw_array. */ -static bw_array_t * -bw_array_new(void) -{ - bw_array_t *b; - time_t start; - b = tor_malloc_zero(sizeof(bw_array_t)); - rephist_total_alloc += sizeof(bw_array_t); - start = time(NULL); - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - return b; -} - -#define bw_array_free(val) \ - FREE_AND_NULL(bw_array_t, bw_array_free_, (val)) - -/** Free storage held by bandwidth array b. */ -static void -bw_array_free_(bw_array_t *b) -{ - if (!b) { - return; - } - - rephist_total_alloc -= sizeof(bw_array_t); - tor_free(b); -} - -/** Recent history of bandwidth observations for read operations. */ -static bw_array_t *read_array = NULL; -/** Recent history of bandwidth observations for write operations. */ -STATIC bw_array_t *write_array = NULL; -/** Recent history of bandwidth observations for read operations for the - directory protocol. */ -static bw_array_t *dir_read_array = NULL; -/** Recent history of bandwidth observations for write operations for the - directory protocol. */ -static bw_array_t *dir_write_array = NULL; - -/** Set up [dir_]read_array and [dir_]write_array, freeing them if they - * already exist. */ -static void -bw_arrays_init(void) -{ - bw_array_free(read_array); - bw_array_free(write_array); - bw_array_free(dir_read_array); - bw_array_free(dir_write_array); - - read_array = bw_array_new(); - write_array = bw_array_new(); - dir_read_array = bw_array_new(); - dir_write_array = bw_array_new(); -} - -/** Remember that we read num_bytes bytes in second when. - * - * Add num_bytes to the current running total for when. - * - * when can go back to time, but it's safe to ignore calls - * earlier than the latest when you've heard of. - */ -void -rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) -{ -/* Maybe a circular array for recent seconds, and step to a new point - * every time a new second shows up. Or simpler is to just to have - * a normal array and push down each item every second; it's short. - */ -/* When a new second has rolled over, compute the sum of the bytes we've - * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it - * somewhere. See rep_hist_bandwidth_assess() below. - */ - add_obs(write_array, when, num_bytes); -} - -/** Remember that we wrote num_bytes bytes in second when. - * (like rep_hist_note_bytes_written() above) - */ -void -rep_hist_note_bytes_read(uint64_t num_bytes, time_t when) -{ -/* if we're smart, we can make this func and the one above share code */ - add_obs(read_array, when, num_bytes); -} - -/** Remember that we wrote num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) - */ -void -rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when) -{ - add_obs(dir_write_array, when, num_bytes); -} - -/** Remember that we read num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) - */ -void -rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when) -{ - add_obs(dir_read_array, when, num_bytes); -} - -/** Helper: Return the largest value in b->maxima. (This is equal to the - * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last - * NUM_SECS_BW_SUM_IS_VALID seconds.) - */ -STATIC uint64_t -find_largest_max(bw_array_t *b) -{ - int i; - uint64_t max; - max=0; - for (i=0; i<NUM_TOTALS; ++i) { - if (b->maxima[i]>max) - max = b->maxima[i]; - } - return max; -} - -/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly) - * seconds. Find one sum for reading and one for writing. They don't have - * to be at the same time. - * - * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. - */ -MOCK_IMPL(int, -rep_hist_bandwidth_assess,(void)) -{ - uint64_t w,r; - r = find_largest_max(read_array); - w = find_largest_max(write_array); - if (r>w) - return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); - else - return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE); -} - -/** Print the bandwidth history of b (either [dir-]read_array or - * [dir-]write_array) into the buffer pointed to by buf. The format is - * simply comma separated numbers, from oldest to newest. - * - * It returns the number of bytes written. - */ -static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) -{ - char *cp = buf; - int i, n; - const or_options_t *options = get_options(); - uint64_t cutoff; - - if (b->num_maxes_set <= b->next_max_idx) { - /* We haven't been through the circular array yet; time starts at i=0.*/ - i = 0; - } else { - /* We've been around the array at least once. The next i to be - overwritten is the oldest. */ - i = b->next_max_idx; - } - - if (options->RelayBandwidthRate) { - /* We don't want to report that we used more bandwidth than the max we're - * willing to relay; otherwise everybody will know how much traffic - * we used ourself. */ - cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL; - } else { - cutoff = UINT64_MAX; - } - - for (n=0; n<b->num_maxes_set; ++n,++i) { - uint64_t total; - if (i >= NUM_TOTALS) - i -= NUM_TOTALS; - tor_assert(i < NUM_TOTALS); - /* Round the bandwidth used down to the nearest 1k. */ - total = b->totals[i] & ~0x3ff; - if (total > cutoff) - total = cutoff; - - if (n==(b->num_maxes_set-1)) - tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total)); - else - tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total)); - cp += strlen(cp); - } - return cp-buf; -} - -/** Allocate and return lines for representing this server's bandwidth - * history in its descriptor. We publish these lines in our extra-info - * descriptor. - */ -char * -rep_hist_get_bandwidth_lines(void) -{ - char *buf, *cp; - char t[ISO_TIME_LEN+1]; - int r; - bw_array_t *b = NULL; - const char *desc = NULL; - size_t len; - - /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ -/* The n,n,n part above. Largest representation of a uint64_t is 20 chars - * long, plus the comma. */ -#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) - len = (67+MAX_HIST_VALUE_LEN)*4; - buf = tor_malloc_zero(len); - cp = buf; - for (r=0;r<4;++r) { - char tmp[MAX_HIST_VALUE_LEN]; - size_t slen; - switch (r) { - case 0: - b = write_array; - desc = "write-history"; - break; - case 1: - b = read_array; - desc = "read-history"; - break; - case 2: - b = dir_write_array; - desc = "dirreq-write-history"; - break; - case 3: - b = dir_read_array; - desc = "dirreq-read-history"; - break; - } - tor_assert(b); - slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); - /* If we don't have anything to write, skip to the next entry. */ - if (slen == 0) - continue; - format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", - desc, t, NUM_SECS_BW_SUM_INTERVAL); - cp += strlen(cp); - strlcat(cp, tmp, len-(cp-buf)); - cp += slen; - strlcat(cp, "\n", len-(cp-buf)); - ++cp; - } - return buf; -} - -/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum - * entries of an or_state_t. Done before writing out a new state file. */ -static void -rep_hist_update_bwhist_state_section(or_state_t *state, - const bw_array_t *b, - smartlist_t **s_values, - smartlist_t **s_maxima, - time_t *s_begins, - int *s_interval) -{ - int i,j; - uint64_t maxval; - - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (*s_maxima) { - SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); - smartlist_free(*s_maxima); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_new(); - *s_maxima = smartlist_new(); - return; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - - *s_values = smartlist_new(); - *s_maxima = smartlist_new(); - /* Set i to first position in circular array */ - i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; - for (j=0; j < b->num_maxes_set; ++j,++i) { - if (i >= NUM_TOTALS) - i = 0; - smartlist_add_asprintf(*s_values, "%"PRIu64, - (b->totals[i] & ~0x3ff)); - maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; - smartlist_add_asprintf(*s_maxima, "%"PRIu64, - (maxval & ~0x3ff)); - } - smartlist_add_asprintf(*s_values, "%"PRIu64, - (b->total_in_period & ~0x3ff)); - maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; - smartlist_add_asprintf(*s_maxima, "%"PRIu64, - (maxval & ~0x3ff)); -} - -/** Update state with the newest bandwidth history. Done before - * writing out a new state file. */ -void -rep_hist_update_state(or_state_t *state) -{ -#define UPDATE(arrname,st) \ - rep_hist_update_bwhist_state_section(state,\ - (arrname),\ - &state->BWHistory ## st ## Values, \ - &state->BWHistory ## st ## Maxima, \ - &state->BWHistory ## st ## Ends, \ - &state->BWHistory ## st ## Interval) - - UPDATE(write_array, Write); - UPDATE(read_array, Read); - UPDATE(dir_write_array, DirWrite); - UPDATE(dir_read_array, DirRead); - - if (server_mode(get_options())) { - or_state_mark_dirty(state, time(NULL)+(2*3600)); - } -#undef UPDATE -} - -/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval - * entries in an or_state_t. Done while reading the state file. */ -static int -rep_hist_load_bwhist_state_section(bw_array_t *b, - const smartlist_t *s_values, - const smartlist_t *s_maxima, - const time_t s_begins, - const int s_interval) -{ - time_t now = time(NULL); - int retval = 0; - time_t start; - - uint64_t v, mv; - int i,ok,ok_m = 0; - int have_maxima = s_maxima && s_values && - (smartlist_len(s_values) == smartlist_len(s_maxima)); - - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - return 0; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { - const char *maxstr = NULL; - v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); - if (have_maxima) { - maxstr = smartlist_get(s_maxima, cp_sl_idx); - mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); - mv *= NUM_SECS_ROLLING_MEASURE; - } else { - /* No maxima known; guess average rate to be conservative. */ - mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; - } - if (!ok) { - retval = -1; - log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); - } - if (maxstr && !ok_m) { - retval = -1; - log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", - maxstr); - } - - if (start < now) { - time_t cur_start = start; - time_t actual_interval_len = s_interval; - uint64_t cur_val = 0; - /* Calculate the average per second. This is the best we can do - * because our state file doesn't have per-second resolution. */ - if (start + s_interval > now) - actual_interval_len = now - start; - cur_val = v / actual_interval_len; - /* This is potentially inefficient, but since we don't do it very - * often it should be ok. */ - while (cur_start < start + actual_interval_len) { - add_obs(b, cur_start, cur_val); - ++cur_start; - } - b->max_total = mv; - /* This will result in some fairly choppy history if s_interval - * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ - start += actual_interval_len; - } - } SMARTLIST_FOREACH_END(cp); - } - - /* Clean up maxima and observed */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - - return retval; -} - -/** Set bandwidth history from the state file we just loaded. */ -int -rep_hist_load_state(or_state_t *state, char **err) -{ - int all_ok = 1; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); - tor_assert(dir_read_array && dir_write_array); - -#define LOAD(arrname,st) \ - if (rep_hist_load_bwhist_state_section( \ - (arrname), \ - state->BWHistory ## st ## Values, \ - state->BWHistory ## st ## Maxima, \ - state->BWHistory ## st ## Ends, \ - state->BWHistory ## st ## Interval)<0) \ - all_ok = 0 - - LOAD(write_array, Write); - LOAD(read_array, Read); - LOAD(dir_write_array, DirWrite); - LOAD(dir_read_array, DirRead); - -#undef LOAD - if (!all_ok) { - *err = tor_strdup("Parsing of bandwidth history values failed"); - /* and create fresh arrays */ - bw_arrays_init(); - return -1; - } - return 0; -} - /*** Exit port statistics ***/ /* Some constants */ @@ -2921,18 +2356,6 @@ rep_hist_free_all(void) hs_stats_free(hs_stats); digestmap_free(history_map, free_or_history); - bw_array_free(read_array); - read_array = NULL; - - bw_array_free(write_array); - write_array = NULL; - - bw_array_free(dir_read_array); - dir_read_array = NULL; - - bw_array_free(dir_write_array); - dir_write_array = NULL; - tor_free(exit_bytes_read); tor_free(exit_bytes_written); tor_free(exit_streams); diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index d08b8833c..cdedfdca6 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -14,18 +14,9 @@ void rep_hist_init(void); void rep_hist_dump_stats(time_t now, int severity); -void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when); void rep_hist_make_router_pessimal(const char *id, time_t when); -void rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when); - -MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); -char *rep_hist_get_bandwidth_lines(void); -void rep_hist_update_state(or_state_t *state); -int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, @@ -98,16 +89,8 @@ extern uint32_t rephist_total_num; #ifdef TOR_UNIT_TESTS extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; -extern struct bw_array_t *write_array; #endif -#ifdef REPHIST_PRIVATE -typedef struct bw_array_t bw_array_t; -STATIC uint64_t find_largest_max(bw_array_t *b); -STATIC void commit_max(bw_array_t *b); -STATIC void advance_obs(bw_array_t *b); -#endif /* defined(REPHIST_PRIVATE) */ - /** * Represents the type of a cell for padding accounting */ diff --git a/src/test/test_relay.c b/src/test/test_relay.c index 066aeaa7b..060ca1b75 100644 --- a/src/test/test_relay.c +++ b/src/test/test_relay.c @@ -3,12 +3,12 @@ #define CIRCUITBUILD_PRIVATE #define RELAY_PRIVATE -#define REPHIST_PRIVATE +#define BWHIST_PRIVATE #include "core/or/or.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/channeltls.h" -#include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "core/or/relay.h" #include "lib/container/order.h" /* For init/free stuff */ diff --git a/src/test/test_router.c b/src/test/test_router.c index cf0c2b3dd..9822729f5 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -24,7 +24,7 @@ #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" -#include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/encoding/confline.h" diff --git a/src/test/testing_common.c b/src/test/testing_common.c index b3337f24b..cd93055df 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -18,6 +18,7 @@ #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/stats/predict_ports.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "lib/err/backtrace.h" #include "test/test.h" @@ -333,6 +334,7 @@ main(int c, const char **v) return 1; } rep_hist_init(); + bwhist_init(); setup_directory(); initialize_mainloop_events(); options_init(options);

1 0

[tor/master] Refactor bwhist_get_bandwidth_lines()
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 11da5229d103fd13300b88a0efbc1da0b3970782 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:18:15 2020 -0400 Refactor bwhist_get_bandwidth_lines() We've done a lot to improve our internal APIs since we wrote this code, and it shows. We can just use a buf_t to build up the bandwidth lines, and save a bunch of stack fiddling. Additionally, we can use a function to format a single line, and thereby get rid of the cheezy pattern that does for (i=0;i<n;++i) { switch (i) { ... } ... } --- src/feature/stats/bwhist.c | 81 ++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 49 deletions(-) diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index 527f0128f..fef8a3b2d 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -324,6 +324,29 @@ bwhist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) return cp-buf; } +/** Encode a single bandwidth history line into buf. */ +static void +bwhist_get_one_bandwidth_line(buf_t *buf, const char *desc, + const bw_array_t *b) +{ + /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ + /* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) + + char tmp[MAX_HIST_VALUE_LEN]; + char end[ISO_TIME_LEN+1]; + + size_t slen = bwhist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + return; + + format_iso_time(end, b->next_period-NUM_SECS_BW_SUM_INTERVAL); + buf_add_printf(buf, "%s %s (%d s) %s\n", + desc, end, NUM_SECS_BW_SUM_INTERVAL, tmp); +} + /** Allocate and return lines for representing this server's bandwidth * history in its descriptor. We publish these lines in our extra-info * descriptor. @@ -331,56 +354,16 @@ bwhist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) char * bwhist_get_bandwidth_lines(void) { - char *buf, *cp; - char t[ISO_TIME_LEN+1]; - int r; - bw_array_t *b = NULL; - const char *desc = NULL; - size_t len; + buf_t *buf = buf_new(); - /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ -/* The n,n,n part above. Largest representation of a uint64_t is 20 chars - * long, plus the comma. */ -#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) - len = (67+MAX_HIST_VALUE_LEN)*4; - buf = tor_malloc_zero(len); - cp = buf; - for (r=0;r<4;++r) { - char tmp[MAX_HIST_VALUE_LEN]; - size_t slen; - switch (r) { - case 0: - b = write_array; - desc = "write-history"; - break; - case 1: - b = read_array; - desc = "read-history"; - break; - case 2: - b = dir_write_array; - desc = "dirreq-write-history"; - break; - case 3: - b = dir_read_array; - desc = "dirreq-read-history"; - break; - } - tor_assert(b); - slen = bwhist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); - /* If we don't have anything to write, skip to the next entry. */ - if (slen == 0) - continue; - format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", - desc, t, NUM_SECS_BW_SUM_INTERVAL); - cp += strlen(cp); - strlcat(cp, tmp, len-(cp-buf)); - cp += slen; - strlcat(cp, "\n", len-(cp-buf)); - ++cp; - } - return buf; + bwhist_get_one_bandwidth_line(buf, "write-history", write_array); + bwhist_get_one_bandwidth_line(buf, "read-history", read_array); + bwhist_get_one_bandwidth_line(buf, "dirreq-write-history", dir_write_array); + bwhist_get_one_bandwidth_line(buf, "dirreq-read-history", dir_read_array); + + char *result = buf_extract(buf, NULL); + buf_free(buf); + return result; } /** Write a single bw_array_t into the Values, Ends, Interval, and Maximum

1 0

[tor/master] Add IPv6 read and write history to bwhist, state, and extrainfo.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit c5db7667d6964344e87fb7ee71df3f2ebe8131e6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:29:24 2020 -0400 Add IPv6 read and write history to bwhist, state, and extrainfo. These values are stored, persisted, and published. They are not yet actually filled with anything. --- src/app/config/or_state_st.h | 8 ++++++++ src/app/config/statefile.c | 8 ++++++++ src/feature/stats/bwhist.c | 21 +++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index 8c4e9d5e6..31b7f8a98 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -65,6 +65,14 @@ struct or_state_t { int BWHistoryWriteInterval; struct smartlist_t *BWHistoryWriteValues; struct smartlist_t *BWHistoryWriteMaxima; + time_t BWHistoryIPv6ReadEnds; + int BWHistoryIPv6ReadInterval; + struct smartlist_t *BWHistoryIPv6ReadValues; + struct smartlist_t *BWHistoryIPv6ReadMaxima; + time_t BWHistoryIPv6WriteEnds; + int BWHistoryIPv6WriteInterval; + struct smartlist_t *BWHistoryIPv6WriteValues; + struct smartlist_t *BWHistoryIPv6WriteMaxima; time_t BWHistoryDirReadEnds; int BWHistoryDirReadInterval; struct smartlist_t *BWHistoryDirReadValues; diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 4198c8a38..b25167d2e 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -112,6 +112,14 @@ static const config_var_t state_vars_[] = { V(BWHistoryWriteInterval, POSINT, "900"), V(BWHistoryWriteValues, CSV, ""), V(BWHistoryWriteMaxima, CSV, ""), + V(BWHistoryIPv6ReadEnds, ISOTIME, NULL), + V(BWHistoryIPv6ReadInterval, POSINT, "900"), + V(BWHistoryIPv6ReadValues, CSV, ""), + V(BWHistoryIPv6ReadMaxima, CSV, ""), + V(BWHistoryIPv6WriteEnds, ISOTIME, NULL), + V(BWHistoryIPv6WriteInterval, POSINT, "900"), + V(BWHistoryIPv6WriteValues, CSV, ""), + V(BWHistoryIPv6WriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), V(BWHistoryDirReadInterval, POSINT, "900"), V(BWHistoryDirReadValues, CSV, ""), diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index fef8a3b2d..44717f33b 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -162,10 +162,14 @@ bw_array_free_(bw_array_t *b) tor_free(b); } -/** Recent history of bandwidth observations for read operations. */ +/** Recent history of bandwidth observations for (all) read operations. */ static bw_array_t *read_array = NULL; -/** Recent history of bandwidth observations for write operations. */ +/** Recent history of bandwidth observations for IPv6 read operations. */ +static bw_array_t *read_array_ipv6 = NULL; +/** Recent history of bandwidth observations for (all) write operations. */ STATIC bw_array_t *write_array = NULL; +/** Recent history of bandwidth observations for IPv6 write operations. */ +static bw_array_t *write_array_ipv6 = NULL; /** Recent history of bandwidth observations for read operations for the directory protocol. */ static bw_array_t *dir_read_array = NULL; @@ -179,12 +183,16 @@ void bwhist_init(void) { bw_array_free(read_array); + bw_array_free(read_array_ipv6); bw_array_free(write_array); + bw_array_free(write_array_ipv6); bw_array_free(dir_read_array); bw_array_free(dir_write_array); read_array = bw_array_new(); + read_array_ipv6 = bw_array_new(); write_array = bw_array_new(); + write_array_ipv6 = bw_array_new(); dir_read_array = bw_array_new(); dir_write_array = bw_array_new(); } @@ -358,6 +366,8 @@ bwhist_get_bandwidth_lines(void) bwhist_get_one_bandwidth_line(buf, "write-history", write_array); bwhist_get_one_bandwidth_line(buf, "read-history", read_array); + bwhist_get_one_bandwidth_line(buf, "ipv6-write-history", write_array_ipv6); + bwhist_get_one_bandwidth_line(buf, "ipv6-read-history", read_array_ipv6); bwhist_get_one_bandwidth_line(buf, "dirreq-write-history", dir_write_array); bwhist_get_one_bandwidth_line(buf, "dirreq-read-history", dir_read_array); @@ -441,6 +451,8 @@ bwhist_update_state(or_state_t *state) UPDATE(write_array, Write); UPDATE(read_array, Read); + UPDATE(write_array_ipv6, IPv6Write); + UPDATE(read_array_ipv6, IPv6Read); UPDATE(dir_write_array, DirWrite); UPDATE(dir_read_array, DirRead); @@ -535,6 +547,7 @@ bwhist_load_state(or_state_t *state, char **err) /* Assert they already have been malloced */ tor_assert(read_array && write_array); + tor_assert(read_array_ipv6 && write_array_ipv6); tor_assert(dir_read_array && dir_write_array); #define LOAD(arrname,st) \ @@ -548,6 +561,8 @@ bwhist_load_state(or_state_t *state, char **err) LOAD(write_array, Write); LOAD(read_array, Read); + LOAD(write_array_ipv6, IPv6Write); + LOAD(read_array_ipv6, IPv6Read); LOAD(dir_write_array, DirWrite); LOAD(dir_read_array, DirRead); @@ -565,7 +580,9 @@ void bwhist_free_all(void) { bw_array_free(read_array); + bw_array_free(read_array_ipv6); bw_array_free(write_array); + bw_array_free(write_array_ipv6); bw_array_free(dir_read_array); bw_array_free(dir_write_array); }

1 0

[tor/master] Rename private bandwidth-history identifiers to start with "bwhist".
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit c5eb601e60760eb56d41ec08970cb151511bf2ea Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 07:54:04 2020 -0400 Rename private bandwidth-history identifiers to start with "bwhist". This commit is a simple search-and-replace in bwhist.c --- src/feature/stats/bwhist.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index 2bc533e66..527f0128f 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -280,7 +280,7 @@ bwhist_bandwidth_assess,(void)) * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) +bwhist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@ -367,7 +367,7 @@ bwhist_get_bandwidth_lines(void) break; } tor_assert(b); - slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + slen = bwhist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); /* If we don't have anything to write, skip to the next entry. */ if (slen == 0) continue; @@ -386,7 +386,7 @@ bwhist_get_bandwidth_lines(void) /** Write a single bw_array_t into the Values, Ends, Interval, and Maximum * entries of an or_state_t. Done before writing out a new state file. */ static void -rep_hist_update_bwhist_state_section(or_state_t *state, +bwhist_update_bwhist_state_section(or_state_t *state, const bw_array_t *b, smartlist_t **s_values, smartlist_t **s_maxima, @@ -449,7 +449,7 @@ void bwhist_update_state(or_state_t *state) { #define UPDATE(arrname,st) \ - rep_hist_update_bwhist_state_section(state,\ + bwhist_update_bwhist_state_section(state,\ (arrname),\ &state->BWHistory ## st ## Values, \ &state->BWHistory ## st ## Maxima, \ @@ -470,7 +470,7 @@ bwhist_update_state(or_state_t *state) /** Load a single bw_array_t from its Values, Ends, Maxima, and Interval * entries in an or_state_t. Done while reading the state file. */ static int -rep_hist_load_bwhist_state_section(bw_array_t *b, +bwhist_load_bwhist_state_section(bw_array_t *b, const smartlist_t *s_values, const smartlist_t *s_maxima, const time_t s_begins, @@ -555,7 +555,7 @@ bwhist_load_state(or_state_t *state, char **err) tor_assert(dir_read_array && dir_write_array); #define LOAD(arrname,st) \ - if (rep_hist_load_bwhist_state_section( \ + if (bwhist_load_bwhist_state_section( \ (arrname), \ state->BWHistory ## st ## Values, \ state->BWHistory ## st ## Maxima, \

1 0

[tor/master] Rename public bandwidth-history identifiers to start with "bwhist".
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 2fc8257ac4c3b404f5e41eda8ea98eaaad6c3a5c Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 07:54:04 2020 -0400 Rename public bandwidth-history identifiers to start with "bwhist". This is an automated commit, generated by this command: ./scripts/maint/rename_c_identifier.py \ rep_hist_note_bytes_read bwhist_note_bytes_read \ rep_hist_note_bytes_written bwhist_note_bytes_written \ rep_hist_note_dir_bytes_read bwhist_note_dir_bytes_read \ rep_hist_note_dir_bytes_written bwhist_note_dir_bytes_written \ rep_hist_get_bandwidth_lines bwhist_get_bandwidth_lines \ rep_hist_update_state bwhist_update_state \ rep_hist_load_state bwhist_load_state \ rep_hist_bandwidth_assess bwhist_bandwidth_assess --- src/app/config/statefile.c | 4 ++-- src/core/mainloop/connection.c | 8 ++++---- src/core/or/circuitlist.c | 2 +- src/feature/relay/router.c | 6 +++--- src/feature/stats/bwhist.c | 24 ++++++++++++------------ src/feature/stats/bwhist.h | 16 ++++++++-------- src/test/test_router.c | 4 ++-- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 63205007c..4198c8a38 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -324,7 +324,7 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (rep_hist_load_state(global_state, &err)<0) { + if (bwhist_load_state(global_state, &err)<0) { log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); ret = -1; @@ -523,7 +523,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ (void) subsystems_flush_state(get_state_mgr(), global_state); entry_guards_update_state(global_state); - rep_hist_update_state(global_state); + bwhist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); if (accounting_is_enabled(get_options())) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 7f1b99311..960735fc2 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -3345,9 +3345,9 @@ record_num_bytes_transferred_impl(connection_t *conn, /* Count bytes of answering direct and tunneled directory requests */ if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) { if (num_read > 0) - rep_hist_note_dir_bytes_read(num_read, now); + bwhist_note_dir_bytes_read(num_read, now); if (num_written > 0) - rep_hist_note_dir_bytes_written(num_written, now); + bwhist_note_dir_bytes_written(num_written, now); } /* Linked connections and internal IPs aren't counted for statistics or @@ -3367,10 +3367,10 @@ record_num_bytes_transferred_impl(connection_t *conn, num_written, now); if (num_read > 0) { - rep_hist_note_bytes_read(num_read, now); + bwhist_note_bytes_read(num_read, now); } if (num_written > 0) { - rep_hist_note_bytes_written(num_written, now); + bwhist_note_bytes_written(num_written, now); } if (conn->type == CONN_TYPE_EXIT) rep_hist_note_exit_bytes(conn->port, num_written, num_read); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index a6949d943..f0f182e5d 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2174,7 +2174,7 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c, /* Report the missing bytes as written, to avoid asymmetry. * We must use time() for consistency with rephist, even though on * some very old rare platforms, approx_time() may be faster. */ - rep_hist_note_bytes_written(written_sync, time(NULL)); + bwhist_note_bytes_written(written_sync, time(NULL)); } /** Mark circ to be closed next time we call diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index a55b30067..b63950ea1 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2099,7 +2099,7 @@ router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out)) ri->bandwidthburst = relay_get_effective_bwburst(options); /* Report bandwidth, unless we're hibernating or shutting down */ - ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); + ri->bandwidthcapacity = hibernating ? 0 : bwhist_bandwidth_assess(); if (dns_seems_to_be_broken() || has_dns_init_failed()) { /* DNS is screwed up; don't claim to be an exit. */ @@ -2545,7 +2545,7 @@ check_descriptor_bandwidth_changed(time_t now) /* Consider ourselves to have zero bandwidth if we're hibernating or * shutting down. */ - cur = hibernating ? 0 : rep_hist_bandwidth_assess(); + cur = hibernating ? 0 : bwhist_bandwidth_assess(); if ((prev != cur && (!prev || !cur)) || cur > (prev * BANDWIDTH_CHANGE_FACTOR) || @@ -3209,7 +3209,7 @@ extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); /* Bandwidth usage stats don't have their own option */ { - contents = rep_hist_get_bandwidth_lines(); + contents = bwhist_get_bandwidth_lines(); smartlist_add(chunks, contents); } /* geoip hashes aren't useful unless we are publishing other stats */ diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index a3b971a25..2bc533e66 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -197,7 +197,7 @@ bwhist_init(void) * earlier than the latest when you've heard of. */ void -rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) +bwhist_note_bytes_written(uint64_t num_bytes, time_t when) { /* Maybe a circular array for recent seconds, and step to a new point * every time a new second shows up. Or simpler is to just to have @@ -205,35 +205,35 @@ rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) */ /* When a new second has rolled over, compute the sum of the bytes we've * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it - * somewhere. See rep_hist_bandwidth_assess() below. + * somewhere. See bwhist_bandwidth_assess() below. */ add_obs(write_array, when, num_bytes); } /** Remember that we wrote num_bytes bytes in second when. - * (like rep_hist_note_bytes_written() above) + * (like bwhist_note_bytes_written() above) */ void -rep_hist_note_bytes_read(uint64_t num_bytes, time_t when) +bwhist_note_bytes_read(uint64_t num_bytes, time_t when) { /* if we're smart, we can make this func and the one above share code */ add_obs(read_array, when, num_bytes); } /** Remember that we wrote num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) + * when. (like bwhist_note_bytes_written() above) */ void -rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when) +bwhist_note_dir_bytes_written(uint64_t num_bytes, time_t when) { add_obs(dir_write_array, when, num_bytes); } /** Remember that we read num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) + * when. (like bwhist_note_bytes_written() above) */ void -rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when) +bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when) { add_obs(dir_read_array, when, num_bytes); } @@ -262,7 +262,7 @@ find_largest_max(bw_array_t *b) * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. */ MOCK_IMPL(int, -rep_hist_bandwidth_assess,(void)) +bwhist_bandwidth_assess,(void)) { uint64_t w,r; r = find_largest_max(read_array); @@ -329,7 +329,7 @@ rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) * descriptor. */ char * -rep_hist_get_bandwidth_lines(void) +bwhist_get_bandwidth_lines(void) { char *buf, *cp; char t[ISO_TIME_LEN+1]; @@ -446,7 +446,7 @@ rep_hist_update_bwhist_state_section(or_state_t *state, /** Update state with the newest bandwidth history. Done before * writing out a new state file. */ void -rep_hist_update_state(or_state_t *state) +bwhist_update_state(or_state_t *state) { #define UPDATE(arrname,st) \ rep_hist_update_bwhist_state_section(state,\ @@ -546,7 +546,7 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, /** Set bandwidth history from the state file we just loaded. */ int -rep_hist_load_state(or_state_t *state, char **err) +bwhist_load_state(or_state_t *state, char **err) { int all_ok = 1; diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h index afe0d9f4e..17d2a8394 100644 --- a/src/feature/stats/bwhist.h +++ b/src/feature/stats/bwhist.h @@ -15,16 +15,16 @@ void bwhist_init(void); void bwhist_free_all(void); -void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when); -void rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when); +void bwhist_note_bytes_read(uint64_t num_bytes, time_t when); +void bwhist_note_bytes_written(uint64_t num_bytes, time_t when); +void bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when); +void bwhist_note_dir_bytes_written(uint64_t num_bytes, time_t when); -MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); -char *rep_hist_get_bandwidth_lines(void); +MOCK_DECL(int, bwhist_bandwidth_assess, (void)); +char *bwhist_get_bandwidth_lines(void); struct or_state_t; -void rep_hist_update_state(struct or_state_t *state); -int rep_hist_load_state(struct or_state_t *state, char **err); +void bwhist_update_state(struct or_state_t *state); +int bwhist_load_state(struct or_state_t *state, char **err); #ifdef BWHIST_PRIVATE typedef struct bw_array_t bw_array_t; diff --git a/src/test/test_router.c b/src/test/test_router.c index 9822729f5..010289397 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -226,13 +226,13 @@ test_router_check_descriptor_bandwidth_changed(void *arg) /* When uptime is less than 24h and bandwidthcapacity does not change * Uptime: 10800, last_changed: x, Previous bw: 10000, Current bw: 20001 */ - MOCK(rep_hist_bandwidth_assess, mock_rep_hist_bandwidth_assess); + MOCK(bwhist_bandwidth_assess, mock_rep_hist_bandwidth_assess); setup_full_capture_of_logs(LOG_INFO); check_descriptor_bandwidth_changed(time(NULL) + 6*60*60 + 1); expect_log_msg_containing( "Measured bandwidth has changed; rebuilding descriptor."); UNMOCK(get_uptime); - UNMOCK(rep_hist_bandwidth_assess); + UNMOCK(bwhist_bandwidth_assess); teardown_capture_of_logs(); /* When uptime is more than 24h */

1 0

[tor/master] Record IPv6 bandwidth history as appropriate.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 27c5cadf7e86551c7d62d27b107098266bb4c11b Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:44:46 2020 -0400 Record IPv6 bandwidth history as appropriate. --- src/core/mainloop/connection.c | 5 +++-- src/core/or/circuitlist.c | 8 +++++++- src/feature/stats/bwhist.c | 8 ++++++-- src/feature/stats/bwhist.h | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 960735fc2..2f3c70365 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -3366,11 +3366,12 @@ record_num_bytes_transferred_impl(connection_t *conn, rep_hist_note_or_conn_bytes(conn->global_identifier, num_read, num_written, now); + const bool is_ipv6 = (conn->socket_family == AF_INET6); if (num_read > 0) { - bwhist_note_bytes_read(num_read, now); + bwhist_note_bytes_read(num_read, now, is_ipv6); } if (num_written > 0) { - bwhist_note_bytes_written(num_written, now); + bwhist_note_bytes_written(num_written, now, is_ipv6); } if (conn->type == CONN_TYPE_EXIT) rep_hist_note_exit_bytes(conn->port, num_written, num_read); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index f0f182e5d..28b3fa8ff 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2167,6 +2167,12 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c, else cell_size = CELL_MAX_NETWORK_SIZE; + /* If we know the channel, find out if it's IPv6. */ + tor_addr_t remote_addr; + bool is_ipv6 = chan && + channel_get_addr_if_possible(chan, &remote_addr) && + tor_addr_family(&remote_addr) == AF_INET6; + /* The missing written bytes are the cell counts times their cell * size plus TLS per cell overhead */ written_sync = cells*(cell_size+TLS_PER_CELL_OVERHEAD); @@ -2174,7 +2180,7 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c, /* Report the missing bytes as written, to avoid asymmetry. * We must use time() for consistency with rephist, even though on * some very old rare platforms, approx_time() may be faster. */ - bwhist_note_bytes_written(written_sync, time(NULL)); + bwhist_note_bytes_written(written_sync, time(NULL), is_ipv6); } /** Mark circ to be closed next time we call diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index 44717f33b..e74a2881f 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -205,7 +205,7 @@ bwhist_init(void) * earlier than the latest when you've heard of. */ void -bwhist_note_bytes_written(uint64_t num_bytes, time_t when) +bwhist_note_bytes_written(uint64_t num_bytes, time_t when, bool ipv6) { /* Maybe a circular array for recent seconds, and step to a new point * every time a new second shows up. Or simpler is to just to have @@ -216,16 +216,20 @@ bwhist_note_bytes_written(uint64_t num_bytes, time_t when) * somewhere. See bwhist_bandwidth_assess() below. */ add_obs(write_array, when, num_bytes); + if (ipv6) + add_obs(write_array_ipv6, when, num_bytes); } /** Remember that we wrote num_bytes bytes in second when. * (like bwhist_note_bytes_written() above) */ void -bwhist_note_bytes_read(uint64_t num_bytes, time_t when) +bwhist_note_bytes_read(uint64_t num_bytes, time_t when, bool ipv6) { /* if we're smart, we can make this func and the one above share code */ add_obs(read_array, when, num_bytes); + if (ipv6) + add_obs(read_array_ipv6, when, num_bytes); } /** Remember that we wrote num_bytes directory bytes in second diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h index 17d2a8394..d556f5a02 100644 --- a/src/feature/stats/bwhist.h +++ b/src/feature/stats/bwhist.h @@ -15,8 +15,8 @@ void bwhist_init(void); void bwhist_free_all(void); -void bwhist_note_bytes_read(uint64_t num_bytes, time_t when); -void bwhist_note_bytes_written(uint64_t num_bytes, time_t when); +void bwhist_note_bytes_read(uint64_t num_bytes, time_t when, bool ipv6); +void bwhist_note_bytes_written(uint64_t num_bytes, time_t when, bool ipv6); void bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when); void bwhist_note_dir_bytes_written(uint64_t num_bytes, time_t when);

1 0