July 2020 - tor-commits - lists.torproject.org

[tor/master] doc: Add a WARNING section to Tracing.md
by nickm＠torproject.org 10 Jul '20

10 Jul '20

commit 942ecfa835bc50ed11912df034abf5f184d46033 Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Mar 11 12:26:10 2020 -0400 doc: Add a WARNING section to Tracing.md Explain what is safe or not with tracing data. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- doc/HACKING/Tracing.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/HACKING/Tracing.md b/doc/HACKING/Tracing.md index 8cf68321a..… [View More]

1 0

[tor/master] changes: Add changes file for #32910
by nickm＠torproject.org 10 Jul '20

10 Jul '20

commit dfaa0a82acdfbd65dde0a30fa8fd598304a86816 Author: David Goulet <dgoulet(a)torproject.org> Date: Fri Jul 10 12:12:26 2020 -0400 changes: Add changes file for #32910 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket32910 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changes/ticket32910 b/changes/ticket32910 new file mode 100644 index 000000000..e3d64d433 --- /dev/null +++ b/changes/ticket32910 @@ -0,0 +1,5 @@ + o Major feature … [View More]

1 0

[tor/master] Merge remote-tracking branch 'tor-gitlab/mr/29'
by nickm＠torproject.org 10 Jul '20

10 Jul '20

commit 6fb1637255122dc3660486cf731c0f4640fd98db Merge: 1a4e475d5 dfaa0a82a Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 12:53:46 2020 -0400 Merge remote-tracking branch 'tor-gitlab/mr/29' changes/ticket32910 | 5 + configure.ac | 84 +++++++- doc/HACKING/Tracing.md | 150 +++++++++----- scripts/maint/practracker/exceptions.txt | 2 + src/app/include.am | 4 +- src/… [View More]

1 0

[tor/master] Split bandwidth history functions into a separate C file.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 8390df917b7e63696c70037765737037cd9162a0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 07:50:17 2020 -0400 Split bandwidth history functions into a separate C file. These are logically independent from the rest of rephist, and make more sense in isolation. The next patch will rename them too. --- src/app/config/statefile.c | 2 +- src/app/main/main.c | 2 + src/app/main/shutdown.c | 2 + src/core/mainloop/… [View More]connection.c | 1 + src/core/or/circuitlist.c | 1 + src/feature/relay/router.c | 1 + src/feature/stats/bwhist.c | 588 +++++++++++++++++++++++++++++++++++++++++ src/feature/stats/bwhist.h | 40 +++ src/feature/stats/include.am | 2 + src/feature/stats/rephist.c | 577 ---------------------------------------- src/feature/stats/rephist.h | 17 -- src/test/test_relay.c | 4 +- src/test/test_router.c | 2 +- src/test/testing_common.c | 2 + 14 files changed, 643 insertions(+), 598 deletions(-) diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index dcc55f189..63205007c 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -40,7 +40,7 @@ #include "feature/control/control_events.h" #include "feature/client/entrynodes.h" #include "feature/hibernate/hibernate.h" -#include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "feature/relay/router.h" #include "feature/relay/routermode.h" #include "lib/sandbox/sandbox.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index 89ba78742..7b6a665b7 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -53,6 +53,7 @@ #include "feature/rend/rendcache.h" #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" #include "lib/buf/buffers.h" @@ -549,6 +550,7 @@ tor_init(int argc, char *argv[]) /* Initialize the history structures. */ rep_hist_init(); + bwhist_init(); /* Initialize the service cache. */ rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index aac15246b..4a556333d 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -47,6 +47,7 @@ #include "feature/relay/relay_config.h" #include "feature/rend/rendcache.h" #include "feature/rend/rendclient.h" +#include "feature/stats/bwhist.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/evloop/compat_libevent.h" @@ -121,6 +122,7 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); + bwhist_free_all(); circuit_free_all(); circpad_machines_free(); entry_guards_free_all(); diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index af823335a..7f1b99311 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -106,6 +106,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/geoip/geoip.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index f4d6cd3c1..a6949d943 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -90,6 +90,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/stats/predict_ports.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 57da735e8..a55b30067 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -44,6 +44,7 @@ #include "feature/relay/selftest.h" #include "lib/geoip/geoip.h" #include "feature/stats/geoip_stats.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_format.h" diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c new file mode 100644 index 000000000..a3b971a25 --- /dev/null +++ b/src/feature/stats/bwhist.c @@ -0,0 +1,588 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file bwhist.c + * @brief Tracking for relay bandwidth history + * + * This module handles bandwidth usage history, used by relays to + * self-report how much bandwidth they've used for different + * purposes over last day or so, in order to generate the + * {dirreq-,}{read,write}-history lines in that they publish. + **/ + +#define BWHIST_PRIVATE +#include "orconfig.h" +#include "core/or/or.h" +#include "feature/stats/bwhist.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "feature/relay/routermode.h" + +#include "app/config/or_state_st.h" +#include "app/config/or_options_st.h" + +/** For how many seconds do we keep track of individual per-second bandwidth + * totals? */ +#define NUM_SECS_ROLLING_MEASURE 10 +/** How large are the intervals for which we track and report bandwidth use? */ +#define NUM_SECS_BW_SUM_INTERVAL (24*60*60) +/** How far in the past do we remember and publish bandwidth use? */ +#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60) +/** How many bandwidth usage intervals do we remember? (derived) */ +#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL) + +/** Structure to track bandwidth use, and remember the maxima for a given + * time period. + */ +struct bw_array_t { + /** Observation array: Total number of bytes transferred in each of the last + * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */ + uint64_t obs[NUM_SECS_ROLLING_MEASURE]; + int cur_obs_idx; /**< Current position in obs. */ + time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */ + uint64_t total_obs; /**< Total for all members of obs except + * obs[cur_obs_idx] */ + uint64_t max_total; /**< Largest value that total_obs has taken on in the + * current period. */ + uint64_t total_in_period; /**< Total bytes transferred in the current + * period. */ + + /** When does the next period begin? */ + time_t next_period; + /** Where in 'maxima' should the maximum bandwidth usage for the current + * period be stored? */ + int next_max_idx; + /** How many values in maxima/totals have been set ever? */ + int num_maxes_set; + /** Circular array of the maximum + * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last + * NUM_TOTALS periods */ + uint64_t maxima[NUM_TOTALS]; + /** Circular array of the total bandwidth usage for the last NUM_TOTALS + * periods */ + uint64_t totals[NUM_TOTALS]; +}; + +/** Shift the current period of b forward by one. */ +STATIC void +commit_max(bw_array_t *b) +{ + /* Store total from current period. */ + b->totals[b->next_max_idx] = b->total_in_period; + /* Store maximum from current period. */ + b->maxima[b->next_max_idx++] = b->max_total; + /* Advance next_period and next_max_idx */ + b->next_period += NUM_SECS_BW_SUM_INTERVAL; + if (b->next_max_idx == NUM_TOTALS) + b->next_max_idx = 0; + if (b->num_maxes_set < NUM_TOTALS) + ++b->num_maxes_set; + /* Reset max_total. */ + b->max_total = 0; + /* Reset total_in_period. */ + b->total_in_period = 0; +} + +/** Shift the current observation time of b forward by one second. */ +STATIC void +advance_obs(bw_array_t *b) +{ + int nextidx; + uint64_t total; + + /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE + * seconds; adjust max_total as needed.*/ + total = b->total_obs + b->obs[b->cur_obs_idx]; + if (total > b->max_total) + b->max_total = total; + + nextidx = b->cur_obs_idx+1; + if (nextidx == NUM_SECS_ROLLING_MEASURE) + nextidx = 0; + + b->total_obs = total - b->obs[nextidx]; + b->obs[nextidx]=0; + b->cur_obs_idx = nextidx; + + if (++b->cur_obs_time >= b->next_period) + commit_max(b); +} + +/** Add n bytes to the number of bytes in b for second + * when. */ +static inline void +add_obs(bw_array_t *b, time_t when, uint64_t n) +{ + if (when < b->cur_obs_time) + return; /* Don't record data in the past. */ + + /* If we're currently adding observations for an earlier second than + * 'when', advance b->cur_obs_time and b->cur_obs_idx by an + * appropriate number of seconds, and do all the other housekeeping. */ + while (when > b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ + advance_obs(b); + } + + b->obs[b->cur_obs_idx] += n; + b->total_in_period += n; +} + +/** Allocate, initialize, and return a new bw_array. */ +static bw_array_t * +bw_array_new(void) +{ + bw_array_t *b; + time_t start; + b = tor_malloc_zero(sizeof(bw_array_t)); + start = time(NULL); + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + return b; +} + +#define bw_array_free(val) \ + FREE_AND_NULL(bw_array_t, bw_array_free_, (val)) + +/** Free storage held by bandwidth array b. */ +static void +bw_array_free_(bw_array_t *b) +{ + if (!b) { + return; + } + + tor_free(b); +} + +/** Recent history of bandwidth observations for read operations. */ +static bw_array_t *read_array = NULL; +/** Recent history of bandwidth observations for write operations. */ +STATIC bw_array_t *write_array = NULL; +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up structures for bandwidth history, clearing them if they already + * exist. */ +void +bwhist_init(void) +{ + bw_array_free(read_array); + bw_array_free(write_array); + bw_array_free(dir_read_array); + bw_array_free(dir_write_array); + + read_array = bw_array_new(); + write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); +} + +/** Remember that we read num_bytes bytes in second when. + * + * Add num_bytes to the current running total for when. + * + * when can go back to time, but it's safe to ignore calls + * earlier than the latest when you've heard of. + */ +void +rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) +{ +/* Maybe a circular array for recent seconds, and step to a new point + * every time a new second shows up. Or simpler is to just to have + * a normal array and push down each item every second; it's short. + */ +/* When a new second has rolled over, compute the sum of the bytes we've + * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it + * somewhere. See rep_hist_bandwidth_assess() below. + */ + add_obs(write_array, when, num_bytes); +} + +/** Remember that we wrote num_bytes bytes in second when. + * (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_bytes_read(uint64_t num_bytes, time_t when) +{ +/* if we're smart, we can make this func and the one above share code */ + add_obs(read_array, when, num_bytes); +} + +/** Remember that we wrote num_bytes directory bytes in second + * when. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when) +{ + add_obs(dir_write_array, when, num_bytes); +} + +/** Remember that we read num_bytes directory bytes in second + * when. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when) +{ + add_obs(dir_read_array, when, num_bytes); +} + +/** Helper: Return the largest value in b->maxima. (This is equal to the + * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last + * NUM_SECS_BW_SUM_IS_VALID seconds.) + */ +STATIC uint64_t +find_largest_max(bw_array_t *b) +{ + int i; + uint64_t max; + max=0; + for (i=0; i<NUM_TOTALS; ++i) { + if (b->maxima[i]>max) + max = b->maxima[i]; + } + return max; +} + +/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly) + * seconds. Find one sum for reading and one for writing. They don't have + * to be at the same time. + * + * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. + */ +MOCK_IMPL(int, +rep_hist_bandwidth_assess,(void)) +{ + uint64_t w,r; + r = find_largest_max(read_array); + w = find_largest_max(write_array); + if (r>w) + return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); + else + return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE); +} + +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. + * + * It returns the number of bytes written. + */ +static size_t +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) +{ + char *cp = buf; + int i, n; + const or_options_t *options = get_options(); + uint64_t cutoff; + + if (b->num_maxes_set <= b->next_max_idx) { + /* We haven't been through the circular array yet; time starts at i=0.*/ + i = 0; + } else { + /* We've been around the array at least once. The next i to be + overwritten is the oldest. */ + i = b->next_max_idx; + } + + if (options->RelayBandwidthRate) { + /* We don't want to report that we used more bandwidth than the max we're + * willing to relay; otherwise everybody will know how much traffic + * we used ourself. */ + cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL; + } else { + cutoff = UINT64_MAX; + } + + for (n=0; n<b->num_maxes_set; ++n,++i) { + uint64_t total; + if (i >= NUM_TOTALS) + i -= NUM_TOTALS; + tor_assert(i < NUM_TOTALS); + /* Round the bandwidth used down to the nearest 1k. */ + total = b->totals[i] & ~0x3ff; + if (total > cutoff) + total = cutoff; + + if (n==(b->num_maxes_set-1)) + tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total)); + else + tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total)); + cp += strlen(cp); + } + return cp-buf; +} + +/** Allocate and return lines for representing this server's bandwidth + * history in its descriptor. We publish these lines in our extra-info + * descriptor. + */ +char * +rep_hist_get_bandwidth_lines(void) +{ + char *buf, *cp; + char t[ISO_TIME_LEN+1]; + int r; + bw_array_t *b = NULL; + const char *desc = NULL; + size_t len; + + /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ +/* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) + len = (67+MAX_HIST_VALUE_LEN)*4; + buf = tor_malloc_zero(len); + cp = buf; + for (r=0;r<4;++r) { + char tmp[MAX_HIST_VALUE_LEN]; + size_t slen; + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } + tor_assert(b); + slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + continue; + format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); + tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", + desc, t, NUM_SECS_BW_SUM_INTERVAL); + cp += strlen(cp); + strlcat(cp, tmp, len-(cp-buf)); + cp += slen; + strlcat(cp, "\n", len-(cp-buf)); + ++cp; + } + return buf; +} + +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. Done before writing out a new state file. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + int i,j; + uint64_t maxval; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_new(); + *s_maxima = smartlist_new(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_new(); + *s_maxima = smartlist_new(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + if (i >= NUM_TOTALS) + i = 0; + smartlist_add_asprintf(*s_values, "%"PRIu64, + (b->totals[i] & ~0x3ff)); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + smartlist_add_asprintf(*s_maxima, "%"PRIu64, + (maxval & ~0x3ff)); + } + smartlist_add_asprintf(*s_values, "%"PRIu64, + (b->total_in_period & ~0x3ff)); + maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; + smartlist_add_asprintf(*s_maxima, "%"PRIu64, + (maxval & ~0x3ff)); +} + +/** Update state with the newest bandwidth history. Done before + * writing out a new state file. */ +void +rep_hist_update_state(or_state_t *state) +{ +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) + + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); + + if (server_mode(get_options())) { + or_state_mark_dirty(state, time(NULL)+(2*3600)); + } +#undef UPDATE +} + +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. Done while reading the state file. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) +{ + time_t now = time(NULL); + int retval = 0; + time_t start; + + uint64_t v, mv; + int i,ok,ok_m = 0; + int have_maxima = s_maxima && s_values && + (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; + v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; + } + if (!ok) { + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); + } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + + if (start < now) { + time_t cur_start = start; + time_t actual_interval_len = s_interval; + uint64_t cur_val = 0; + /* Calculate the average per second. This is the best we can do + * because our state file doesn't have per-second resolution. */ + if (start + s_interval > now) + actual_interval_len = now - start; + cur_val = v / actual_interval_len; + /* This is potentially inefficient, but since we don't do it very + * often it should be ok. */ + while (cur_start < start + actual_interval_len) { + add_obs(b, cur_start, cur_val); + ++cur_start; + } + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += actual_interval_len; + } + } SMARTLIST_FOREACH_END(cp); + } + + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; + } + b->total_obs = 0; + + return retval; +} + +/** Set bandwidth history from the state file we just loaded. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD + if (!all_ok) { + *err = tor_strdup("Parsing of bandwidth history values failed"); + /* and create fresh arrays */ + bwhist_init(); + return -1; + } + return 0; +} + +void +bwhist_free_all(void) +{ + bw_array_free(read_array); + bw_array_free(write_array); + bw_array_free(dir_read_array); + bw_array_free(dir_write_array); +} diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h new file mode 100644 index 000000000..afe0d9f4e --- /dev/null +++ b/src/feature/stats/bwhist.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file bwhist.h + * @brief Header for feature/stats/bwhist.c + **/ + +#ifndef TOR_FEATURE_STATS_BWHIST_H +#define TOR_FEATURE_STATS_BWHIST_H + +void bwhist_init(void); +void bwhist_free_all(void); + +void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when); +void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when); + +MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); +char *rep_hist_get_bandwidth_lines(void); +struct or_state_t; +void rep_hist_update_state(struct or_state_t *state); +int rep_hist_load_state(struct or_state_t *state, char **err); + +#ifdef BWHIST_PRIVATE +typedef struct bw_array_t bw_array_t; +STATIC uint64_t find_largest_max(bw_array_t *b); +STATIC void commit_max(bw_array_t *b); +STATIC void advance_obs(bw_array_t *b); +#endif /* defined(REPHIST_PRIVATE) */ + +#ifdef TOR_UNIT_TESTS +extern struct bw_array_t *write_array; +#endif + +#endif /* !defined(TOR_FEATURE_STATS_BWHIST_H) */ diff --git a/src/feature/stats/include.am b/src/feature/stats/include.am index 8789bc3d9..32e760a08 100644 --- a/src/feature/stats/include.am +++ b/src/feature/stats/include.am @@ -1,12 +1,14 @@ # ADD_C_FILE: INSERT SOURCES HERE. LIBTOR_APP_A_SOURCES += \ + src/feature/stats/bwhist.c \ src/feature/stats/geoip_stats.c \ src/feature/stats/rephist.c \ src/feature/stats/predict_ports.c # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ + src/feature/stats/bwhist.h \ src/feature/stats/geoip_stats.h \ src/feature/stats/rephist.h \ src/feature/stats/predict_ports.h diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index d4826cd1a..6f409a46f 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -18,11 +18,6 @@ * stability information about various relays, including "uptime", * "weighted fractional uptime" and "mean time between failures". * - * <li>Bandwidth usage history, used by relays to self-report how much - * bandwidth they've used for different purposes over last day or so, - * in order to generate the {dirreq-,}{read,write}-history lines in - * that they publish. - * * <li>Predicted ports, used by clients to remember how long it's been * since they opened an exit connection to each given target * port. Clients use this information in order to try to keep circuits @@ -77,13 +72,11 @@ #define REPHIST_PRIVATE #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/statefile.h" #include "core/or/circuitlist.h" #include "core/or/connection_or.h" #include "feature/dirauth/authmode.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" -#include "feature/relay/routermode.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/container/order.h" @@ -92,14 +85,11 @@ #include "feature/nodelist/networkstatus_st.h" #include "core/or/or_circuit_st.h" -#include "app/config/or_state_st.h" #ifdef HAVE_FCNTL_H #include <fcntl.h> #endif -static void bw_arrays_init(void); - /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; /** Number of or_history_t objects currently allocated. */ @@ -232,7 +222,6 @@ void rep_hist_init(void) { history_map = digestmap_new(); - bw_arrays_init(); } /** We have just decided that this router with identity digest id is @@ -973,560 +962,6 @@ rep_hist_load_mtbf_data(time_t now) return r; } -/** For how many seconds do we keep track of individual per-second bandwidth - * totals? */ -#define NUM_SECS_ROLLING_MEASURE 10 -/** How large are the intervals for which we track and report bandwidth use? */ -#define NUM_SECS_BW_SUM_INTERVAL (24*60*60) -/** How far in the past do we remember and publish bandwidth use? */ -#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60) -/** How many bandwidth usage intervals do we remember? (derived) */ -#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL) - -/** Structure to track bandwidth use, and remember the maxima for a given - * time period. - */ -struct bw_array_t { - /** Observation array: Total number of bytes transferred in each of the last - * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */ - uint64_t obs[NUM_SECS_ROLLING_MEASURE]; - int cur_obs_idx; /**< Current position in obs. */ - time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */ - uint64_t total_obs; /**< Total for all members of obs except - * obs[cur_obs_idx] */ - uint64_t max_total; /**< Largest value that total_obs has taken on in the - * current period. */ - uint64_t total_in_period; /**< Total bytes transferred in the current - * period. */ - - /** When does the next period begin? */ - time_t next_period; - /** Where in 'maxima' should the maximum bandwidth usage for the current - * period be stored? */ - int next_max_idx; - /** How many values in maxima/totals have been set ever? */ - int num_maxes_set; - /** Circular array of the maximum - * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last - * NUM_TOTALS periods */ - uint64_t maxima[NUM_TOTALS]; - /** Circular array of the total bandwidth usage for the last NUM_TOTALS - * periods */ - uint64_t totals[NUM_TOTALS]; -}; - -/** Shift the current period of b forward by one. */ -STATIC void -commit_max(bw_array_t *b) -{ - /* Store total from current period. */ - b->totals[b->next_max_idx] = b->total_in_period; - /* Store maximum from current period. */ - b->maxima[b->next_max_idx++] = b->max_total; - /* Advance next_period and next_max_idx */ - b->next_period += NUM_SECS_BW_SUM_INTERVAL; - if (b->next_max_idx == NUM_TOTALS) - b->next_max_idx = 0; - if (b->num_maxes_set < NUM_TOTALS) - ++b->num_maxes_set; - /* Reset max_total. */ - b->max_total = 0; - /* Reset total_in_period. */ - b->total_in_period = 0; -} - -/** Shift the current observation time of b forward by one second. */ -STATIC void -advance_obs(bw_array_t *b) -{ - int nextidx; - uint64_t total; - - /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE - * seconds; adjust max_total as needed.*/ - total = b->total_obs + b->obs[b->cur_obs_idx]; - if (total > b->max_total) - b->max_total = total; - - nextidx = b->cur_obs_idx+1; - if (nextidx == NUM_SECS_ROLLING_MEASURE) - nextidx = 0; - - b->total_obs = total - b->obs[nextidx]; - b->obs[nextidx]=0; - b->cur_obs_idx = nextidx; - - if (++b->cur_obs_time >= b->next_period) - commit_max(b); -} - -/** Add n bytes to the number of bytes in b for second - * when. */ -static inline void -add_obs(bw_array_t *b, time_t when, uint64_t n) -{ - if (when < b->cur_obs_time) - return; /* Don't record data in the past. */ - - /* If we're currently adding observations for an earlier second than - * 'when', advance b->cur_obs_time and b->cur_obs_idx by an - * appropriate number of seconds, and do all the other housekeeping. */ - while (when > b->cur_obs_time) { - /* Doing this one second at a time is potentially inefficient, if we start - with a state file that is very old. Fortunately, it doesn't seem to - show up in profiles, so we can just ignore it for now. */ - advance_obs(b); - } - - b->obs[b->cur_obs_idx] += n; - b->total_in_period += n; -} - -/** Allocate, initialize, and return a new bw_array. */ -static bw_array_t * -bw_array_new(void) -{ - bw_array_t *b; - time_t start; - b = tor_malloc_zero(sizeof(bw_array_t)); - rephist_total_alloc += sizeof(bw_array_t); - start = time(NULL); - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - return b; -} - -#define bw_array_free(val) \ - FREE_AND_NULL(bw_array_t, bw_array_free_, (val)) - -/** Free storage held by bandwidth array b. */ -static void -bw_array_free_(bw_array_t *b) -{ - if (!b) { - return; - } - - rephist_total_alloc -= sizeof(bw_array_t); - tor_free(b); -} - -/** Recent history of bandwidth observations for read operations. */ -static bw_array_t *read_array = NULL; -/** Recent history of bandwidth observations for write operations. */ -STATIC bw_array_t *write_array = NULL; -/** Recent history of bandwidth observations for read operations for the - directory protocol. */ -static bw_array_t *dir_read_array = NULL; -/** Recent history of bandwidth observations for write operations for the - directory protocol. */ -static bw_array_t *dir_write_array = NULL; - -/** Set up [dir_]read_array and [dir_]write_array, freeing them if they - * already exist. */ -static void -bw_arrays_init(void) -{ - bw_array_free(read_array); - bw_array_free(write_array); - bw_array_free(dir_read_array); - bw_array_free(dir_write_array); - - read_array = bw_array_new(); - write_array = bw_array_new(); - dir_read_array = bw_array_new(); - dir_write_array = bw_array_new(); -} - -/** Remember that we read num_bytes bytes in second when. - * - * Add num_bytes to the current running total for when. - * - * when can go back to time, but it's safe to ignore calls - * earlier than the latest when you've heard of. - */ -void -rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) -{ -/* Maybe a circular array for recent seconds, and step to a new point - * every time a new second shows up. Or simpler is to just to have - * a normal array and push down each item every second; it's short. - */ -/* When a new second has rolled over, compute the sum of the bytes we've - * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it - * somewhere. See rep_hist_bandwidth_assess() below. - */ - add_obs(write_array, when, num_bytes); -} - -/** Remember that we wrote num_bytes bytes in second when. - * (like rep_hist_note_bytes_written() above) - */ -void -rep_hist_note_bytes_read(uint64_t num_bytes, time_t when) -{ -/* if we're smart, we can make this func and the one above share code */ - add_obs(read_array, when, num_bytes); -} - -/** Remember that we wrote num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) - */ -void -rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when) -{ - add_obs(dir_write_array, when, num_bytes); -} - -/** Remember that we read num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) - */ -void -rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when) -{ - add_obs(dir_read_array, when, num_bytes); -} - -/** Helper: Return the largest value in b->maxima. (This is equal to the - * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last - * NUM_SECS_BW_SUM_IS_VALID seconds.) - */ -STATIC uint64_t -find_largest_max(bw_array_t *b) -{ - int i; - uint64_t max; - max=0; - for (i=0; i<NUM_TOTALS; ++i) { - if (b->maxima[i]>max) - max = b->maxima[i]; - } - return max; -} - -/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly) - * seconds. Find one sum for reading and one for writing. They don't have - * to be at the same time. - * - * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. - */ -MOCK_IMPL(int, -rep_hist_bandwidth_assess,(void)) -{ - uint64_t w,r; - r = find_largest_max(read_array); - w = find_largest_max(write_array); - if (r>w) - return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); - else - return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE); -} - -/** Print the bandwidth history of b (either [dir-]read_array or - * [dir-]write_array) into the buffer pointed to by buf. The format is - * simply comma separated numbers, from oldest to newest. - * - * It returns the number of bytes written. - */ -static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) -{ - char *cp = buf; - int i, n; - const or_options_t *options = get_options(); - uint64_t cutoff; - - if (b->num_maxes_set <= b->next_max_idx) { - /* We haven't been through the circular array yet; time starts at i=0.*/ - i = 0; - } else { - /* We've been around the array at least once. The next i to be - overwritten is the oldest. */ - i = b->next_max_idx; - } - - if (options->RelayBandwidthRate) { - /* We don't want to report that we used more bandwidth than the max we're - * willing to relay; otherwise everybody will know how much traffic - * we used ourself. */ - cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL; - } else { - cutoff = UINT64_MAX; - } - - for (n=0; n<b->num_maxes_set; ++n,++i) { - uint64_t total; - if (i >= NUM_TOTALS) - i -= NUM_TOTALS; - tor_assert(i < NUM_TOTALS); - /* Round the bandwidth used down to the nearest 1k. */ - total = b->totals[i] & ~0x3ff; - if (total > cutoff) - total = cutoff; - - if (n==(b->num_maxes_set-1)) - tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total)); - else - tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total)); - cp += strlen(cp); - } - return cp-buf; -} - -/** Allocate and return lines for representing this server's bandwidth - * history in its descriptor. We publish these lines in our extra-info - * descriptor. - */ -char * -rep_hist_get_bandwidth_lines(void) -{ - char *buf, *cp; - char t[ISO_TIME_LEN+1]; - int r; - bw_array_t *b = NULL; - const char *desc = NULL; - size_t len; - - /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ -/* The n,n,n part above. Largest representation of a uint64_t is 20 chars - * long, plus the comma. */ -#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) - len = (67+MAX_HIST_VALUE_LEN)*4; - buf = tor_malloc_zero(len); - cp = buf; - for (r=0;r<4;++r) { - char tmp[MAX_HIST_VALUE_LEN]; - size_t slen; - switch (r) { - case 0: - b = write_array; - desc = "write-history"; - break; - case 1: - b = read_array; - desc = "read-history"; - break; - case 2: - b = dir_write_array; - desc = "dirreq-write-history"; - break; - case 3: - b = dir_read_array; - desc = "dirreq-read-history"; - break; - } - tor_assert(b); - slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); - /* If we don't have anything to write, skip to the next entry. */ - if (slen == 0) - continue; - format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", - desc, t, NUM_SECS_BW_SUM_INTERVAL); - cp += strlen(cp); - strlcat(cp, tmp, len-(cp-buf)); - cp += slen; - strlcat(cp, "\n", len-(cp-buf)); - ++cp; - } - return buf; -} - -/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum - * entries of an or_state_t. Done before writing out a new state file. */ -static void -rep_hist_update_bwhist_state_section(or_state_t *state, - const bw_array_t *b, - smartlist_t **s_values, - smartlist_t **s_maxima, - time_t *s_begins, - int *s_interval) -{ - int i,j; - uint64_t maxval; - - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (*s_maxima) { - SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); - smartlist_free(*s_maxima); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_new(); - *s_maxima = smartlist_new(); - return; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - - *s_values = smartlist_new(); - *s_maxima = smartlist_new(); - /* Set i to first position in circular array */ - i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; - for (j=0; j < b->num_maxes_set; ++j,++i) { - if (i >= NUM_TOTALS) - i = 0; - smartlist_add_asprintf(*s_values, "%"PRIu64, - (b->totals[i] & ~0x3ff)); - maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; - smartlist_add_asprintf(*s_maxima, "%"PRIu64, - (maxval & ~0x3ff)); - } - smartlist_add_asprintf(*s_values, "%"PRIu64, - (b->total_in_period & ~0x3ff)); - maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; - smartlist_add_asprintf(*s_maxima, "%"PRIu64, - (maxval & ~0x3ff)); -} - -/** Update state with the newest bandwidth history. Done before - * writing out a new state file. */ -void -rep_hist_update_state(or_state_t *state) -{ -#define UPDATE(arrname,st) \ - rep_hist_update_bwhist_state_section(state,\ - (arrname),\ - &state->BWHistory ## st ## Values, \ - &state->BWHistory ## st ## Maxima, \ - &state->BWHistory ## st ## Ends, \ - &state->BWHistory ## st ## Interval) - - UPDATE(write_array, Write); - UPDATE(read_array, Read); - UPDATE(dir_write_array, DirWrite); - UPDATE(dir_read_array, DirRead); - - if (server_mode(get_options())) { - or_state_mark_dirty(state, time(NULL)+(2*3600)); - } -#undef UPDATE -} - -/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval - * entries in an or_state_t. Done while reading the state file. */ -static int -rep_hist_load_bwhist_state_section(bw_array_t *b, - const smartlist_t *s_values, - const smartlist_t *s_maxima, - const time_t s_begins, - const int s_interval) -{ - time_t now = time(NULL); - int retval = 0; - time_t start; - - uint64_t v, mv; - int i,ok,ok_m = 0; - int have_maxima = s_maxima && s_values && - (smartlist_len(s_values) == smartlist_len(s_maxima)); - - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - return 0; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { - const char *maxstr = NULL; - v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); - if (have_maxima) { - maxstr = smartlist_get(s_maxima, cp_sl_idx); - mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); - mv *= NUM_SECS_ROLLING_MEASURE; - } else { - /* No maxima known; guess average rate to be conservative. */ - mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; - } - if (!ok) { - retval = -1; - log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); - } - if (maxstr && !ok_m) { - retval = -1; - log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", - maxstr); - } - - if (start < now) { - time_t cur_start = start; - time_t actual_interval_len = s_interval; - uint64_t cur_val = 0; - /* Calculate the average per second. This is the best we can do - * because our state file doesn't have per-second resolution. */ - if (start + s_interval > now) - actual_interval_len = now - start; - cur_val = v / actual_interval_len; - /* This is potentially inefficient, but since we don't do it very - * often it should be ok. */ - while (cur_start < start + actual_interval_len) { - add_obs(b, cur_start, cur_val); - ++cur_start; - } - b->max_total = mv; - /* This will result in some fairly choppy history if s_interval - * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ - start += actual_interval_len; - } - } SMARTLIST_FOREACH_END(cp); - } - - /* Clean up maxima and observed */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - - return retval; -} - -/** Set bandwidth history from the state file we just loaded. */ -int -rep_hist_load_state(or_state_t *state, char **err) -{ - int all_ok = 1; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); - tor_assert(dir_read_array && dir_write_array); - -#define LOAD(arrname,st) \ - if (rep_hist_load_bwhist_state_section( \ - (arrname), \ - state->BWHistory ## st ## Values, \ - state->BWHistory ## st ## Maxima, \ - state->BWHistory ## st ## Ends, \ - state->BWHistory ## st ## Interval)<0) \ - all_ok = 0 - - LOAD(write_array, Write); - LOAD(read_array, Read); - LOAD(dir_write_array, DirWrite); - LOAD(dir_read_array, DirRead); - -#undef LOAD - if (!all_ok) { - *err = tor_strdup("Parsing of bandwidth history values failed"); - /* and create fresh arrays */ - bw_arrays_init(); - return -1; - } - return 0; -} - /*** Exit port statistics ***/ /* Some constants */ @@ -2921,18 +2356,6 @@ rep_hist_free_all(void) hs_stats_free(hs_stats); digestmap_free(history_map, free_or_history); - bw_array_free(read_array); - read_array = NULL; - - bw_array_free(write_array); - write_array = NULL; - - bw_array_free(dir_read_array); - dir_read_array = NULL; - - bw_array_free(dir_write_array); - dir_write_array = NULL; - tor_free(exit_bytes_read); tor_free(exit_bytes_written); tor_free(exit_streams); diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index d08b8833c..cdedfdca6 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -14,18 +14,9 @@ void rep_hist_init(void); void rep_hist_dump_stats(time_t now, int severity); -void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when); void rep_hist_make_router_pessimal(const char *id, time_t when); -void rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when); - -MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); -char *rep_hist_get_bandwidth_lines(void); -void rep_hist_update_state(or_state_t *state); -int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, @@ -98,16 +89,8 @@ extern uint32_t rephist_total_num; #ifdef TOR_UNIT_TESTS extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; -extern struct bw_array_t *write_array; #endif -#ifdef REPHIST_PRIVATE -typedef struct bw_array_t bw_array_t; -STATIC uint64_t find_largest_max(bw_array_t *b); -STATIC void commit_max(bw_array_t *b); -STATIC void advance_obs(bw_array_t *b); -#endif /* defined(REPHIST_PRIVATE) */ - /** * Represents the type of a cell for padding accounting */ diff --git a/src/test/test_relay.c b/src/test/test_relay.c index 066aeaa7b..060ca1b75 100644 --- a/src/test/test_relay.c +++ b/src/test/test_relay.c @@ -3,12 +3,12 @@ #define CIRCUITBUILD_PRIVATE #define RELAY_PRIVATE -#define REPHIST_PRIVATE +#define BWHIST_PRIVATE #include "core/or/or.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/channeltls.h" -#include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "core/or/relay.h" #include "lib/container/order.h" /* For init/free stuff */ diff --git a/src/test/test_router.c b/src/test/test_router.c index cf0c2b3dd..9822729f5 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -24,7 +24,7 @@ #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" -#include "feature/stats/rephist.h" +#include "feature/stats/bwhist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/encoding/confline.h" diff --git a/src/test/testing_common.c b/src/test/testing_common.c index b3337f24b..cd93055df 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -18,6 +18,7 @@ #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/stats/predict_ports.h" +#include "feature/stats/bwhist.h" #include "feature/stats/rephist.h" #include "lib/err/backtrace.h" #include "test/test.h" @@ -333,6 +334,7 @@ main(int c, const char **v) return 1; } rep_hist_init(); + bwhist_init(); setup_directory(); initialize_mainloop_events(); options_init(options); [View Less]

1 0

[tor/master] Refactor bwhist_get_bandwidth_lines()
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 11da5229d103fd13300b88a0efbc1da0b3970782 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:18:15 2020 -0400 Refactor bwhist_get_bandwidth_lines() We've done a lot to improve our internal APIs since we wrote this code, and it shows. We can just use a buf_t to build up the bandwidth lines, and save a bunch of stack fiddling. Additionally, we can use a function to format a single line, and thereby get rid of the cheezy pattern … [View More]that does for (i=0;i<n;++i) { switch (i) { ... } ... } --- src/feature/stats/bwhist.c | 81 ++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 49 deletions(-) diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index 527f0128f..fef8a3b2d 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -324,6 +324,29 @@ bwhist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) return cp-buf; } +/** Encode a single bandwidth history line into buf. */ +static void +bwhist_get_one_bandwidth_line(buf_t *buf, const char *desc, + const bw_array_t *b) +{ + /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ + /* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) + + char tmp[MAX_HIST_VALUE_LEN]; + char end[ISO_TIME_LEN+1]; + + size_t slen = bwhist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + return; + + format_iso_time(end, b->next_period-NUM_SECS_BW_SUM_INTERVAL); + buf_add_printf(buf, "%s %s (%d s) %s\n", + desc, end, NUM_SECS_BW_SUM_INTERVAL, tmp); +} + /** Allocate and return lines for representing this server's bandwidth * history in its descriptor. We publish these lines in our extra-info * descriptor. @@ -331,56 +354,16 @@ bwhist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) char * bwhist_get_bandwidth_lines(void) { - char *buf, *cp; - char t[ISO_TIME_LEN+1]; - int r; - bw_array_t *b = NULL; - const char *desc = NULL; - size_t len; + buf_t *buf = buf_new(); - /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ -/* The n,n,n part above. Largest representation of a uint64_t is 20 chars - * long, plus the comma. */ -#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) - len = (67+MAX_HIST_VALUE_LEN)*4; - buf = tor_malloc_zero(len); - cp = buf; - for (r=0;r<4;++r) { - char tmp[MAX_HIST_VALUE_LEN]; - size_t slen; - switch (r) { - case 0: - b = write_array; - desc = "write-history"; - break; - case 1: - b = read_array; - desc = "read-history"; - break; - case 2: - b = dir_write_array; - desc = "dirreq-write-history"; - break; - case 3: - b = dir_read_array; - desc = "dirreq-read-history"; - break; - } - tor_assert(b); - slen = bwhist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); - /* If we don't have anything to write, skip to the next entry. */ - if (slen == 0) - continue; - format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", - desc, t, NUM_SECS_BW_SUM_INTERVAL); - cp += strlen(cp); - strlcat(cp, tmp, len-(cp-buf)); - cp += slen; - strlcat(cp, "\n", len-(cp-buf)); - ++cp; - } - return buf; + bwhist_get_one_bandwidth_line(buf, "write-history", write_array); + bwhist_get_one_bandwidth_line(buf, "read-history", read_array); + bwhist_get_one_bandwidth_line(buf, "dirreq-write-history", dir_write_array); + bwhist_get_one_bandwidth_line(buf, "dirreq-read-history", dir_read_array); + + char *result = buf_extract(buf, NULL); + buf_free(buf); + return result; } /** Write a single bw_array_t into the Values, Ends, Interval, and Maximum [View Less]

1 0

[tor/master] Add IPv6 read and write history to bwhist, state, and extrainfo.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit c5db7667d6964344e87fb7ee71df3f2ebe8131e6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:29:24 2020 -0400 Add IPv6 read and write history to bwhist, state, and extrainfo. These values are stored, persisted, and published. They are not yet actually filled with anything. --- src/app/config/or_state_st.h | 8 ++++++++ src/app/config/statefile.c | 8 ++++++++ src/feature/stats/bwhist.c | 21 +++++++++++++++++++-- 3 files changed, 35 … [View More]insertions(+), 2 deletions(-) diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index 8c4e9d5e6..31b7f8a98 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -65,6 +65,14 @@ struct or_state_t { int BWHistoryWriteInterval; struct smartlist_t *BWHistoryWriteValues; struct smartlist_t *BWHistoryWriteMaxima; + time_t BWHistoryIPv6ReadEnds; + int BWHistoryIPv6ReadInterval; + struct smartlist_t *BWHistoryIPv6ReadValues; + struct smartlist_t *BWHistoryIPv6ReadMaxima; + time_t BWHistoryIPv6WriteEnds; + int BWHistoryIPv6WriteInterval; + struct smartlist_t *BWHistoryIPv6WriteValues; + struct smartlist_t *BWHistoryIPv6WriteMaxima; time_t BWHistoryDirReadEnds; int BWHistoryDirReadInterval; struct smartlist_t *BWHistoryDirReadValues; diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 4198c8a38..b25167d2e 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -112,6 +112,14 @@ static const config_var_t state_vars_[] = { V(BWHistoryWriteInterval, POSINT, "900"), V(BWHistoryWriteValues, CSV, ""), V(BWHistoryWriteMaxima, CSV, ""), + V(BWHistoryIPv6ReadEnds, ISOTIME, NULL), + V(BWHistoryIPv6ReadInterval, POSINT, "900"), + V(BWHistoryIPv6ReadValues, CSV, ""), + V(BWHistoryIPv6ReadMaxima, CSV, ""), + V(BWHistoryIPv6WriteEnds, ISOTIME, NULL), + V(BWHistoryIPv6WriteInterval, POSINT, "900"), + V(BWHistoryIPv6WriteValues, CSV, ""), + V(BWHistoryIPv6WriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), V(BWHistoryDirReadInterval, POSINT, "900"), V(BWHistoryDirReadValues, CSV, ""), diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index fef8a3b2d..44717f33b 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -162,10 +162,14 @@ bw_array_free_(bw_array_t *b) tor_free(b); } -/** Recent history of bandwidth observations for read operations. */ +/** Recent history of bandwidth observations for (all) read operations. */ static bw_array_t *read_array = NULL; -/** Recent history of bandwidth observations for write operations. */ +/** Recent history of bandwidth observations for IPv6 read operations. */ +static bw_array_t *read_array_ipv6 = NULL; +/** Recent history of bandwidth observations for (all) write operations. */ STATIC bw_array_t *write_array = NULL; +/** Recent history of bandwidth observations for IPv6 write operations. */ +static bw_array_t *write_array_ipv6 = NULL; /** Recent history of bandwidth observations for read operations for the directory protocol. */ static bw_array_t *dir_read_array = NULL; @@ -179,12 +183,16 @@ void bwhist_init(void) { bw_array_free(read_array); + bw_array_free(read_array_ipv6); bw_array_free(write_array); + bw_array_free(write_array_ipv6); bw_array_free(dir_read_array); bw_array_free(dir_write_array); read_array = bw_array_new(); + read_array_ipv6 = bw_array_new(); write_array = bw_array_new(); + write_array_ipv6 = bw_array_new(); dir_read_array = bw_array_new(); dir_write_array = bw_array_new(); } @@ -358,6 +366,8 @@ bwhist_get_bandwidth_lines(void) bwhist_get_one_bandwidth_line(buf, "write-history", write_array); bwhist_get_one_bandwidth_line(buf, "read-history", read_array); + bwhist_get_one_bandwidth_line(buf, "ipv6-write-history", write_array_ipv6); + bwhist_get_one_bandwidth_line(buf, "ipv6-read-history", read_array_ipv6); bwhist_get_one_bandwidth_line(buf, "dirreq-write-history", dir_write_array); bwhist_get_one_bandwidth_line(buf, "dirreq-read-history", dir_read_array); @@ -441,6 +451,8 @@ bwhist_update_state(or_state_t *state) UPDATE(write_array, Write); UPDATE(read_array, Read); + UPDATE(write_array_ipv6, IPv6Write); + UPDATE(read_array_ipv6, IPv6Read); UPDATE(dir_write_array, DirWrite); UPDATE(dir_read_array, DirRead); @@ -535,6 +547,7 @@ bwhist_load_state(or_state_t *state, char **err) /* Assert they already have been malloced */ tor_assert(read_array && write_array); + tor_assert(read_array_ipv6 && write_array_ipv6); tor_assert(dir_read_array && dir_write_array); #define LOAD(arrname,st) \ @@ -548,6 +561,8 @@ bwhist_load_state(or_state_t *state, char **err) LOAD(write_array, Write); LOAD(read_array, Read); + LOAD(write_array_ipv6, IPv6Write); + LOAD(read_array_ipv6, IPv6Read); LOAD(dir_write_array, DirWrite); LOAD(dir_read_array, DirRead); @@ -565,7 +580,9 @@ void bwhist_free_all(void) { bw_array_free(read_array); + bw_array_free(read_array_ipv6); bw_array_free(write_array); + bw_array_free(write_array_ipv6); bw_array_free(dir_read_array); bw_array_free(dir_write_array); } [View Less]

1 0

[tor/master] Rename private bandwidth-history identifiers to start with "bwhist".
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit c5eb601e60760eb56d41ec08970cb151511bf2ea Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 07:54:04 2020 -0400 Rename private bandwidth-history identifiers to start with "bwhist". This commit is a simple search-and-replace in bwhist.c --- src/feature/stats/bwhist.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index 2bc533e66..527f0128f 100644 --- a/src/feature/… [View More]stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -280,7 +280,7 @@ bwhist_bandwidth_assess,(void)) * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) +bwhist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@ -367,7 +367,7 @@ bwhist_get_bandwidth_lines(void) break; } tor_assert(b); - slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + slen = bwhist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); /* If we don't have anything to write, skip to the next entry. */ if (slen == 0) continue; @@ -386,7 +386,7 @@ bwhist_get_bandwidth_lines(void) /** Write a single bw_array_t into the Values, Ends, Interval, and Maximum * entries of an or_state_t. Done before writing out a new state file. */ static void -rep_hist_update_bwhist_state_section(or_state_t *state, +bwhist_update_bwhist_state_section(or_state_t *state, const bw_array_t *b, smartlist_t **s_values, smartlist_t **s_maxima, @@ -449,7 +449,7 @@ void bwhist_update_state(or_state_t *state) { #define UPDATE(arrname,st) \ - rep_hist_update_bwhist_state_section(state,\ + bwhist_update_bwhist_state_section(state,\ (arrname),\ &state->BWHistory ## st ## Values, \ &state->BWHistory ## st ## Maxima, \ @@ -470,7 +470,7 @@ bwhist_update_state(or_state_t *state) /** Load a single bw_array_t from its Values, Ends, Maxima, and Interval * entries in an or_state_t. Done while reading the state file. */ static int -rep_hist_load_bwhist_state_section(bw_array_t *b, +bwhist_load_bwhist_state_section(bw_array_t *b, const smartlist_t *s_values, const smartlist_t *s_maxima, const time_t s_begins, @@ -555,7 +555,7 @@ bwhist_load_state(or_state_t *state, char **err) tor_assert(dir_read_array && dir_write_array); #define LOAD(arrname,st) \ - if (rep_hist_load_bwhist_state_section( \ + if (bwhist_load_bwhist_state_section( \ (arrname), \ state->BWHistory ## st ## Values, \ state->BWHistory ## st ## Maxima, \ [View Less]

1 0

[tor/master] Rename public bandwidth-history identifiers to start with "bwhist".
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 2fc8257ac4c3b404f5e41eda8ea98eaaad6c3a5c Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 07:54:04 2020 -0400 Rename public bandwidth-history identifiers to start with "bwhist". This is an automated commit, generated by this command: ./scripts/maint/rename_c_identifier.py \ rep_hist_note_bytes_read bwhist_note_bytes_read \ rep_hist_note_bytes_written bwhist_note_bytes_written \ rep_hist_note_dir_bytes_read … [View More]bwhist_note_dir_bytes_read \ rep_hist_note_dir_bytes_written bwhist_note_dir_bytes_written \ rep_hist_get_bandwidth_lines bwhist_get_bandwidth_lines \ rep_hist_update_state bwhist_update_state \ rep_hist_load_state bwhist_load_state \ rep_hist_bandwidth_assess bwhist_bandwidth_assess --- src/app/config/statefile.c | 4 ++-- src/core/mainloop/connection.c | 8 ++++---- src/core/or/circuitlist.c | 2 +- src/feature/relay/router.c | 6 +++--- src/feature/stats/bwhist.c | 24 ++++++++++++------------ src/feature/stats/bwhist.h | 16 ++++++++-------- src/test/test_router.c | 4 ++-- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 63205007c..4198c8a38 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -324,7 +324,7 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (rep_hist_load_state(global_state, &err)<0) { + if (bwhist_load_state(global_state, &err)<0) { log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); ret = -1; @@ -523,7 +523,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ (void) subsystems_flush_state(get_state_mgr(), global_state); entry_guards_update_state(global_state); - rep_hist_update_state(global_state); + bwhist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); if (accounting_is_enabled(get_options())) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 7f1b99311..960735fc2 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -3345,9 +3345,9 @@ record_num_bytes_transferred_impl(connection_t *conn, /* Count bytes of answering direct and tunneled directory requests */ if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) { if (num_read > 0) - rep_hist_note_dir_bytes_read(num_read, now); + bwhist_note_dir_bytes_read(num_read, now); if (num_written > 0) - rep_hist_note_dir_bytes_written(num_written, now); + bwhist_note_dir_bytes_written(num_written, now); } /* Linked connections and internal IPs aren't counted for statistics or @@ -3367,10 +3367,10 @@ record_num_bytes_transferred_impl(connection_t *conn, num_written, now); if (num_read > 0) { - rep_hist_note_bytes_read(num_read, now); + bwhist_note_bytes_read(num_read, now); } if (num_written > 0) { - rep_hist_note_bytes_written(num_written, now); + bwhist_note_bytes_written(num_written, now); } if (conn->type == CONN_TYPE_EXIT) rep_hist_note_exit_bytes(conn->port, num_written, num_read); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index a6949d943..f0f182e5d 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2174,7 +2174,7 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c, /* Report the missing bytes as written, to avoid asymmetry. * We must use time() for consistency with rephist, even though on * some very old rare platforms, approx_time() may be faster. */ - rep_hist_note_bytes_written(written_sync, time(NULL)); + bwhist_note_bytes_written(written_sync, time(NULL)); } /** Mark circ to be closed next time we call diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index a55b30067..b63950ea1 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2099,7 +2099,7 @@ router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out)) ri->bandwidthburst = relay_get_effective_bwburst(options); /* Report bandwidth, unless we're hibernating or shutting down */ - ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); + ri->bandwidthcapacity = hibernating ? 0 : bwhist_bandwidth_assess(); if (dns_seems_to_be_broken() || has_dns_init_failed()) { /* DNS is screwed up; don't claim to be an exit. */ @@ -2545,7 +2545,7 @@ check_descriptor_bandwidth_changed(time_t now) /* Consider ourselves to have zero bandwidth if we're hibernating or * shutting down. */ - cur = hibernating ? 0 : rep_hist_bandwidth_assess(); + cur = hibernating ? 0 : bwhist_bandwidth_assess(); if ((prev != cur && (!prev || !cur)) || cur > (prev * BANDWIDTH_CHANGE_FACTOR) || @@ -3209,7 +3209,7 @@ extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); /* Bandwidth usage stats don't have their own option */ { - contents = rep_hist_get_bandwidth_lines(); + contents = bwhist_get_bandwidth_lines(); smartlist_add(chunks, contents); } /* geoip hashes aren't useful unless we are publishing other stats */ diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index a3b971a25..2bc533e66 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -197,7 +197,7 @@ bwhist_init(void) * earlier than the latest when you've heard of. */ void -rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) +bwhist_note_bytes_written(uint64_t num_bytes, time_t when) { /* Maybe a circular array for recent seconds, and step to a new point * every time a new second shows up. Or simpler is to just to have @@ -205,35 +205,35 @@ rep_hist_note_bytes_written(uint64_t num_bytes, time_t when) */ /* When a new second has rolled over, compute the sum of the bytes we've * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it - * somewhere. See rep_hist_bandwidth_assess() below. + * somewhere. See bwhist_bandwidth_assess() below. */ add_obs(write_array, when, num_bytes); } /** Remember that we wrote num_bytes bytes in second when. - * (like rep_hist_note_bytes_written() above) + * (like bwhist_note_bytes_written() above) */ void -rep_hist_note_bytes_read(uint64_t num_bytes, time_t when) +bwhist_note_bytes_read(uint64_t num_bytes, time_t when) { /* if we're smart, we can make this func and the one above share code */ add_obs(read_array, when, num_bytes); } /** Remember that we wrote num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) + * when. (like bwhist_note_bytes_written() above) */ void -rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when) +bwhist_note_dir_bytes_written(uint64_t num_bytes, time_t when) { add_obs(dir_write_array, when, num_bytes); } /** Remember that we read num_bytes directory bytes in second - * when. (like rep_hist_note_bytes_written() above) + * when. (like bwhist_note_bytes_written() above) */ void -rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when) +bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when) { add_obs(dir_read_array, when, num_bytes); } @@ -262,7 +262,7 @@ find_largest_max(bw_array_t *b) * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. */ MOCK_IMPL(int, -rep_hist_bandwidth_assess,(void)) +bwhist_bandwidth_assess,(void)) { uint64_t w,r; r = find_largest_max(read_array); @@ -329,7 +329,7 @@ rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) * descriptor. */ char * -rep_hist_get_bandwidth_lines(void) +bwhist_get_bandwidth_lines(void) { char *buf, *cp; char t[ISO_TIME_LEN+1]; @@ -446,7 +446,7 @@ rep_hist_update_bwhist_state_section(or_state_t *state, /** Update state with the newest bandwidth history. Done before * writing out a new state file. */ void -rep_hist_update_state(or_state_t *state) +bwhist_update_state(or_state_t *state) { #define UPDATE(arrname,st) \ rep_hist_update_bwhist_state_section(state,\ @@ -546,7 +546,7 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, /** Set bandwidth history from the state file we just loaded. */ int -rep_hist_load_state(or_state_t *state, char **err) +bwhist_load_state(or_state_t *state, char **err) { int all_ok = 1; diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h index afe0d9f4e..17d2a8394 100644 --- a/src/feature/stats/bwhist.h +++ b/src/feature/stats/bwhist.h @@ -15,16 +15,16 @@ void bwhist_init(void); void bwhist_free_all(void); -void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when); -void rep_hist_note_dir_bytes_read(uint64_t num_bytes, time_t when); -void rep_hist_note_dir_bytes_written(uint64_t num_bytes, time_t when); +void bwhist_note_bytes_read(uint64_t num_bytes, time_t when); +void bwhist_note_bytes_written(uint64_t num_bytes, time_t when); +void bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when); +void bwhist_note_dir_bytes_written(uint64_t num_bytes, time_t when); -MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); -char *rep_hist_get_bandwidth_lines(void); +MOCK_DECL(int, bwhist_bandwidth_assess, (void)); +char *bwhist_get_bandwidth_lines(void); struct or_state_t; -void rep_hist_update_state(struct or_state_t *state); -int rep_hist_load_state(struct or_state_t *state, char **err); +void bwhist_update_state(struct or_state_t *state); +int bwhist_load_state(struct or_state_t *state, char **err); #ifdef BWHIST_PRIVATE typedef struct bw_array_t bw_array_t; diff --git a/src/test/test_router.c b/src/test/test_router.c index 9822729f5..010289397 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -226,13 +226,13 @@ test_router_check_descriptor_bandwidth_changed(void *arg) /* When uptime is less than 24h and bandwidthcapacity does not change * Uptime: 10800, last_changed: x, Previous bw: 10000, Current bw: 20001 */ - MOCK(rep_hist_bandwidth_assess, mock_rep_hist_bandwidth_assess); + MOCK(bwhist_bandwidth_assess, mock_rep_hist_bandwidth_assess); setup_full_capture_of_logs(LOG_INFO); check_descriptor_bandwidth_changed(time(NULL) + 6*60*60 + 1); expect_log_msg_containing( "Measured bandwidth has changed; rebuilding descriptor."); UNMOCK(get_uptime); - UNMOCK(rep_hist_bandwidth_assess); + UNMOCK(bwhist_bandwidth_assess); teardown_capture_of_logs(); /* When uptime is more than 24h */ [View Less]

1 0

[tor/master] Record IPv6 bandwidth history as appropriate.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit 27c5cadf7e86551c7d62d27b107098266bb4c11b Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:44:46 2020 -0400 Record IPv6 bandwidth history as appropriate. --- src/core/mainloop/connection.c | 5 +++-- src/core/or/circuitlist.c | 8 +++++++- src/feature/stats/bwhist.c | 8 ++++++-- src/feature/stats/bwhist.h | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c … [View More]index 960735fc2..2f3c70365 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -3366,11 +3366,12 @@ record_num_bytes_transferred_impl(connection_t *conn, rep_hist_note_or_conn_bytes(conn->global_identifier, num_read, num_written, now); + const bool is_ipv6 = (conn->socket_family == AF_INET6); if (num_read > 0) { - bwhist_note_bytes_read(num_read, now); + bwhist_note_bytes_read(num_read, now, is_ipv6); } if (num_written > 0) { - bwhist_note_bytes_written(num_written, now); + bwhist_note_bytes_written(num_written, now, is_ipv6); } if (conn->type == CONN_TYPE_EXIT) rep_hist_note_exit_bytes(conn->port, num_written, num_read); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index f0f182e5d..28b3fa8ff 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2167,6 +2167,12 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c, else cell_size = CELL_MAX_NETWORK_SIZE; + /* If we know the channel, find out if it's IPv6. */ + tor_addr_t remote_addr; + bool is_ipv6 = chan && + channel_get_addr_if_possible(chan, &remote_addr) && + tor_addr_family(&remote_addr) == AF_INET6; + /* The missing written bytes are the cell counts times their cell * size plus TLS per cell overhead */ written_sync = cells*(cell_size+TLS_PER_CELL_OVERHEAD); @@ -2174,7 +2180,7 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c, /* Report the missing bytes as written, to avoid asymmetry. * We must use time() for consistency with rephist, even though on * some very old rare platforms, approx_time() may be faster. */ - bwhist_note_bytes_written(written_sync, time(NULL)); + bwhist_note_bytes_written(written_sync, time(NULL), is_ipv6); } /** Mark circ to be closed next time we call diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index 44717f33b..e74a2881f 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -205,7 +205,7 @@ bwhist_init(void) * earlier than the latest when you've heard of. */ void -bwhist_note_bytes_written(uint64_t num_bytes, time_t when) +bwhist_note_bytes_written(uint64_t num_bytes, time_t when, bool ipv6) { /* Maybe a circular array for recent seconds, and step to a new point * every time a new second shows up. Or simpler is to just to have @@ -216,16 +216,20 @@ bwhist_note_bytes_written(uint64_t num_bytes, time_t when) * somewhere. See bwhist_bandwidth_assess() below. */ add_obs(write_array, when, num_bytes); + if (ipv6) + add_obs(write_array_ipv6, when, num_bytes); } /** Remember that we wrote num_bytes bytes in second when. * (like bwhist_note_bytes_written() above) */ void -bwhist_note_bytes_read(uint64_t num_bytes, time_t when) +bwhist_note_bytes_read(uint64_t num_bytes, time_t when, bool ipv6) { /* if we're smart, we can make this func and the one above share code */ add_obs(read_array, when, num_bytes); + if (ipv6) + add_obs(read_array_ipv6, when, num_bytes); } /** Remember that we wrote num_bytes directory bytes in second diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h index 17d2a8394..d556f5a02 100644 --- a/src/feature/stats/bwhist.h +++ b/src/feature/stats/bwhist.h @@ -15,8 +15,8 @@ void bwhist_init(void); void bwhist_free_all(void); -void bwhist_note_bytes_read(uint64_t num_bytes, time_t when); -void bwhist_note_bytes_written(uint64_t num_bytes, time_t when); +void bwhist_note_bytes_read(uint64_t num_bytes, time_t when, bool ipv6); +void bwhist_note_bytes_written(uint64_t num_bytes, time_t when, bool ipv6); void bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when); void bwhist_note_dir_bytes_written(uint64_t num_bytes, time_t when); [View Less]

1 0

[tor/master] Make channel_get_addr_if_possible() take a const channel_t.
by dgoulet＠torproject.org 10 Jul '20

10 Jul '20

commit a81827d99ffdd36d87ca91637f022558783d44ac Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jul 10 08:41:00 2020 -0400 Make channel_get_addr_if_possible() take a const channel_t. (There is no reason that looking up the address of a channel should ever change it.) --- src/core/or/channel.c | 2 +- src/core/or/channel.h | 4 ++-- src/core/or/channeltls.c | 7 ++++--- src/test/test_dos.c | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) … [View More]diff --git a/src/core/or/channel.c b/src/core/or/channel.c index a05554472..e21f5a12c 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -2849,7 +2849,7 @@ channel_get_canonical_remote_descr,(channel_t *chan)) * doesn't let us do this. */ MOCK_IMPL(int, -channel_get_addr_if_possible,(channel_t *chan, tor_addr_t *addr_out)) +channel_get_addr_if_possible,(const channel_t *chan, tor_addr_t *addr_out)) { tor_assert(chan); tor_assert(addr_out); diff --git a/src/core/or/channel.h b/src/core/or/channel.h index 4968c8714..79e5fea9c 100644 --- a/src/core/or/channel.h +++ b/src/core/or/channel.h @@ -335,7 +335,7 @@ struct channel_t { * provided tor_addr_t *, and return 1 if successful or 0 if no address * available. */ - int (*get_remote_addr)(channel_t *, tor_addr_t *); + int (*get_remote_addr)(const channel_t *, tor_addr_t *); int (*get_transport_name)(channel_t *chan, char **transport_out); #define GRD_FLAG_ORIGINAL 1 @@ -723,7 +723,7 @@ MOCK_DECL(void, channel_dump_statistics, (channel_t *chan, int severity)); void channel_dump_transport_statistics(channel_t *chan, int severity); const char * channel_get_actual_remote_descr(channel_t *chan); const char * channel_get_actual_remote_address(channel_t *chan); -MOCK_DECL(int, channel_get_addr_if_possible, (channel_t *chan, +MOCK_DECL(int, channel_get_addr_if_possible, (const channel_t *chan, tor_addr_t *addr_out)); MOCK_DECL(const char *, channel_get_canonical_remote_descr,(channel_t *chan)); int channel_has_queued_writes(channel_t *chan); diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 4db373097..ff4505acf 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -104,7 +104,8 @@ static const char * channel_tls_describe_transport_method(channel_t *chan); static void channel_tls_free_method(channel_t *chan); static double channel_tls_get_overhead_estimate_method(channel_t *chan); static int -channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out); +channel_tls_get_remote_addr_method(const channel_t *chan, + tor_addr_t *addr_out); static int channel_tls_get_transport_name_method(channel_t *chan, char **transport_out); static const char * @@ -515,10 +516,10 @@ channel_tls_get_overhead_estimate_method(channel_t *chan) * succeeds for this transport). */ static int -channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out) +channel_tls_get_remote_addr_method(const channel_t *chan, tor_addr_t *addr_out) { int rv = 0; - channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + const channel_tls_t *tlschan = BASE_CHAN_TO_TLS((channel_t*) chan); tor_assert(tlschan); tor_assert(addr_out); diff --git a/src/test/test_dos.c b/src/test/test_dos.c index 527e5bbe7..3be09047d 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -108,7 +108,7 @@ test_dos_conn_creation(void *arg) /** Helper mock: Place a fake IP addr for this channel in addr_out */ static int -mock_channel_get_addr_if_possible(channel_t *chan, tor_addr_t *addr_out) +mock_channel_get_addr_if_possible(const channel_t *chan, tor_addr_t *addr_out) { (void)chan; tt_int_op(AF_INET,OP_EQ, tor_addr_parse(addr_out, "18.0.0.1")); [View Less]

1 0