April 2018 - tor-commits - lists.torproject.org

[tor/master] Refactor "timestamp" not to be its own type coupled to token buffers
by nickm＠torproject.org 13 Apr '18

13 Apr '18

commit 003e6595bf355fd13f85eb31e7850cec086ecd07 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 13 12:03:29 2018 -0400 Refactor "timestamp" not to be its own type coupled to token buffers Really, the uint32_t is only an optimization; any kind of unit should work fine. Some users might want to use time_t or monotime_coarse_t or something like that. --- src/common/token_bucket.c | 19 ++++++++----------- src/common/token_bucket.h | 12 ++---------- src/test/test_bwmgt.c | 8 ++++---- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c index 8be377599..7c81264e2 100644 --- a/src/common/token_bucket.c +++ b/src/common/token_bucket.c @@ -48,12 +48,9 @@ token_bucket_cfg_init(token_bucket_cfg_t *cfg, */ void token_bucket_raw_reset(token_bucket_raw_t *bucket, - token_bucket_timestamp_t *stamp, - const token_bucket_cfg_t *cfg, - uint32_t now) + const token_bucket_cfg_t *cfg) { bucket->bucket = cfg->burst; - stamp->last_refilled_at = now; } /** @@ -165,10 +162,9 @@ void token_bucket_rw_reset(token_bucket_rw_t *bucket, uint32_t now_ts) { - token_bucket_raw_reset(&bucket->read_bucket, &bucket->stamp, - &bucket->cfg, now_ts); - token_bucket_raw_reset(&bucket->write_bucket, &bucket->stamp, - &bucket->cfg, now_ts); + token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; } /** @@ -180,9 +176,10 @@ token_bucket_rw_reset(token_bucket_rw_t *bucket, */ int token_bucket_rw_refill(token_bucket_rw_t *bucket, - uint32_t now_ts) + uint32_t now_ts) { - const uint32_t elapsed_ticks = (now_ts - bucket->stamp.last_refilled_at); + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); if (elapsed_ticks > UINT32_MAX-(300*1000)) { /* Either about 48 days have passed since the last refill, or the * monotonic clock has somehow moved backwards. (We're looking at you, @@ -208,7 +205,7 @@ token_bucket_rw_refill(token_bucket_rw_t *bucket, &bucket->cfg, elapsed_steps)) flags |= TB_WRITE; - bucket->stamp.last_refilled_at = now_ts; + bucket->last_refilled_at_timestamp = now_ts; return flags; } diff --git a/src/common/token_bucket.h b/src/common/token_bucket.h index 2507cd225..44315fb6b 100644 --- a/src/common/token_bucket.h +++ b/src/common/token_bucket.h @@ -27,12 +27,6 @@ typedef struct token_bucket_raw_t { int32_t bucket; } token_bucket_raw_t; -/** A timestamp for a token bucket. The units of this timestamp are - * unspecified, but must match with the rate set in the token_bucket_cfg_t. */ -typedef struct token_bucket_timestamp_t { - uint32_t last_refilled_at; -} token_bucket_timestamp_t; - void token_bucket_cfg_init(token_bucket_cfg_t *cfg, uint32_t rate, uint32_t burst); @@ -41,9 +35,7 @@ void token_bucket_raw_adjust(token_bucket_raw_t *bucket, const token_bucket_cfg_t *cfg); void token_bucket_raw_reset(token_bucket_raw_t *bucket, - token_bucket_timestamp_t *stamp, - const token_bucket_cfg_t *cfg, - uint32_t now); + const token_bucket_cfg_t *cfg); int token_bucket_raw_dec(token_bucket_raw_t *bucket, ssize_t n); @@ -67,7 +59,7 @@ typedef struct token_bucket_rw_t { token_bucket_cfg_t cfg; token_bucket_raw_t read_bucket; token_bucket_raw_t write_bucket; - token_bucket_timestamp_t stamp; + uint32_t last_refilled_at_timestamp; } token_bucket_rw_t; void token_bucket_rw_init(token_bucket_rw_t *bucket, diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c index 17914aeb9..003671a42 100644 --- a/src/test/test_bwmgt.c +++ b/src/test/test_bwmgt.c @@ -36,7 +36,7 @@ test_bwmgt_token_buf_init(void *arg) tt_uint_op(rate_per_sec, OP_LT, 16*KB+160); } // Bucket starts out full: - tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS); tt_int_op(b.read_bucket.bucket, OP_EQ, 64*KB); done: @@ -138,7 +138,7 @@ test_bwmgt_token_buf_refill(void *arg) tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2)); tt_int_op(b.read_bucket.bucket, OP_GT, 40*KB - 400); tt_int_op(b.read_bucket.bucket, OP_LT, 40*KB + 400); - tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS + SEC*3/2); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + SEC*3/2); /* No time: nothing happens. */ { @@ -150,12 +150,12 @@ test_bwmgt_token_buf_refill(void *arg) /* Another 30 seconds: fill the bucket. */ tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2 + SEC*30)); tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); - tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS + SEC*3/2 + SEC*30); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + SEC*3/2 + SEC*30); /* Another 30 seconds: nothing happens. */ tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2 + SEC*60)); tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); - tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS + SEC*3/2 + SEC*60); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + SEC*3/2 + SEC*60); /* Empty the bucket, let two seconds pass, and make sure that a refill is * noticed. */

1 0

[tor/master] Start re-refactoring the token bucket interface.
by nickm＠torproject.org 13 Apr '18

13 Apr '18

commit 0b40ed5e70a7200300d63db009a4f2234ff12728 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 13 11:30:53 2018 -0400 Start re-refactoring the token bucket interface. Begin by creating a lowest-level triple of the types needed to implement a token bucket: a configuration, a timestamp, and the raw bucket itself. Note that for low-level buckets, the units of the timestamp and the bucket itself are unspecified: each user can use a different type. (This patch breaks check-spaces; a later patch will fix it) --- src/common/token_bucket.c | 143 +++++++++++++++++++++++++++++++--------------- src/common/token_bucket.h | 67 ++++++++++++++++++---- src/test/test_bwmgt.c | 99 ++++++++++++++++---------------- 3 files changed, 203 insertions(+), 106 deletions(-) diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c index 3191568ac..1ac1bd6d3 100644 --- a/src/common/token_bucket.c +++ b/src/common/token_bucket.c @@ -9,8 +9,9 @@ * Tor uses these token buckets to keep track of bandwidth usage, and * sometimes other things too. * - * The time units we use internally are based on "timestamp" units -- see - * monotime_coarse_to_stamp() for a rationale. + * There are two layers of abstraction here: "raw" token buckets, in which all + * the pieces are decoupled, and "read-write" token buckets, which combine all + * the moving parts into one. * * Token buckets may become negative. **/ @@ -20,6 +21,51 @@ #include "token_bucket.h" #include "util_bug.h" +/** + * Set the rate and burst value in a token_bucket_cfg. + * + * Note that the rate value is in arbitrary units, but those units will + * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and + * so on. + */ +void +token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst) +{ + tor_assert_nonfatal(rate > 0); + tor_assert_nonfatal(burst > 0); + if (burst > TOKEN_BUCKET_MAX_BURST) + burst = TOKEN_BUCKET_MAX_BURST; + + cfg->rate = rate; + cfg->burst = burst; +} + +/** + * Initialize a raw token bucket and its associated timestamp to the "full" + * state, according to cfg. + */ +void +token_bucket_raw_reset(token_bucket_raw_t *bucket, + token_bucket_timestamp_t *stamp, + const token_bucket_cfg_t *cfg, + uint32_t now) +{ + bucket->bucket = cfg->burst; + stamp->last_refilled_at = now; +} + +/** + * Adust a preexisting token bucket to respect the new configuration + * cfg, by decreasing its current level if needed. */ +void +token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = MIN(bucket->bucket, cfg->burst); +} + /** Convert a rate in bytes per second to a rate in bytes per step */ static uint32_t rate_per_sec_to_rate_per_step(uint32_t rate) @@ -57,18 +103,14 @@ token_bucket_rw_init(token_bucket_rw_t *bucket, */ void token_bucket_rw_adjust(token_bucket_rw_t *bucket, - uint32_t rate, - uint32_t burst) + uint32_t rate, + uint32_t burst) { - tor_assert_nonfatal(rate > 0); - tor_assert_nonfatal(burst > 0); - if (burst > TOKEN_BUCKET_RW_MAX_BURST) - burst = TOKEN_BUCKET_RW_MAX_BURST; - - bucket->rate = rate_per_sec_to_rate_per_step(rate); - bucket->burst = burst; - bucket->read_bucket = MIN(bucket->read_bucket, (int32_t)burst); - bucket->write_bucket = MIN(bucket->write_bucket, (int32_t)burst); + token_bucket_cfg_init(&bucket->cfg, + rate_per_sec_to_rate_per_step(rate), + burst); + token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg); } /** @@ -76,36 +118,41 @@ token_bucket_rw_adjust(token_bucket_rw_t *bucket, */ void token_bucket_rw_reset(token_bucket_rw_t *bucket, - uint32_t now_ts) + uint32_t now_ts) { - bucket->read_bucket = bucket->burst; - bucket->write_bucket = bucket->burst; - bucket->last_refilled_at_ts = now_ts; + token_bucket_raw_reset(&bucket->read_bucket, &bucket->stamp, + &bucket->cfg, now_ts); + token_bucket_raw_reset(&bucket->write_bucket, &bucket->stamp, + &bucket->cfg, now_ts); } -/* Helper: see token_bucket_rw_refill */ -static int -refill_single_bucket(int32_t *bucketptr, - const uint32_t rate, - const int32_t burst, - const uint32_t elapsed_steps) +/** + * Given an amount of elapsed time units, and a bucket configuration + * cfg, refill the level of bucket accordingly. Note that the + * units of time in elapsed must correspond to those used to set the + * rate in cfg, or the result will be illogical. + */ +int +token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed) { - const int was_empty = (*bucketptr <= 0); + const int was_empty = (bucket->bucket <= 0); /* The casts here prevent an underflow. * * Note that even if the bucket value is negative, subtracting it from * "burst" will still produce a correct result. If this result is - * ridiculously high, then the "elapsed_steps > gap / rate" check below + * ridiculously high, then the "elapsed > gap / rate" check below * should catch it. */ - const size_t gap = ((size_t)burst) - ((size_t)*bucketptr); + const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket); - if (elapsed_steps > gap / rate) { - *bucketptr = burst; + if (elapsed > gap / cfg->rate) { + bucket->bucket = cfg->burst; } else { - *bucketptr += rate * elapsed_steps; + bucket->bucket += cfg->rate * elapsed; } - return was_empty && *bucketptr > 0; + return was_empty && bucket->bucket > 0; } /** @@ -119,7 +166,7 @@ int token_bucket_rw_refill(token_bucket_rw_t *bucket, uint32_t now_ts) { - const uint32_t elapsed_ticks = (now_ts - bucket->last_refilled_at_ts); + const uint32_t elapsed_ticks = (now_ts - bucket->stamp.last_refilled_at); if (elapsed_ticks > UINT32_MAX-(300*1000)) { /* Either about 48 days have passed since the last refill, or the * monotonic clock has somehow moved backwards. (We're looking at you, @@ -132,31 +179,35 @@ token_bucket_rw_refill(token_bucket_rw_t *bucket, if (!elapsed_steps) { /* Note that if less than one whole step elapsed, we don't advance the - * time in last_refilled_at_ts. That's intentional: we want to make sure + * time in last_refilled_at. That's intentional: we want to make sure * that we add some bytes to it eventually. */ return 0; } int flags = 0; - if (refill_single_bucket(&bucket->read_bucket, - bucket->rate, bucket->burst, elapsed_steps)) + if (token_bucket_raw_refill_steps(&bucket->read_bucket, + &bucket->cfg, elapsed_steps)) flags |= TB_READ; - if (refill_single_bucket(&bucket->write_bucket, - bucket->rate, bucket->burst, elapsed_steps)) + if (token_bucket_raw_refill_steps(&bucket->write_bucket, + &bucket->cfg, elapsed_steps)) flags |= TB_WRITE; - bucket->last_refilled_at_ts = now_ts; + bucket->stamp.last_refilled_at = now_ts; return flags; } -static int -decrement_single_bucket(int32_t *bucketptr, - ssize_t n) +/** + * Decrement a provided bucket by n units. Note that n + * must be nonnegative. + */ +int +token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n) { if (BUG(n < 0)) return 0; - const int becomes_empty = *bucketptr > 0 && n >= *bucketptr; - *bucketptr -= n; + const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket; + bucket->bucket -= n; return becomes_empty; } @@ -170,7 +221,7 @@ int token_bucket_rw_dec_read(token_bucket_rw_t *bucket, ssize_t n) { - return decrement_single_bucket(&bucket->read_bucket, n); + return token_bucket_raw_dec(&bucket->read_bucket, n); } /** @@ -183,7 +234,7 @@ int token_bucket_rw_dec_write(token_bucket_rw_t *bucket, ssize_t n) { - return decrement_single_bucket(&bucket->write_bucket, n); + return token_bucket_raw_dec(&bucket->write_bucket, n); } /** @@ -192,9 +243,9 @@ token_bucket_rw_dec_write(token_bucket_rw_t *bucket, */ void token_bucket_rw_dec(token_bucket_rw_t *bucket, - ssize_t n_read, ssize_t n_written) + ssize_t n_read, ssize_t n_written) { token_bucket_rw_dec_read(bucket, n_read); - token_bucket_rw_dec_read(bucket, n_written); + token_bucket_rw_dec_write(bucket, n_written); } diff --git a/src/common/token_bucket.h b/src/common/token_bucket.h index f9d04f79a..2507cd225 100644 --- a/src/common/token_bucket.h +++ b/src/common/token_bucket.h @@ -11,15 +11,64 @@ #include "torint.h" -typedef struct token_bucket_rw_t { +/** Largest allowable burst value for a token buffer. */ +#define TOKEN_BUCKET_MAX_BURST INT32_MAX + +/** A generic token buffer configuration: determines the number of tokens + * added to the bucket in each time unit (the "rate"), and the maximum number + * of tokens in the bucket (the "burst") */ +typedef struct token_bucket_cfg_t { uint32_t rate; int32_t burst; - int32_t read_bucket; - int32_t write_bucket; - uint32_t last_refilled_at_ts; -} token_bucket_rw_t; +} token_bucket_cfg_t; + +/** A raw token bucket, decoupled from its configuration and timestamp. */ +typedef struct token_bucket_raw_t { + int32_t bucket; +} token_bucket_raw_t; + +/** A timestamp for a token bucket. The units of this timestamp are + * unspecified, but must match with the rate set in the token_bucket_cfg_t. */ +typedef struct token_bucket_timestamp_t { + uint32_t last_refilled_at; +} token_bucket_timestamp_t; + +void token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst); + +void token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +void token_bucket_raw_reset(token_bucket_raw_t *bucket, + token_bucket_timestamp_t *stamp, + const token_bucket_cfg_t *cfg, + uint32_t now); -#define TOKEN_BUCKET_RW_MAX_BURST INT32_MAX +int token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n); + +int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed_steps); + +static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket); +/** Return the current number of bytes set in a token bucket. */ +static inline size_t +token_bucket_raw_get(const token_bucket_raw_t *bucket) +{ + return bucket->bucket >= 0 ? bucket->bucket : 0; +} + +/** A convenience type containing all the pieces needed for a coupled + * read-bucket and write-bucket that have the same rate limit, and which use + * "timestamp units" (see compat_time.h) for their time. */ +typedef struct token_bucket_rw_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t read_bucket; + token_bucket_raw_t write_bucket; + token_bucket_timestamp_t stamp; +} token_bucket_rw_t; void token_bucket_rw_init(token_bucket_rw_t *bucket, uint32_t rate, @@ -50,8 +99,7 @@ static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket); static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket) { - const ssize_t b = bucket->read_bucket; - return b >= 0 ? b : 0; + return token_bucket_raw_get(&bucket->read_bucket); } static inline size_t token_bucket_rw_get_write( @@ -59,8 +107,7 @@ static inline size_t token_bucket_rw_get_write( static inline size_t token_bucket_rw_get_write(const token_bucket_rw_t *bucket) { - const ssize_t b = bucket->write_bucket; - return b >= 0 ? b : 0; + return token_bucket_raw_get(&bucket->write_bucket); } #ifdef TOKEN_BUCKET_PRIVATE diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c index 9e4748af7..17914aeb9 100644 --- a/src/test/test_bwmgt.c +++ b/src/test/test_bwmgt.c @@ -25,19 +25,19 @@ test_bwmgt_token_buf_init(void *arg) token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); // Burst is correct - tt_uint_op(b.burst, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 64*KB); // Rate is correct, within 1 percent. { uint32_t ticks_per_sec = (uint32_t) monotime_msec_to_approx_coarse_stamp_units(1000); - uint32_t rate_per_sec = (b.rate * ticks_per_sec / TICKS_PER_STEP); + uint32_t rate_per_sec = (b.cfg.rate * ticks_per_sec / TICKS_PER_STEP); tt_uint_op(rate_per_sec, OP_GT, 16*KB-160); tt_uint_op(rate_per_sec, OP_LT, 16*KB+160); } // Bucket starts out full: - tt_uint_op(b.last_refilled_at_ts, OP_EQ, START_TS); - tt_int_op(b.read_bucket, OP_EQ, 64*KB); + tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS); + tt_int_op(b.read_bucket.bucket, OP_EQ, 64*KB); done: ; @@ -51,31 +51,31 @@ test_bwmgt_token_buf_adjust(void *arg) token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); - uint32_t rate_orig = b.rate; + uint32_t rate_orig = b.cfg.rate; // Increasing burst token_bucket_rw_adjust(&b, 16*KB, 128*KB); - tt_uint_op(b.rate, OP_EQ, rate_orig); - tt_uint_op(b.read_bucket, OP_EQ, 64*KB); - tt_uint_op(b.burst, OP_EQ, 128*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 128*KB); // Decreasing burst but staying above bucket token_bucket_rw_adjust(&b, 16*KB, 96*KB); - tt_uint_op(b.rate, OP_EQ, rate_orig); - tt_uint_op(b.read_bucket, OP_EQ, 64*KB); - tt_uint_op(b.burst, OP_EQ, 96*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 96*KB); // Decreasing burst below bucket, token_bucket_rw_adjust(&b, 16*KB, 48*KB); - tt_uint_op(b.rate, OP_EQ, rate_orig); - tt_uint_op(b.read_bucket, OP_EQ, 48*KB); - tt_uint_op(b.burst, OP_EQ, 48*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 48*KB); // Changing rate. token_bucket_rw_adjust(&b, 32*KB, 48*KB); - tt_uint_op(b.rate, OP_GE, rate_orig*2 - 10); - tt_uint_op(b.rate, OP_LE, rate_orig*2 + 10); - tt_uint_op(b.read_bucket, OP_EQ, 48*KB); - tt_uint_op(b.burst, OP_EQ, 48*KB); + tt_uint_op(b.cfg.rate, OP_GE, rate_orig*2 - 10); + tt_uint_op(b.cfg.rate, OP_LE, rate_orig*2 + 10); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 48*KB); done: ; @@ -90,29 +90,29 @@ test_bwmgt_token_buf_dec(void *arg) // full-to-not-full. tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, KB)); - tt_int_op(b.read_bucket, OP_EQ, 63*KB); + tt_int_op(b.read_bucket.bucket, OP_EQ, 63*KB); // Full to almost-not-full tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 63*KB - 1)); - tt_int_op(b.read_bucket, OP_EQ, 1); + tt_int_op(b.read_bucket.bucket, OP_EQ, 1); // almost-not-full to empty. tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 1)); - tt_int_op(b.read_bucket, OP_EQ, 0); + tt_int_op(b.read_bucket.bucket, OP_EQ, 0); // reset bucket, try full-to-empty token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB)); - tt_int_op(b.read_bucket, OP_EQ, 0); + tt_int_op(b.read_bucket.bucket, OP_EQ, 0); // reset bucket, try underflow. token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB + 1)); - tt_int_op(b.read_bucket, OP_EQ, -1); + tt_int_op(b.read_bucket.bucket, OP_EQ, -1); // A second underflow does not make the bucket empty. tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 1000)); - tt_int_op(b.read_bucket, OP_EQ, -1001); + tt_int_op(b.read_bucket.bucket, OP_EQ, -1001); done: ; @@ -125,68 +125,67 @@ test_bwmgt_token_buf_refill(void *arg) token_bucket_rw_t b; const uint32_t SEC = (uint32_t)monotime_msec_to_approx_coarse_stamp_units(1000); - printf("%d\n", (int)SEC); token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); /* Make the buffer much emptier, then let one second elapse. */ token_bucket_rw_dec_read(&b, 48*KB); - tt_int_op(b.read_bucket, OP_EQ, 16*KB); + tt_int_op(b.read_bucket.bucket, OP_EQ, 16*KB); tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC)); - tt_int_op(b.read_bucket, OP_GT, 32*KB - 300); - tt_int_op(b.read_bucket, OP_LT, 32*KB + 300); + tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB - 300); + tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB + 300); /* Another half second. */ tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2)); - tt_int_op(b.read_bucket, OP_GT, 40*KB - 400); - tt_int_op(b.read_bucket, OP_LT, 40*KB + 400); - tt_uint_op(b.last_refilled_at_ts, OP_EQ, START_TS + SEC*3/2); + tt_int_op(b.read_bucket.bucket, OP_GT, 40*KB - 400); + tt_int_op(b.read_bucket.bucket, OP_LT, 40*KB + 400); + tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS + SEC*3/2); /* No time: nothing happens. */ { - const uint32_t bucket_orig = b.read_bucket; + const uint32_t bucket_orig = b.read_bucket.bucket; tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2)); - tt_int_op(b.read_bucket, OP_EQ, bucket_orig); + tt_int_op(b.read_bucket.bucket, OP_EQ, bucket_orig); } /* Another 30 seconds: fill the bucket. */ tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2 + SEC*30)); - tt_int_op(b.read_bucket, OP_EQ, b.burst); - tt_uint_op(b.last_refilled_at_ts, OP_EQ, START_TS + SEC*3/2 + SEC*30); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS + SEC*3/2 + SEC*30); /* Another 30 seconds: nothing happens. */ tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2 + SEC*60)); - tt_int_op(b.read_bucket, OP_EQ, b.burst); - tt_uint_op(b.last_refilled_at_ts, OP_EQ, START_TS + SEC*3/2 + SEC*60); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + tt_uint_op(b.stamp.last_refilled_at, OP_EQ, START_TS + SEC*3/2 + SEC*60); /* Empty the bucket, let two seconds pass, and make sure that a refill is * noticed. */ - tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.burst)); - tt_int_op(0, OP_EQ, b.read_bucket); + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.cfg.burst)); + tt_int_op(0, OP_EQ, b.read_bucket.bucket); tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2 + SEC*61)); tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*3/2 + SEC*62)); - tt_int_op(b.read_bucket, OP_GT, 32*KB-400); - tt_int_op(b.read_bucket, OP_LT, 32*KB+400); + tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB+400); /* Underflow the bucket, make sure we detect when it has tokens again. */ - tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.read_bucket+16*KB)); - tt_int_op(-16*KB, OP_EQ, b.read_bucket); + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.read_bucket.bucket+16*KB)); + tt_int_op(-16*KB, OP_EQ, b.read_bucket.bucket); // half a second passes... tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*64)); - tt_int_op(b.read_bucket, OP_GT, -8*KB-300); - tt_int_op(b.read_bucket, OP_LT, -8*KB+300); + tt_int_op(b.read_bucket.bucket, OP_GT, -8*KB-300); + tt_int_op(b.read_bucket.bucket, OP_LT, -8*KB+300); // a second passes tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*65)); - tt_int_op(b.read_bucket, OP_GT, 8*KB-400); - tt_int_op(b.read_bucket, OP_LT, 8*KB+400); + tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400); // We step a second backwards, and nothing happens. tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + SEC*64)); - tt_int_op(b.read_bucket, OP_GT, 8*KB-400); - tt_int_op(b.read_bucket, OP_LT, 8*KB+400); + tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400); // A ridiculous amount of time passes. tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, INT32_MAX)); - tt_int_op(b.read_bucket, OP_EQ, b.burst); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); done: ;

1 0

[tor/master] Never pick a rate of 0.
by nickm＠torproject.org 13 Apr '18

13 Apr '18

commit 9c405ba59528b931aefc9ed80fed3e5d9f231d27 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 13 11:38:57 2018 -0400 Never pick a rate of 0. (The tests caught this one.) --- src/common/token_bucket.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c index 1ac1bd6d3..abfcc680c 100644 --- a/src/common/token_bucket.c +++ b/src/common/token_bucket.c @@ -76,8 +76,9 @@ rate_per_sec_to_rate_per_step(uint32_t rate) (rate / 1000) * to_approximate_msec(TICKS_PER_STEP). But to minimize rounding error, we do it this way instead, and divide last. */ - return (uint32_t) + uint32_t val = (uint32_t) monotime_coarse_stamp_units_to_approx_msec(rate*TICKS_PER_STEP)/1000; + return val ? val : 1; } /**

1 0

[doctor/master] Check for clock skew
by atagar＠torproject.org 13 Apr '18

13 Apr '18

commit 5399092235e6d1f74841347b01adddaf9df63d3a Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 12:52:35 2018 -0700 Check for clock skew Roger had the great suggestion that we should check for clock skew... https://trac.torproject.org/projects/tor/ticket/25768 Expanded stem a bit along with this to surface response headers. --- consensus_health_checker.py | 13 ++++++++++--- data/consensus_health.cfg | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index 48c1224..981c4a6 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -927,7 +927,7 @@ def get_votes(): def _get_documents(label, resource): - documents, times_taken, issues = {}, {}, [] + documents, times_taken, clock_skew, issues = {}, {}, {}, [] for authority in DIRECTORY_AUTHORITIES.values(): if authority.v3ident is None: @@ -940,9 +940,12 @@ def _get_documents(label, resource): ) try: - start_time = time.time() + start_time = datetime.datetime.utcnow() documents[authority.nickname] = query.run()[0] - times_taken[authority.nickname] = time.time() - start_time + response_timestamp = datetime.datetime.strptime(query.reply_headers.get('date'), '%a, %d %b %Y %H:%M:%S %Z') + + times_taken[authority.nickname] = (datetime.datetime.utcnow() - start_time).total_seconds() + clock_skew[authority.nickname] = abs((start_time - response_timestamp).total_seconds()) except Exception as exc: issues.append(Issue(Runlevel.ERROR, 'AUTHORITY_UNAVAILABLE', fetch_type = label, authority = authority.nickname, url = query.download_url, error = exc, to = [authority.nickname])) @@ -954,6 +957,10 @@ def _get_documents(label, resource): if time_taken > median_time * 5: issues.append(Issue(Runlevel.NOTICE, 'LATENCY', authority = nickname, time_taken = '%0.1fs' % time_taken, median_time = '%0.1fs' % median_time, authority_times = authority_times, to = [nickname])) + for nickname, difference in clock_skew.items(): + if difference > 10: + issues.append(Issue(Runlevel.NOTICE, 'CLOCK_SKEW', authority = nickname, difference = difference, to = [nickname])) + return documents, issues diff --git a/data/consensus_health.cfg b/data/consensus_health.cfg index 39aa2e5..994b361 100644 --- a/data/consensus_health.cfg +++ b/data/consensus_health.cfg @@ -1,6 +1,7 @@ # message templates for notifications we send msg LATENCY => Downloading the consensus from {authority} took {time_taken}. Median download time is {median_time}: {authority_times} +msg CLOCK_SKEW => The system clock of {authority} is {difference} seconds off msg MISSING_LATEST_CONSENSUS => The consensuses published by the following directory authorities are more than one hour old and therefore not fresh anymore: {authorities} msg MISSING_AUTHORITY_DESC => {authority} is missing the server descriptor of {peer} msg CONSENSUS_METHOD_UNSUPPORTED => The following directory authorities do not support the consensus method that the consensus uses: {authorities}

1 0

[translation/support-miscellaneous] Update translations for support-miscellaneous
by translation＠torproject.org 13 Apr '18

13 Apr '18

commit 69b979c265ed7e1b53c88e6d69b4935fa17db725 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Apr 13 19:20:43 2018 +0000 Update translations for support-miscellaneous --- de.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/de.json b/de.json index 0e9978046..fb386e096 100644 --- a/de.json +++ b/de.json @@ -2,13 +2,13 @@ "misc-1": { "id": "#misc-1", "control": "misc-1", - "title": "I have a compelling reason to trace a Tor user. Can you help?", - "description": "There is nothing the Tor developers can do to trace Tor users. The same protections that keep bad people from breaking Tor's anonymity also prevent us from tracking users." + "title": "Ich habe einen zwingenden Grund einen Tor Benutzer zu verfolgen. Können Sie mir helfen?", + "description": "Die Tor Entwickler haben keine Möglichkeit einen Tor Benutzer zu verfolgen. Der gleiche Mechanismus der verhindert das böse Menschen die Tors Anonymität gefährden trifft auch bei uns Entwicklern zu." }, "misc-2": { "id": "#misc-2", "control": "misc-2", - "title": "Why don't you prevent bad people from doing bad things when using Tor?", + "title": "Warum verhindert Ihr nicht das böse Menschen mit Tor böse Dinge machen?", "description": "Tor is designed to defend human rights and privacy by preventing anyone from censoring things, even us. We hate that there are some people who use Tor to do terrible things, but we can't do anything to get rid of them without also undermining the human rights activists, journalists, abuse survivors, and other people who use Tor for good things. If we wanted to block certain people from using Tor, we'd basically be adding a backdoor to the software, which would open up our vulnerable users to attacks from bad regimes and other adversaries." }, "misc-3": {

1 0

[stem/master] Stacktrace when tests call is_accessible after stopping
by atagar＠torproject.org 13 Apr '18

13 Apr '18

commit 2018f3fd2642f43aeab8e1e0d2142a6a4905e651 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 11:49:08 2018 -0700 Stacktrace when tests call is_accessible after stopping Huh. I can't repro this failure and I'm unsure how it arose, but our jenkins tests failed a test run with... ====================================================================== ERROR: test_attachstream ---------------------------------------------------------------------- Traceback (most recent call last): File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/require.py", line 57, in wrapped if condition(): File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/require.py", line 68, in _can_access_controller return test.runner.get_runner().is_accessible() File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/runner.py", line 341, in is_accessible return Torrc.PORT in self._custom_opts or Torrc.SOCKET in self._custom_opts TypeError: argument of type 'NoneType' is not iterable This should only arise if we haven't called start() on our Runner, or called its stop() method before the test ran. In either case there's no doubt a second bug here but for the moment just making this obvious fix. All callers expect the opts to be a collection, so no reason not to initialize it to one. --- test/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runner.py b/test/runner.py index bd3fe8a4..160e4827 100644 --- a/test/runner.py +++ b/test/runner.py @@ -161,7 +161,7 @@ class Runner(object): self._tor_cmd = None self._tor_cwd = '' self._torrc_contents = '' - self._custom_opts = None + self._custom_opts = [] self._tor_process = None self._chroot_path = None @@ -308,7 +308,7 @@ class Runner(object): self._tor_cmd = None self._tor_cwd = '' self._torrc_contents = '' - self._custom_opts = None + self._custom_opts = [] self._tor_process = None println('done', STATUS)

1 0

[stem/master] Response headers when downloading descriptors
by atagar＠torproject.org 13 Apr '18

13 Apr '18

commit 187d37661f06ed0db754d43f299be68ce1e78f7b Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 13 11:42:35 2018 -0700 Response headers when downloading descriptors Adding a reply_headers attribute to our Query class so callers can get the headers the DirPort responded with. This is needed for... https://trac.torproject.org/projects/tor/ticket/25768 Generally speaking though good data to make available to our callers. --- docs/change_log.rst | 1 + stem/descriptor/remote.py | 13 ++++++++++- test/unit/descriptor/remote.py | 50 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index ac827bdf..e4053d91 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -58,6 +58,7 @@ The following are only available within Stem's `git repository * Added zstd and lzma compression support (:spec:`1cb56af`) * Added server descriptor's new is_hidden_service_dir attribute * Don't retry downloading descriptors when we've timed out + * Added the reply_headers attribute to :class:`~stem.descriptor.remote.Query` * Supplying a User-Agent when downloading descriptors. * Reduced maximum descriptors fetched by the remote module to match tor's new limit (:trac:`24743`) * Consensus **shared_randomness_*_reveal_count** attributes undocumented, and unavailable if retrieved before their corresponding shared_randomness_*_value attribute (:trac:`25046`) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 30ae388b..8e5b759d 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -326,6 +326,13 @@ class Query(object): .. versionchanged:: 1.7.0 Added the compression argument. + .. versionchanged:: 1.7.0 + Added the reply_headers attribute. + + The class this provides changed between Python versions. In python2 + this was called httplib.HTTPMessage, whereas in python3 the class was + renamed to http.client.HTTPMessage. + :var str resource: resource being fetched, such as '/tor/server/all' :var str descriptor_type: type of descriptors being fetched (for options see :func:`~stem.descriptor.__init__.parse_file`), this is guessed from the @@ -356,6 +363,8 @@ class Query(object): **True**, skips these checks otherwise :var stem.descriptor.__init__.DocumentHandler document_handler: method in which to parse a :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` + :var http.client.HTTPMessage reply_headers: headers provided in the response, + **None** if we haven't yet made our request :var dict kwargs: additional arguments for the descriptor constructor :param bool start: start making the request when constructed (default is **True**) @@ -409,6 +418,7 @@ class Query(object): self.validate = validate self.document_handler = document_handler + self.reply_headers = None self.kwargs = kwargs self._downloader_thread = None @@ -538,7 +548,7 @@ class Query(object): ) data = response.read() - encoding = response.info().get('Content-Encoding') + encoding = response.headers.get('Content-Encoding') # Tor doesn't include compression headers. As such when using gzip we # need to include '32' for automatic header detection... @@ -560,6 +570,7 @@ class Query(object): data = lzma.decompress(data) self.content = data.strip() + self.reply_headers = response.headers self.runtime = time.time() - self.start_time log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime)) except: diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 1dfccc09..a033d1c1 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.remote. """ +import io import socket import tempfile import time @@ -15,6 +16,11 @@ from stem.descriptor.remote import Compression from test.unit.descriptor import read_resource try: + from http.client import HTTPMessage # python3 +except ImportError: + from httplib import HTTPMessage # python2 + +try: # added in python 2.7 from collections import OrderedDict except ImportError: @@ -111,11 +117,30 @@ FALLBACK_ENTRY = b"""\ /* extrainfo=1 */ """ +HEADER = '\r\n'.join([ + 'Date: Fri, 13 Apr 2018 16:35:50 GMT', + 'Content-Type: application/octet-stream', + 'X-Your-Address-Is: 97.103.17.56', + 'Pragma: no-cache', + 'Content-Encoding: %s', +]) + def _urlopen_mock(data, encoding = 'identity'): urlopen_mock = Mock() urlopen_mock().read.return_value = data - urlopen_mock().info().get.return_value = encoding + + if stem.prereq.is_python_3(): + headers = HTTPMessage() + + for line in HEADER.splitlines(): + key, value = line.split(': ', 1) + headers.add_header(key, encoding if key == 'Content-Encoding' else value) + + urlopen_mock().headers = headers + else: + urlopen_mock().headers = HTTPMessage(io.BytesIO(HEADER % encoding)) + return urlopen_mock @@ -220,6 +245,29 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual('moria1', descriptors[0].nickname) @patch(URL_OPEN, _urlopen_mock(TEST_DESCRIPTOR)) + def test_reply_headers(self): + query = stem.descriptor.remote.get_server_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31', start = False) + self.assertEqual(None, query.reply_headers) # initially we don't have a reply + query.run() + + self.assertEqual('Fri, 13 Apr 2018 16:35:50 GMT', query.reply_headers.get('date')) + self.assertEqual('application/octet-stream', query.reply_headers.get('content-type')) + self.assertEqual('97.103.17.56', query.reply_headers.get('x-your-address-is')) + self.assertEqual('no-cache', query.reply_headers.get('pragma')) + self.assertEqual('identity', query.reply_headers.get('content-encoding')) + + # getting headers should be case insensitive + self.assertEqual('identity', query.reply_headers.get('CoNtEnT-ENCODING')) + + # request a header that isn't present + self.assertEqual(None, query.reply_headers.get('no-such-header')) + self.assertEqual('default', query.reply_headers.get('no-such-header', 'default')) + + descriptors = list(query) + self.assertEqual(1, len(descriptors)) + self.assertEqual('moria1', descriptors[0].nickname) + + @patch(URL_OPEN, _urlopen_mock(TEST_DESCRIPTOR)) def test_query_download(self): """ Check Query functionality when we successfully download a descriptor.

1 0

[tor/release-0.3.3] Merge branch 'maint-0.3.3' into release-0.3.3
by nickm＠torproject.org 13 Apr '18

13 Apr '18

commit f95f6a44982bd8130a840b564501ed8d87e92bb6 Merge: 50406c272 4d8de6c6b Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 13 13:09:15 2018 -0400 Merge branch 'maint-0.3.3' into release-0.3.3 changes/ticket25248 | 4 + doc/tor.1.txt | 225 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 134 insertions(+), 95 deletions(-)

1 0

[tor/release-0.3.3] Merge remote-tracking branch 'dgoulet/ticket25248_033_02' into maint-0.3.3
by nickm＠torproject.org 13 Apr '18

13 Apr '18

commit 4d8de6c6be53bd881755b19dfece563953340cc8 Merge: 2d6914e39 21b347e29 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 13 13:09:11 2018 -0400 Merge remote-tracking branch 'dgoulet/ticket25248_033_02' into maint-0.3.3 changes/ticket25248 | 4 + doc/tor.1.txt | 225 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 134 insertions(+), 95 deletions(-)

1 0

[tor/release-0.3.3] doc: Fix typo and clarify that DoS options are relay only
by nickm＠torproject.org 13 Apr '18

13 Apr '18

commit 21b347e291871ab89c140c9842f72cee76dd1c29 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Wed Apr 11 02:47:00 2018 +0000 doc: Fix typo and clarify that DoS options are relay only --- doc/tor.1.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 8ce374c12..369cdb526 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2412,7 +2412,8 @@ DENIAL OF SERVICE MITIGATION OPTIONS Tor has three built-in mitigation options that can be individually enabled/disabled and fine-tuned, but by default Tor directory authorities will define reasonable values for relays and no explicit configuration is required -to make use of these protections. The mitigations are: +to make use of these protections. The mitigations take place at relays, +and are as follows: 1. If a single client address makes too many concurrent connections (this is configurable via DoSConnectionMaxConcurrentCount), hang up on further @@ -2429,13 +2430,13 @@ to make use of these protections. The mitigations are: Tor2Web client), ignore the request. These defenses can be manually controlled by torrc options, but relays will -also take guidance from consensus parameters, so there's no need to configure -anything manually. In doubt, do not change those values. +also take guidance from consensus parameters using these same names, so there's +no need to configure anything manually. In doubt, do not change those values. The values set by the consensus, if any, can be found here: https://consensus-health.torproject.org/#consensusparams -If any of the DoS mitigations are enabled, an heartbeat message will appear in +If any of the DoS mitigations are enabled, a heartbeat message will appear in your log at NOTICE level which looks like: DoS mitigation since startup: 429042 circuits rejected, 17 marked addresses.

1 0