commit 8e55cafd672353979e7628d5dad9e12429b401dd Author: Nick Mathewson nickm@torproject.org Date: Tue Aug 19 10:59:15 2014 -0400

Count zlib buffer memory towards OOM totals.

Part of 11792.

(Uses the zlib-endorsed formula for memory needs for inflate/deflate from "zconf.h".) --- changes/bug11792 | 8 ++++++- src/common/torgzip.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/common/torgzip.h | 3 +++ src/or/circuitlist.c | 8 +++++++ src/or/relay.c | 1 + 5 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/changes/bug11792 b/changes/bug11792 index 1d38189..7395b4a 100644 --- a/changes/bug11792 +++ b/changes/bug11792 @@ -2,4 +2,10 @@ - When closing an edge connection because we've run out of memory, also count the amount of memory that any tunnelled directory connection attached to that connection had consumed. Part of - ticket 11792. \ No newline at end of file + ticket 11792. + + - When considering whether we're running low on memory, consider + memory that was allocated as part of zlib buffers as well. + Count that memory as reclaimed by our OOM handler. Part of + ticket 11792. + diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 15451ee..67fd742 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -46,6 +46,12 @@

#include <zlib.h>

+static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static size_t total_zlib_allocation = 0; + /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; * set to -1 if we haven't checked yet. */ static int gzip_is_supported = -1; @@ -411,6 +417,9 @@ struct tor_zlib_state_t { size_t input_so_far; /** Number of bytes written so far. Used to detect zlib bombs. */ size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; };

/** Construct and return a tor_zlib_state_t object using <b>method</b>. If @@ -420,6 +429,7 @@ tor_zlib_state_t * tor_zlib_new(int compress, compress_method_t method) { tor_zlib_state_t *out; + int bits;

if (method == GZIP_METHOD && !is_gzip_supported()) { /* Old zlib version don't support gzip in inflateInit2 */ @@ -432,14 +442,19 @@ tor_zlib_new(int compress, compress_method_t method) out->stream.zfree = Z_NULL; out->stream.opaque = NULL; out->compress = compress; + bits = method_bits(method); if (compress) { if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK) + bits, 8, Z_DEFAULT_STRATEGY) != Z_OK) goto err; } else { - if (inflateInit2(&out->stream, method_bits(method)) != Z_OK) + if (inflateInit2(&out->stream, bits) != Z_OK) goto err; } + out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8); + + total_zlib_allocation += out->allocation; + return out;

err: @@ -517,6 +532,8 @@ tor_zlib_free(tor_zlib_state_t *state) if (!state) return;

+ total_zlib_allocation -= state->allocation; + if (state->compress) deflateEnd(&state->stream); else @@ -525,3 +542,47 @@ tor_zlib_free(tor_zlib_state_t *state) tor_free(state); }

+/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_state_size(const tor_zlib_state_t *state) +{ + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return total_zlib_allocation; +} diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 5db03fe..d407bf4 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -55,5 +55,8 @@ tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, int finish); void tor_zlib_free(tor_zlib_state_t *state);

+size_t tor_zlib_state_size(const tor_zlib_state_t *state); +size_t tor_zlib_get_total_allocation(void); + #endif

diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 9aeb3eb..c738f93 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1811,6 +1811,14 @@ marked_circuit_single_conn_free_bytes(connection_t *conn) result += buf_allocation(conn->outbuf); buf_clear(conn->outbuf); } + if (conn->type == CONN_TYPE_DIR) { + dir_connection_t *dir_conn = TO_DIR_CONN(conn); + if (dir_conn->zlib_state) { + result += tor_zlib_state_size(dir_conn->zlib_state); + tor_zlib_free(dir_conn->zlib_state); + dir_conn->zlib_state = NULL; + } + } return result; }

diff --git a/src/or/relay.c b/src/or/relay.c index f42602d..7d7ef7a 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -2433,6 +2433,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); + alloc += tor_zlib_get_total_allocation(); if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1;