[tor-commits] [tor/master] Count zlib buffer memory towards OOM totals.
nickm at torproject.org
nickm at torproject.org
Sun Aug 24 17:09:46 UTC 2014
commit 8e55cafd672353979e7628d5dad9e12429b401dd
Author: Nick Mathewson <nickm at torproject.org>
Date: Tue Aug 19 10:59:15 2014 -0400
Count zlib buffer memory towards OOM totals.
Part of 11792.
(Uses the zlib-endorsed formula for memory needs for inflate/deflate
from "zconf.h".)
---
changes/bug11792 | 8 ++++++-
src/common/torgzip.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++--
src/common/torgzip.h | 3 +++
src/or/circuitlist.c | 8 +++++++
src/or/relay.c | 1 +
5 files changed, 82 insertions(+), 3 deletions(-)
diff --git a/changes/bug11792 b/changes/bug11792
index 1d38189..7395b4a 100644
--- a/changes/bug11792
+++ b/changes/bug11792
@@ -2,4 +2,10 @@
- When closing an edge connection because we've run out of memory,
also count the amount of memory that any tunnelled directory
connection attached to that connection had consumed. Part of
- ticket 11792.
\ No newline at end of file
+ ticket 11792.
+
+ - When considering whether we're running low on memory, consider
+ memory that was allocated as part of zlib buffers as well.
+ Count that memory as reclaimed by our OOM handler. Part of
+ ticket 11792.
+
diff --git a/src/common/torgzip.c b/src/common/torgzip.c
index 15451ee..67fd742 100644
--- a/src/common/torgzip.c
+++ b/src/common/torgzip.c
@@ -46,6 +46,12 @@
#include <zlib.h>
+static size_t tor_zlib_state_size_precalc(int inflate,
+ int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static size_t total_zlib_allocation = 0;
+
/** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't;
* set to -1 if we haven't checked yet. */
static int gzip_is_supported = -1;
@@ -411,6 +417,9 @@ struct tor_zlib_state_t {
size_t input_so_far;
/** Number of bytes written so far. Used to detect zlib bombs. */
size_t output_so_far;
+
+ /** Approximate number of bytes allocated for this object. */
+ size_t allocation;
};
/** Construct and return a tor_zlib_state_t object using <b>method</b>. If
@@ -420,6 +429,7 @@ tor_zlib_state_t *
tor_zlib_new(int compress, compress_method_t method)
{
tor_zlib_state_t *out;
+ int bits;
if (method == GZIP_METHOD && !is_gzip_supported()) {
/* Old zlib version don't support gzip in inflateInit2 */
@@ -432,14 +442,19 @@ tor_zlib_new(int compress, compress_method_t method)
out->stream.zfree = Z_NULL;
out->stream.opaque = NULL;
out->compress = compress;
+ bits = method_bits(method);
if (compress) {
if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
- method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK)
+ bits, 8, Z_DEFAULT_STRATEGY) != Z_OK)
goto err;
} else {
- if (inflateInit2(&out->stream, method_bits(method)) != Z_OK)
+ if (inflateInit2(&out->stream, bits) != Z_OK)
goto err;
}
+ out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8);
+
+ total_zlib_allocation += out->allocation;
+
return out;
err:
@@ -517,6 +532,8 @@ tor_zlib_free(tor_zlib_state_t *state)
if (!state)
return;
+ total_zlib_allocation -= state->allocation;
+
if (state->compress)
deflateEnd(&state->stream);
else
@@ -525,3 +542,47 @@ tor_zlib_free(tor_zlib_state_t *state)
tor_free(state);
}
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel)
+{
+ windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+ if (inflate) {
+ /* From zconf.h:
+
+ "The memory requirements for inflate are (in bytes) 1 << windowBits
+ that is, 32K for windowBits=15 (default value) plus a few kilobytes
+ for small objects."
+ */
+ return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
+ (1 << 15) + A_FEW_KILOBYTES;
+ } else {
+ /* Also from zconf.h:
+
+ "The memory requirements for deflate are (in bytes):
+ (1 << (windowBits+2)) + (1 << (memLevel+9))
+ ... plus a few kilobytes for small objects."
+ */
+ return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
+ (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+ }
+#undef A_FEW_KILOBYTES
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_state_size(const tor_zlib_state_t *state)
+{
+ return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+ return total_zlib_allocation;
+}
diff --git a/src/common/torgzip.h b/src/common/torgzip.h
index 5db03fe..d407bf4 100644
--- a/src/common/torgzip.h
+++ b/src/common/torgzip.h
@@ -55,5 +55,8 @@ tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
int finish);
void tor_zlib_free(tor_zlib_state_t *state);
+size_t tor_zlib_state_size(const tor_zlib_state_t *state);
+size_t tor_zlib_get_total_allocation(void);
+
#endif
diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c
index 9aeb3eb..c738f93 100644
--- a/src/or/circuitlist.c
+++ b/src/or/circuitlist.c
@@ -1811,6 +1811,14 @@ marked_circuit_single_conn_free_bytes(connection_t *conn)
result += buf_allocation(conn->outbuf);
buf_clear(conn->outbuf);
}
+ if (conn->type == CONN_TYPE_DIR) {
+ dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+ if (dir_conn->zlib_state) {
+ result += tor_zlib_state_size(dir_conn->zlib_state);
+ tor_zlib_free(dir_conn->zlib_state);
+ dir_conn->zlib_state = NULL;
+ }
+ }
return result;
}
diff --git a/src/or/relay.c b/src/or/relay.c
index f42602d..7d7ef7a 100644
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -2433,6 +2433,7 @@ cell_queues_check_size(void)
{
size_t alloc = cell_queues_get_total_allocation();
alloc += buf_get_total_allocation();
+ alloc += tor_zlib_get_total_allocation();
if (alloc >= get_options()->MaxMemInQueues) {
circuits_handle_oom(alloc);
return 1;
More information about the tor-commits
mailing list