commit e42c204f6773e2809b6e2dd90dabf9d177e4515b Author: Alexander Færøy ahf@torproject.org Date: Wed Apr 26 19:53:30 2017 +0200

Approximate memory usage needed for the Zstandard backend.

This patch adds support for measuring the approximated memory usage by the individual `tor_zstd_compress_state_t` object instances.

See: https://bugs.torproject.org/22066 --- src/common/compress_zstd.c | 74 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 8 deletions(-)

diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 4f642c2..4a8f03e 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -108,27 +108,86 @@ struct tor_zstd_compress_state_t { size_t allocation; };

+#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + memory_usage += (preset - 0.5) * 1024 * 1024; + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} +#endif // HAVE_ZSTD. + /** Construct and return a tor_zstd_compress_state_t object using * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for * decompression. */ tor_zstd_compress_state_t * tor_zstd_compress_new(int compress, compress_method_t method, - compression_level_t compression_level) + compression_level_t level) { tor_assert(method == ZSTD_METHOD);

#ifdef HAVE_ZSTD + const int preset = memory_level(level); tor_zstd_compress_state_t *result; size_t retval;

result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); result->compress = compress; - - // FIXME(ahf): We should either try to do the pre-calculation that is done - // with the zlib backend or use a custom allocator here where we pass our - // tor_zstd_compress_state_t as the opaque value. - result->allocation = 0; + result->allocation = tor_zstd_state_size_precalc(compress, preset);

if (compress) { result->u.compress_stream = ZSTD_createCStream(); @@ -138,8 +197,7 @@ tor_zstd_compress_new(int compress, goto err; }

- retval = ZSTD_initCStream(result->u.compress_stream, - memory_level(compression_level)); + retval = ZSTD_initCStream(result->u.compress_stream, preset);

if (ZSTD_isError(retval)) { log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",