[tor-commits] [tor/master] Approximate memory usage needed for the Zstandard backend.

[nickm at torproject.org]

commit e42c204f6773e2809b6e2dd90dabf9d177e4515b
Author: Alexander Færøy <ahf at torproject.org>
Date:   Wed Apr 26 19:53:30 2017 +0200

    Approximate memory usage needed for the Zstandard backend.
    
    This patch adds support for measuring the approximated memory usage by
    the individual `tor_zstd_compress_state_t` object instances.
    
    See: https://bugs.torproject.org/22066
---
 src/common/compress_zstd.c | 74 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 66 insertions(+), 8 deletions(-)

diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c
index 4f642c2..4a8f03e 100644
--- a/src/common/compress_zstd.c
+++ b/src/common/compress_zstd.c
@@ -108,27 +108,86 @@ struct tor_zstd_compress_state_t {
   size_t allocation;
 };
 
+#ifdef HAVE_ZSTD
+/** Return an approximate number of bytes stored in memory to hold the
+ * Zstandard compression/decompression state. */
+static size_t
+tor_zstd_state_size_precalc(int compress, int preset)
+{
+  tor_assert(preset > 0);
+
+  size_t memory_usage = sizeof(tor_zstd_compress_state_t);
+
+  // The Zstandard library provides a number of functions that would be useful
+  // here, but they are, unfortunately, still considered experimental and are
+  // thus only available in libzstd if we link against the library statically.
+  //
+  // The code in this function tries to approximate the calculations without
+  // being able to use the following:
+  //
+  // - We do not have access to neither the internal members of ZSTD_CStream
+  //   and ZSTD_DStream and their internal context objects.
+  //
+  // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
+  //   are unexposed.
+  //
+  // In the future it might be useful to check if libzstd have started
+  // providing these functions in a stable manner and simplify this function.
+  if (compress) {
+    // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
+    memory_usage += 192;
+
+    // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
+    // variables that are not exposed via the public API. We use a _very_
+    // simplified function to calculate the estimated amount of bytes used in
+    // this struct.
+    memory_usage += (preset - 0.5) * 1024 * 1024;
+    // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
+    // - stream->outBuffSize: 128 KB:
+    memory_usage += 128 * 1024;
+    // - stream->inBuffSize: 2048 KB:
+    memory_usage += 2048 * 1024;
+  } else {
+    // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
+    memory_usage += 208;
+    // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
+    memory_usage += 150 * 1024;
+
+    // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
+    // - stream->inBuffSize: 0 KB.
+    // - stream->outBuffSize: 0 KB.
+  }
+
+  return memory_usage;
+}
+#endif // HAVE_ZSTD.
+
 /** Construct and return a tor_zstd_compress_state_t object using
  * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
  * decompression. */
 tor_zstd_compress_state_t *
 tor_zstd_compress_new(int compress,
                       compress_method_t method,
-                      compression_level_t compression_level)
+                      compression_level_t level)
 {
   tor_assert(method == ZSTD_METHOD);
 
 #ifdef HAVE_ZSTD
+  const int preset = memory_level(level);
   tor_zstd_compress_state_t *result;
   size_t retval;
 
   result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
   result->compress = compress;
-
-  // FIXME(ahf): We should either try to do the pre-calculation that is done
-  // with the zlib backend or use a custom allocator here where we pass our
-  // tor_zstd_compress_state_t as the opaque value.
-  result->allocation = 0;
+  result->allocation = tor_zstd_state_size_precalc(compress, preset);
 
   if (compress) {
     result->u.compress_stream = ZSTD_createCStream();
@@ -138,8 +197,7 @@ tor_zstd_compress_new(int compress,
       goto err;
     }
 
-    retval = ZSTD_initCStream(result->u.compress_stream,
-                              memory_level(compression_level));
+    retval = ZSTD_initCStream(result->u.compress_stream, preset);
 
     if (ZSTD_isError(retval)) {
       log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",