[tor-commits] [tor/master] Merge branch 'maint-0.3.4'

nickm at torproject.org nickm at torproject.org
Mon Jul 30 12:17:30 UTC 2018


commit 5823e62fa2a83fb1f7a27c82be930749e3a4782d
Merge: 9ae359754 d102e9c2e
Author: Nick Mathewson <nickm at torproject.org>
Date:   Mon Jul 30 08:17:25 2018 -0400

    Merge branch 'maint-0.3.4'

 changes/bug26785                 | 4 ++++
 configure.ac                     | 3 +++
 src/lib/compress/compress_zstd.c | 4 ++++
 3 files changed, 11 insertions(+)

diff --cc src/lib/compress/compress_zstd.c
index 0a71fed4b,000000000..fe88d4a54
mode 100644,000000..100644
--- a/src/lib/compress/compress_zstd.c
+++ b/src/lib/compress/compress_zstd.c
@@@ -1,537 -1,0 +1,541 @@@
 +/* Copyright (c) 2004, Roger Dingledine.
 + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 + * Copyright (c) 2007-2018, The Tor Project, Inc. */
 +/* See LICENSE for licensing information */
 +
 +/**
 + * \file compress_zstd.c
 + * \brief Compression backend for Zstandard.
 + *
 + * This module should never be invoked directly. Use the compress module
 + * instead.
 + **/
 +
 +#include "orconfig.h"
 +
 +#include "lib/log/log.h"
 +#include "lib/log/util_bug.h"
 +#include "lib/compress/compress.h"
 +#include "lib/compress/compress_zstd.h"
 +#include "lib/string/printf.h"
 +#include "lib/thread/threads.h"
 +
 +#ifdef ENABLE_ZSTD_ADVANCED_APIS
 +/* This is a lie, but we make sure it doesn't get us in trouble by wrapping
 + * all invocations of zstd's static-only functions in a check to make sure
 + * that the compile-time version matches the run-time version. */
 +#define ZSTD_STATIC_LINKING_ONLY
 +#endif
 +
 +#ifdef HAVE_ZSTD
++#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
 +DISABLE_GCC_WARNING(unused-const-variable)
++#endif
 +#include <zstd.h>
++#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
 +ENABLE_GCC_WARNING(unused-const-variable)
 +#endif
++#endif
 +
 +/** Total number of bytes allocated for Zstandard state. */
 +static atomic_counter_t total_zstd_allocation;
 +
 +#ifdef HAVE_ZSTD
 +/** Given <b>level</b> return the memory level. */
 +static int
 +memory_level(compression_level_t level)
 +{
 +  switch (level) {
 +    default:
 +    case BEST_COMPRESSION:
 +    case HIGH_COMPRESSION: return 9;
 +    case MEDIUM_COMPRESSION: return 8;
 +    case LOW_COMPRESSION: return 7;
 +  }
 +}
 +#endif /* defined(HAVE_ZSTD) */
 +
 +/** Return 1 if Zstandard compression is supported; otherwise 0. */
 +int
 +tor_zstd_method_supported(void)
 +{
 +#ifdef HAVE_ZSTD
 +  return 1;
 +#else
 +  return 0;
 +#endif
 +}
 +
 +#ifdef HAVE_ZSTD
 +/** Format a zstd version number as a string in <b>buf</b>. */
 +static void
 +tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
 +{
 +  tor_snprintf(buf, buflen,
 +               "%u.%u.%u",
 +               version_number / 10000 % 100,
 +               version_number / 100 % 100,
 +               version_number % 100);
 +}
 +#endif
 +
 +#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
 +
 +/** Return a string representation of the version of the currently running
 + * version of libzstd. Returns NULL if Zstandard is unsupported. */
 +const char *
 +tor_zstd_get_version_str(void)
 +{
 +#ifdef HAVE_ZSTD
 +  static char version_str[VERSION_STR_MAX_LEN];
 +
 +  tor_zstd_format_version(version_str, sizeof(version_str),
 +                          ZSTD_versionNumber());
 +
 +  return version_str;
 +#else /* !(defined(HAVE_ZSTD)) */
 +  return NULL;
 +#endif /* defined(HAVE_ZSTD) */
 +}
 +
 +/** Return a string representation of the version of the version of libzstd
 + * used at compilation time. Returns NULL if Zstandard is unsupported. */
 +const char *
 +tor_zstd_get_header_version_str(void)
 +{
 +#ifdef HAVE_ZSTD
 +  return ZSTD_VERSION_STRING;
 +#else
 +  return NULL;
 +#endif
 +}
 +
 +#ifdef TOR_UNIT_TESTS
 +static int static_apis_disable_for_testing = 0;
 +#endif
 +
 +/** Return true iff we can use the "static-only" APIs. */
 +int
 +tor_zstd_can_use_static_apis(void)
 +{
 +#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)
 +#ifdef TOR_UNIT_TESTS
 +  if (static_apis_disable_for_testing) {
 +    return 0;
 +  }
 +#endif
 +  return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
 +#else
 +  return 0;
 +#endif
 +}
 +
 +/** Internal Zstandard state for incremental compression/decompression.
 + * The body of this struct is not exposed. */
 +struct tor_zstd_compress_state_t {
 +#ifdef HAVE_ZSTD
 +  union {
 +    /** Compression stream. Used when <b>compress</b> is true. */
 +    ZSTD_CStream *compress_stream;
 +    /** Decompression stream. Used when <b>compress</b> is false. */
 +    ZSTD_DStream *decompress_stream;
 +  } u; /**< Zstandard stream objects. */
 +#endif /* defined(HAVE_ZSTD) */
 +
 +  int compress; /**< True if we are compressing; false if we are inflating */
 +  int have_called_end; /**< True if we are compressing and we've called
 +                        * ZSTD_endStream */
 +
 +  /** Number of bytes read so far.  Used to detect compression bombs. */
 +  size_t input_so_far;
 +  /** Number of bytes written so far.  Used to detect compression bombs. */
 +  size_t output_so_far;
 +
 +  /** Approximate number of bytes allocated for this object. */
 +  size_t allocation;
 +};
 +
 +#ifdef HAVE_ZSTD
 +/** Return an approximate number of bytes stored in memory to hold the
 + * Zstandard compression/decompression state. This is a fake estimate
 + * based on inspecting the zstd source: tor_zstd_state_size_precalc() is
 + * more accurate when it's allowed to use "static-only" functions */
 +static size_t
 +tor_zstd_state_size_precalc_fake(int compress, int preset)
 +{
 +  tor_assert(preset > 0);
 +
 +  size_t memory_usage = sizeof(tor_zstd_compress_state_t);
 +
 +  // The Zstandard library provides a number of functions that would be useful
 +  // here, but they are, unfortunately, still considered experimental and are
 +  // thus only available in libzstd if we link against the library statically.
 +  //
 +  // The code in this function tries to approximate the calculations without
 +  // being able to use the following:
 +  //
 +  // - We do not have access to neither the internal members of ZSTD_CStream
 +  //   and ZSTD_DStream and their internal context objects.
 +  //
 +  // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
 +  //   are unexposed.
 +  //
 +  // In the future it might be useful to check if libzstd have started
 +  // providing these functions in a stable manner and simplify this function.
 +  if (compress) {
 +    // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
 +    // function here. This function uses the following fields to make its
 +    // estimate:
 +
 +    // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
 +    memory_usage += 192;
 +
 +    // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
 +    // variables that are not exposed via the public API. We use a _very_
 +    // simplified function to calculate the estimated amount of bytes used in
 +    // this struct.
 +    // memory_usage += (preset - 0.5) * 1024 * 1024;
 +    memory_usage += (preset * 1024 * 1024) - (512 * 1024);
 +    // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
 +    // - stream->outBuffSize: 128 KB:
 +    memory_usage += 128 * 1024;
 +    // - stream->inBuffSize: 2048 KB:
 +    memory_usage += 2048 * 1024;
 +  } else {
 +    // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
 +    // function here. This function uses the following fields to make its
 +    // estimate:
 +
 +    // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
 +    memory_usage += 208;
 +    // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
 +    memory_usage += 150 * 1024;
 +
 +    // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
 +    // - stream->inBuffSize: 0 KB.
 +    // - stream->outBuffSize: 0 KB.
 +  }
 +
 +  return memory_usage;
 +}
 +
 +/** Return an approximate number of bytes stored in memory to hold the
 + * Zstandard compression/decompression state. */
 +static size_t
 +tor_zstd_state_size_precalc(int compress, int preset)
 +{
 +#ifdef ZSTD_STATIC_LINKING_ONLY
 +  if (tor_zstd_can_use_static_apis()) {
 +    if (compress) {
 +#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE
 +      return ZSTD_estimateCStreamSize(preset);
 +#endif
 +    } else {
 +#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE
 +      /* Could use DStream, but that takes a windowSize. */
 +      return ZSTD_estimateDCtxSize();
 +#endif
 +    }
 +  }
 +#endif
 +  return tor_zstd_state_size_precalc_fake(compress, preset);
 +}
 +#endif /* defined(HAVE_ZSTD) */
 +
 +/** Construct and return a tor_zstd_compress_state_t object using
 + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
 + * decompression. */
 +tor_zstd_compress_state_t *
 +tor_zstd_compress_new(int compress,
 +                      compress_method_t method,
 +                      compression_level_t level)
 +{
 +  tor_assert(method == ZSTD_METHOD);
 +
 +#ifdef HAVE_ZSTD
 +  const int preset = memory_level(level);
 +  tor_zstd_compress_state_t *result;
 +  size_t retval;
 +
 +  result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
 +  result->compress = compress;
 +  result->allocation = tor_zstd_state_size_precalc(compress, preset);
 +
 +  if (compress) {
 +    result->u.compress_stream = ZSTD_createCStream();
 +
 +    if (result->u.compress_stream == NULL) {
 +      // LCOV_EXCL_START
 +      log_warn(LD_GENERAL, "Error while creating Zstandard compression "
 +               "stream");
 +      goto err;
 +      // LCOV_EXCL_STOP
 +    }
 +
 +    retval = ZSTD_initCStream(result->u.compress_stream, preset);
 +
 +    if (ZSTD_isError(retval)) {
 +      // LCOV_EXCL_START
 +      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
 +               ZSTD_getErrorName(retval));
 +      goto err;
 +      // LCOV_EXCL_STOP
 +    }
 +  } else {
 +    result->u.decompress_stream = ZSTD_createDStream();
 +
 +    if (result->u.decompress_stream == NULL) {
 +      // LCOV_EXCL_START
 +      log_warn(LD_GENERAL, "Error while creating Zstandard decompression "
 +               "stream");
 +      goto err;
 +      // LCOV_EXCL_STOP
 +    }
 +
 +    retval = ZSTD_initDStream(result->u.decompress_stream);
 +
 +    if (ZSTD_isError(retval)) {
 +      // LCOV_EXCL_START
 +      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
 +               ZSTD_getErrorName(retval));
 +      goto err;
 +      // LCOV_EXCL_STOP
 +    }
 +  }
 +
 +  atomic_counter_add(&total_zstd_allocation, result->allocation);
 +  return result;
 +
 + err:
 +  // LCOV_EXCL_START
 +  if (compress) {
 +    ZSTD_freeCStream(result->u.compress_stream);
 +  } else {
 +    ZSTD_freeDStream(result->u.decompress_stream);
 +  }
 +
 +  tor_free(result);
 +  return NULL;
 +  // LCOV_EXCL_STOP
 +#else /* !(defined(HAVE_ZSTD)) */
 +  (void)compress;
 +  (void)method;
 +  (void)level;
 +
 +  return NULL;
 +#endif /* defined(HAVE_ZSTD) */
 +}
 +
 +/** Compress/decompress some bytes using <b>state</b>.  Read up to
 + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
 + * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
 + * we've reached the end of the input.
 + *
 + * Return TOR_COMPRESS_DONE if we've finished the entire
 + * compression/decompression.
 + * Return TOR_COMPRESS_OK if we're processed everything from the input.
 + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
 + * Return TOR_COMPRESS_ERROR if the stream is corrupt.
 + */
 +tor_compress_output_t
 +tor_zstd_compress_process(tor_zstd_compress_state_t *state,
 +                          char **out, size_t *out_len,
 +                          const char **in, size_t *in_len,
 +                          int finish)
 +{
 +#ifdef HAVE_ZSTD
 +  size_t retval;
 +
 +  tor_assert(state != NULL);
 +  tor_assert(*in_len <= UINT_MAX);
 +  tor_assert(*out_len <= UINT_MAX);
 +
 +  ZSTD_inBuffer input = { *in, *in_len, 0 };
 +  ZSTD_outBuffer output = { *out, *out_len, 0 };
 +
 +  if (BUG(finish == 0 && state->have_called_end)) {
 +    finish = 1;
 +  }
 +
 +  if (state->compress) {
 +    if (! state->have_called_end)
 +      retval = ZSTD_compressStream(state->u.compress_stream,
 +                                   &output, &input);
 +    else
 +      retval = 0;
 +  } else {
 +    retval = ZSTD_decompressStream(state->u.decompress_stream,
 +                                   &output, &input);
 +  }
 +
 +  state->input_so_far += input.pos;
 +  state->output_so_far += output.pos;
 +
 +  *out = (char *)output.dst + output.pos;
 +  *out_len = output.size - output.pos;
 +  *in = (char *)input.src + input.pos;
 +  *in_len = input.size - input.pos;
 +
 +  if (! state->compress &&
 +      tor_compress_is_compression_bomb(state->input_so_far,
 +                                       state->output_so_far)) {
 +    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
 +    return TOR_COMPRESS_ERROR;
 +  }
 +
 +  if (ZSTD_isError(retval)) {
 +    log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
 +             state->compress ? "compression" : "decompression",
 +             ZSTD_getErrorName(retval));
 +    return TOR_COMPRESS_ERROR;
 +  }
 +
 +  if (state->compress && !state->have_called_end) {
 +    retval = ZSTD_flushStream(state->u.compress_stream, &output);
 +
 +    *out = (char *)output.dst + output.pos;
 +    *out_len = output.size - output.pos;
 +
 +    if (ZSTD_isError(retval)) {
 +      log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
 +               ZSTD_getErrorName(retval));
 +      return TOR_COMPRESS_ERROR;
 +    }
 +
 +    // ZSTD_flushStream returns 0 if the frame is done, or >0 if it
 +    // is incomplete.
 +    if (retval > 0) {
 +      return TOR_COMPRESS_BUFFER_FULL;
 +    }
 +  }
 +
 +  if (!finish) {
 +    // The caller says we're not done with the input, so no need to write an
 +    // epilogue.
 +    return TOR_COMPRESS_OK;
 +  } else if (state->compress) {
 +    if (*in_len) {
 +      // We say that we're not done with the input, so we can't write an
 +      // epilogue.
 +      return TOR_COMPRESS_OK;
 +    }
 +
 +    retval = ZSTD_endStream(state->u.compress_stream, &output);
 +    state->have_called_end = 1;
 +    *out = (char *)output.dst + output.pos;
 +    *out_len = output.size - output.pos;
 +
 +    if (ZSTD_isError(retval)) {
 +      log_warn(LD_GENERAL, "Zstandard compression unable to write "
 +               "epilogue: %s.",
 +               ZSTD_getErrorName(retval));
 +      return TOR_COMPRESS_ERROR;
 +    }
 +
 +    // endStream returns the number of bytes that is needed to write the
 +    // epilogue.
 +    if (retval > 0)
 +      return TOR_COMPRESS_BUFFER_FULL;
 +
 +    return TOR_COMPRESS_DONE;
 +  } else /* if (!state->compress) */ {
 +    // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it
 +    // is incomplete.
 +    // We check this above.
 +    tor_assert_nonfatal(!ZSTD_isError(retval));
 +    // Start a new frame if this frame is done
 +    if (retval == 0)
 +      return TOR_COMPRESS_DONE;
 +    // Don't check out_len, it might have some space left if the next output
 +    // chunk is larger than the remaining space
 +    else if (*in_len > 0)
 +      return  TOR_COMPRESS_BUFFER_FULL;
 +    else
 +      return TOR_COMPRESS_OK;
 +  }
 +
 +#else /* !(defined(HAVE_ZSTD)) */
 +  (void)state;
 +  (void)out;
 +  (void)out_len;
 +  (void)in;
 +  (void)in_len;
 +  (void)finish;
 +
 +  return TOR_COMPRESS_ERROR;
 +#endif /* defined(HAVE_ZSTD) */
 +}
 +
 +/** Deallocate <b>state</b>. */
 +void
 +tor_zstd_compress_free_(tor_zstd_compress_state_t *state)
 +{
 +  if (state == NULL)
 +    return;
 +
 +  atomic_counter_sub(&total_zstd_allocation, state->allocation);
 +
 +#ifdef HAVE_ZSTD
 +  if (state->compress) {
 +    ZSTD_freeCStream(state->u.compress_stream);
 +  } else {
 +    ZSTD_freeDStream(state->u.decompress_stream);
 +  }
 +#endif /* defined(HAVE_ZSTD) */
 +
 +  tor_free(state);
 +}
 +
 +/** Return the approximate number of bytes allocated for <b>state</b>. */
 +size_t
 +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
 +{
 +  tor_assert(state != NULL);
 +  return state->allocation;
 +}
 +
 +/** Return the approximate number of bytes allocated for all Zstandard
 + * states. */
 +size_t
 +tor_zstd_get_total_allocation(void)
 +{
 +  return atomic_counter_get(&total_zstd_allocation);
 +}
 +
 +/** Initialize the zstd module */
 +void
 +tor_zstd_init(void)
 +{
 +  atomic_counter_init(&total_zstd_allocation);
 +}
 +
 +/** Warn if the header and library versions don't match. */
 +void
 +tor_zstd_warn_if_version_mismatched(void)
 +{
 +#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS)
 +  if (! tor_zstd_can_use_static_apis()) {
 +    char header_version[VERSION_STR_MAX_LEN];
 +    char runtime_version[VERSION_STR_MAX_LEN];
 +    tor_zstd_format_version(header_version, sizeof(header_version),
 +                            ZSTD_VERSION_NUMBER);
 +    tor_zstd_format_version(runtime_version, sizeof(runtime_version),
 +                            ZSTD_versionNumber());
 +
 +    log_warn(LD_GENERAL,
 +             "Tor was compiled with zstd %s, but is running with zstd %s. "
 +             "For safety, we'll avoid using advanced zstd functionality.",
 +             header_version, runtime_version);
 +  }
 +#endif
 +}
 +
 +#ifdef TOR_UNIT_TESTS
 +/** Testing only: disable usage of static-only APIs, so we can make sure that
 + * we still work without them. */
 +void
 +tor_zstd_set_static_apis_disabled_for_testing(int disabled)
 +{
 +  static_apis_disable_for_testing = disabled;
 +}
 +#endif



More information about the tor-commits mailing list