commit 880fb3e3a9aba21c0b36aa4aa5659432e90eb827 Author: Nick Mathewson <nickm@torproject.org> Date: Tue Apr 25 09:41:23 2017 -0400 Combine all *compress/*uncompress backend function into one Since we have a streaming API for each compression backend, we don't need a non-streaming API for each: we can build a common non-streaming API at the front-end. --- src/common/compress.c | 182 ++++++++++++++++++++++------ src/common/compress_lzma.c | 290 --------------------------------------------- src/common/compress_lzma.h | 10 -- src/common/compress_zlib.c | 254 --------------------------------------- src/common/compress_zlib.h | 10 -- src/common/compress_zstd.c | 283 ------------------------------------------- src/common/compress_zstd.h | 10 -- src/test/test_util.c | 2 +- 8 files changed, 148 insertions(+), 893 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 38b8184..2729d9f 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -56,6 +56,147 @@ tor_compress_is_compression_bomb(size_t size_in, size_t size_out) return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); } +/** Guess the size that <b>in_len</b> will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)method; + (void)compression_level; + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether <b>compress</b> is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) + return -1; + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + goto err; + } else { + goto done; + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompresing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompresing data: bad input?", + compress?"":"un"); + goto err; // bad data. + default: + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + /** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly * allocated buffer, using the method described in <b>method</b>. Store the * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. @@ -66,19 +207,9 @@ tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method) { - switch (method) { - case GZIP_METHOD: - case ZLIB_METHOD: - return tor_zlib_compress(out, out_len, in, in_len, method); - case LZMA_METHOD: - return tor_lzma_compress(out, out_len, in, in_len, method); - case ZSTD_METHOD: - return tor_zstd_compress(out, out_len, in, in_len, method); - case NO_METHOD: - case UNKNOWN_METHOD: - default: - return -1; - } + return tor_compress_impl(1, out, out_len, in, in_len, method, + HIGH_COMPRESSION, + 1, LOG_WARN); // XXXX "best"? } /** Given zero or more zlib-compressed or gzip-compressed strings of @@ -99,28 +230,9 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level) { - switch (method) { - case GZIP_METHOD: - case ZLIB_METHOD: - return tor_zlib_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - case LZMA_METHOD: - return tor_lzma_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - case ZSTD_METHOD: - return tor_zstd_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - case NO_METHOD: - case UNKNOWN_METHOD: - default: - return -1; - } + return tor_compress_impl(0, out, out_len, in, in_len, method, + HIGH_COMPRESSION, + complete_only, protocol_warn_level); } /** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index ae0327f..9c61d58 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -108,296 +108,6 @@ tor_lzma_get_header_version_str(void) #endif } -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the LZMA method. Store the compressed string in - * *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on - * failure. - */ -int -tor_lzma_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ -#ifdef HAVE_LZMA - lzma_stream stream = LZMA_STREAM_INIT; - lzma_options_lzma stream_options; - lzma_ret retval; - lzma_action action; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == LZMA_METHOD); - - stream.next_in = (unsigned char *)in; - stream.avail_in = in_len; - - lzma_lzma_preset(&stream_options, - memory_level(HIGH_COMPRESSION)); - - retval = lzma_alone_encoder(&stream, &stream_options); - - if (retval != LZMA_OK) { - log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", - lzma_error_str(retval), retval); - goto err; - } - - out_size = in_len / 2; - if (out_size < 1024) - out_size = 1024; - - *out = tor_malloc(out_size); - - stream.next_out = (unsigned char *)*out; - stream.avail_out = out_size; - - action = LZMA_RUN; - - while (1) { - retval = lzma_code(&stream, action); - switch (retval) { - case LZMA_OK: - action = LZMA_FINISH; - break; - case LZMA_STREAM_END: - goto done; - case LZMA_BUF_ERROR: - offset = stream.next_out - ((unsigned char *)*out); - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in LZMA compression."); - goto err; - } - - *out = tor_realloc(*out, out_size); - stream.next_out = (unsigned char *)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " - "compressing."); - goto err; - } - stream.avail_out = (unsigned int)(out_size - offset); - break; - - // We list all the possible values of `lzma_ret` here to silence the - // `switch-enum` warning and to detect if a new member was added. - case LZMA_NO_CHECK: - case LZMA_UNSUPPORTED_CHECK: - case LZMA_GET_CHECK: - case LZMA_MEM_ERROR: - case LZMA_MEMLIMIT_ERROR: - case LZMA_FORMAT_ERROR: - case LZMA_OPTIONS_ERROR: - case LZMA_DATA_ERROR: - case LZMA_PROG_ERROR: - default: - log_warn(LD_GENERAL, "LZMA compression didn't finish: %s.", - lzma_error_str(retval)); - goto err; - } - } - - done: - *out_len = stream.total_out; - lzma_end(&stream); - - if (tor_compress_is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tor instances would think " - "this is a compression bomb."); - goto err; - } - - return 0; - - err: - lzma_end(&stream); - tor_free(*out); - return -1; -#else // HAVE_LZMA. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - - return -1; -#endif // HAVE_LZMA. -} - -/** Given an LZMA compressed string of total length <b>in_len</b> bytes at - * <b>in</b>, uncompress them into a newly allocated buffer. Store the - * uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a failure; - * otherwise we decompress as much as we can. Warn about truncated or corrupt - * inputs at <b>protocol_warn_level</b>. - */ -int -tor_lzma_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ -#ifdef HAVE_LZMA - lzma_stream stream = LZMA_STREAM_INIT; - lzma_ret retval; - lzma_action action; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == LZMA_METHOD); - - stream.next_in = (unsigned char *)in; - stream.avail_in = in_len; - - // FIXME(ahf): This should be something more sensible than - // UINT64_MAX: See #21665. - retval = lzma_alone_decoder(&stream, UINT64_MAX); - - if (retval != LZMA_OK) { - log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", - lzma_error_str(retval), retval); - goto err; - } - - out_size = in_len * 2; - if (out_size < 1024) - out_size = 1024; - - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream.next_out = (unsigned char *)*out; - stream.avail_out = out_size; - - // FIXME(ahf): We should figure out how to use LZMA_FULL_FLUSH to - // make the partial string read tests. - // action = complete_only ? LZMA_FINISH : LZMA_SYNC_FLUSH. // To do this, - // it seems like we have to use LZMA using their "xz" encoder instead of just - // regular LZMA. - (void)complete_only; - action = LZMA_FINISH; - - while (1) { - retval = lzma_code(&stream, action); - switch (retval) { - case LZMA_STREAM_END: - if (stream.avail_in == 0) - goto done; - - // We might have more data here. Reset our stream. - lzma_end(&stream); - - retval = lzma_alone_decoder(&stream, UINT64_MAX); - - if (retval != LZMA_OK) { - log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", - lzma_error_str(retval), retval); - goto err; - } - break; - case LZMA_OK: - break; - case LZMA_BUF_ERROR: - if (stream.avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt LZMA data."); - goto err; - } - - offset = stream.next_out - (unsigned char *)*out; - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in LZMA uncompression."); - goto err; - } - - if (tor_compress_is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible LZMA compression " - "bomb. Not proceeding."); - goto err; - } - - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " - "LZMA data."); - goto err; - } - - *out = tor_realloc(*out, out_size); - stream.next_out = (unsigned char *)(*out + offset); - - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " - "uncompressing."); - goto err; - } - - stream.avail_out = (unsigned int)(out_size - offset); - break; - - // We list all the possible values of `lzma_ret` here to silence the - // `switch-enum` warning and to detect if a new member was added. - case LZMA_NO_CHECK: - case LZMA_UNSUPPORTED_CHECK: - case LZMA_GET_CHECK: - case LZMA_MEM_ERROR: - case LZMA_MEMLIMIT_ERROR: - case LZMA_FORMAT_ERROR: - case LZMA_OPTIONS_ERROR: - case LZMA_DATA_ERROR: - case LZMA_PROG_ERROR: - default: - log_warn(LD_GENERAL, "LZMA decompression didn't finish: %s.", - lzma_error_str(retval)); - goto err; - } - } - - done: - *out_len = stream.next_out - (unsigned char*)*out; - lzma_end(&stream); - - // NUL-terminate our output. - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - - err: - lzma_end(&stream); - tor_free(*out); - return -1; -#else // HAVE_LZMA. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - (void)complete_only; - (void)protocol_warn_level; - - return -1; -#endif // HAVE_LZMA. -} - /** Internal LZMA state for incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_lzma_compress_state_t { diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h index 71de56a..79cf9dc 100644 --- a/src/common/compress_lzma.h +++ b/src/common/compress_lzma.h @@ -17,16 +17,6 @@ const char *tor_lzma_get_version_str(void); const char *tor_lzma_get_header_version_str(void); -int tor_lzma_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); - -int tor_lzma_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - /** Internal state for an incremental LZMA compression/decompression. */ typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index faa179f..37768ee 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -102,260 +102,6 @@ tor_zlib_get_header_version_str(void) return ZLIB_VERSION; } -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the method described in <b>method</b>. Store the - * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. Return - * 0 on success, -1 on failure. - */ -int -tor_zlib_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - memory_level(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (tor_compress_is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given an Zlib/Gzip compressed string of total length <b>in_len</b> bytes - * at <b>in</b>, uncompress them into a newly allocated buffer. Store the - * uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a failure; - * otherwise we decompress as much as we can. Warn about truncated or corrupt - * inputs at <b>protocol_warn_level</b>. - */ -int -tor_zlib_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (tor_compress_is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - /** Internal zlib state for an incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_zlib_compress_state_t { diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h index 6e8e5c5..2dd1254 100644 --- a/src/common/compress_zlib.h +++ b/src/common/compress_zlib.h @@ -17,16 +17,6 @@ const char *tor_zlib_get_version_str(void); const char *tor_zlib_get_header_version_str(void); -int tor_zlib_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); - -int tor_zlib_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - /** Internal state for an incremental zlib/gzip compression/decompression. */ typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 664cce1..76f2991 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -85,289 +85,6 @@ tor_zstd_get_header_version_str(void) #endif } -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the Zstandard method. Store the compressed string - * in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 - * on failure. - */ -int -tor_zstd_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ -#ifdef HAVE_ZSTD - ZSTD_CStream *stream = NULL; - size_t out_size, old_size; - size_t retval; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == ZSTD_METHOD); - - *out = NULL; - - stream = ZSTD_createCStream(); - - if (stream == NULL) { - // Zstandard does not give us any useful error message to why this - // happened. See https://github.com/facebook/zstd/issues/398 - log_warn(LD_GENERAL, "Error while creating Zstandard stream"); - goto err; - } - - retval = ZSTD_initCStream(stream, - memory_level(HIGH_COMPRESSION)); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - // Assume 50% compression and update our buffer in case we need to. - out_size = in_len / 2; - if (out_size < 1024) - out_size = 1024; - - *out = tor_malloc(out_size); - *out_len = 0; - - ZSTD_inBuffer input = { in, in_len, 0 }; - ZSTD_outBuffer output = { *out, out_size, 0 }; - - while (input.pos < input.size) { - retval = ZSTD_compressStream(stream, &output, &input); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream compression error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - if (input.pos < input.size && output.pos == output.size) { - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); - goto err; - } - - if (out_size - output.pos > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " - "compressing."); - goto err; - } - - output.dst = *out = tor_realloc(*out, out_size); - output.size = out_size; - } - } - - while (1) { - retval = ZSTD_endStream(stream, &output); - - if (retval == 0) - break; - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - if (output.pos == output.size) { - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); - goto err; - } - - if (out_size - output.pos > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " - "compressing."); - goto err; - } - - output.dst = *out = tor_realloc(*out, out_size); - output.size = out_size; - } - } - - *out_len = output.pos; - - if (tor_compress_is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tor instances would think " - "this is a compression bomb."); - goto err; - } - - if (stream != NULL) { - ZSTD_freeCStream(stream); - } - - return 0; - - err: - if (stream != NULL) { - ZSTD_freeCStream(stream); - } - - tor_free(*out); - return -1; -#else // HAVE_ZSTD. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - - return -1; -#endif // HAVE_ZSTD. -} - -/** Given a Zstandard compressed string of total length <b>in_len</b> bytes at - * <b>in</b>, uncompress them into a newly allocated buffer. Store the - * uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a failure; - * otherwise we decompress as much as we can. Warn about truncated or corrupt - * inputs at <b>protocol_warn_level</b>. - */ -int -tor_zstd_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ -#ifdef HAVE_ZSTD - ZSTD_DStream *stream = NULL; - size_t retval; - size_t out_size, old_size; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == ZSTD_METHOD); - - // FIXME(ahf): Handle this? - (void)complete_only; - (void)protocol_warn_level; - - *out = NULL; - - stream = ZSTD_createDStream(); - - if (stream == NULL) { - // Zstandard does not give us any useful error message to why this - // happened. See https://github.com/facebook/zstd/issues/398 - log_warn(LD_GENERAL, "Error while creating Zstandard stream"); - goto err; - } - - retval = ZSTD_initDStream(stream); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - out_size = in_len * 2; - if (out_size < 1024) - out_size = 1024; - - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - *out_len = 0; - - ZSTD_inBuffer input = { in, in_len, 0 }; - ZSTD_outBuffer output = { *out, out_size, 0 }; - - while (input.pos < input.size) { - retval = ZSTD_decompressStream(stream, &output, &input); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream decompression error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - if (input.pos < input.size && output.pos == output.size) { - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); - goto err; - } - - if (tor_compress_is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible Zstandard " - "compression bomb. Not proceeding."); - goto err; - } - - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " - "Zstandard data."); - goto err; - } - - if (out_size - output.pos > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " - "decompressing."); - goto err; - } - - output.dst = *out = tor_realloc(*out, out_size); - output.size = out_size; - } - } - - *out_len = output.pos; - - if (stream != NULL) { - ZSTD_freeDStream(stream); - } - - // NUL-terminate our output. - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - - err: - if (stream != NULL) { - ZSTD_freeDStream(stream); - } - - tor_free(*out); - return -1; -#else // HAVE_ZSTD. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - (void)complete_only; - (void)protocol_warn_level; - - return -1; -#endif // HAVE_ZSTD. -} - /** Internal Zstandard state for incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_zstd_compress_state_t { diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index 663cbdd..24e287c 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -17,16 +17,6 @@ const char *tor_zstd_get_version_str(void); const char *tor_zstd_get_header_version_str(void); -int tor_zstd_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); - -int tor_zstd_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - /** Internal state for an incremental Zstandard compression/decompression. */ typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; diff --git a/src/test/test_util.c b/src/test/test_util.c index 1e33de8..d47475a 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2377,7 +2377,7 @@ test_util_gzip_compression_bomb(void *arg) expect_single_log_msg_containing( "We compressed something and got an insanely high " "compression factor; other Tors would think this " - "was a zlib bomb."); + "was a compression bomb."); teardown_capture_of_logs(); /* Here's a compression bomb that we made manually. */