commit 9d5bc1a9354637aa59025f61e577c6d42f8c53ba Author: Alexander Færøy ahf@torproject.org Date: Tue Apr 18 03:14:36 2017 +0200
Move zlib compression code into its own module.
This patch refactors the `torgzip` module to allow us to extend a common compression API to support multiple compression backends.
Additionally we move the gzip/zlib code into its own module under the name `compress_zlib`.
See https://bugs.torproject.org/21664 --- src/common/compress_zlib.c | 524 +++++++++++++++++++++++++++++++++++++++++++++ src/common/compress_zlib.h | 56 +++++ src/common/include.am | 2 + src/common/torgzip.c | 458 ++++----------------------------------- src/common/torgzip.h | 11 +- src/or/config.c | 1 + src/or/main.c | 1 + src/or/relay.c | 1 + 8 files changed, 633 insertions(+), 421 deletions(-)
diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000..38e500c --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,524 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "torgzip.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static size_t total_zlib_allocation = 0; + +/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly + * allocated buffer, using the method described in <b>method</b>. Store the + * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. Return + * 0 on success, -1 on failure. + */ +int +tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + struct z_stream_s *stream = NULL; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + + *out = NULL; + + stream = tor_malloc_zero(sizeof(struct z_stream_s)); + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = NULL; + stream->next_in = (unsigned char*) in; + stream->avail_in = (unsigned int)in_len; + + if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, + method_bits(method, HIGH_COMPRESSION), + tor_compress_memory_level(HIGH_COMPRESSION), + Z_DEFAULT_STRATEGY) != Z_OK) { + //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. + log_warn(LD_GENERAL, "Error from deflateInit2: %s", + stream->msg?stream->msg:"<no message>"); + goto err; + //LCOV_EXCL_STOP + } + + /* Guess 50% compression. */ + out_size = in_len / 2; + if (out_size < 1024) out_size = 1024; + *out = tor_malloc(out_size); + stream->next_out = (unsigned char*)*out; + stream->avail_out = (unsigned int)out_size; + + while (1) { + switch (deflate(stream, Z_FINISH)) + { + case Z_STREAM_END: + goto done; + case Z_OK: + /* In case zlib doesn't work as I think .... */ + if (stream->avail_out >= stream->avail_in+16) + break; + case Z_BUF_ERROR: + offset = stream->next_out - ((unsigned char*)*out); + old_size = out_size; + out_size *= 2; + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in compression."); + goto err; + } + *out = tor_realloc(*out, out_size); + stream->next_out = (unsigned char*)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " + "uncompressing."); + goto err; + } + stream->avail_out = (unsigned int)(out_size - offset); + break; + default: + log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", + stream->msg ? stream->msg : "<no message>"); + goto err; + } + } + done: + *out_len = stream->total_out; +#if defined(OpenBSD) + /* "Hey Rocky! Watch me change an unsigned field to a signed field in a + * third-party API!" + * "Oh, that trick will just make people do unsafe casts to the unsigned + * type in their cross-platform code!" + * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure + * the newly unsigned field isn't negative." */ + tor_assert(stream->total_out >= 0); +#endif + if (deflateEnd(stream)!=Z_OK) { + // LCOV_EXCL_START -- unreachable if we handled the zlib structure right + tor_assert_nonfatal_unreached(); + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + // LCOV_EXCL_STOP + } + tor_free(stream); + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a zlib bomb."); + goto err; + } + + return 0; + err: + if (stream) { + deflateEnd(stream); + tor_free(stream); + } + tor_free(*out); + return -1; +} + +/** Given an Zlib/Gzip compressed string of total length <b>in_len</b> bytes + * at <b>in</b>, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>. + * Return 0 on success, -1 on failure. + * + * If <b>complete_only</b> is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at <b>protocol_warn_level</b>. + */ +int +tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + struct z_stream_s *stream = NULL; + size_t out_size, old_size; + off_t offset; + int r; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + + *out = NULL; + + stream = tor_malloc_zero(sizeof(struct z_stream_s)); + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = NULL; + stream->next_in = (unsigned char*) in; + stream->avail_in = (unsigned int)in_len; + + if (inflateInit2(stream, + method_bits(method, HIGH_COMPRESSION)) != Z_OK) { + // LCOV_EXCL_START -- can only hit this if we give bad inputs. + log_warn(LD_GENERAL, "Error from inflateInit2: %s", + stream->msg?stream->msg:"<no message>"); + goto err; + // LCOV_EXCL_STOP + } + + out_size = in_len * 2; /* guess 50% compression. */ + if (out_size < 1024) out_size = 1024; + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + stream->next_out = (unsigned char*)*out; + stream->avail_out = (unsigned int)out_size; + + while (1) { + switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) + { + case Z_STREAM_END: + if (stream->avail_in == 0) + goto done; + /* There may be more compressed data here. */ + if ((r = inflateEnd(stream)) != Z_OK) { + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + } + if (inflateInit2(stream, + method_bits(method,HIGH_COMPRESSION)) != Z_OK) { + log_warn(LD_GENERAL, "Error from second inflateInit2: %s", + stream->msg?stream->msg:"<no message>"); + goto err; + } + break; + case Z_OK: + if (!complete_only && stream->avail_in == 0) + goto done; + /* In case zlib doesn't work as I think.... */ + if (stream->avail_out >= stream->avail_in+16) + break; + case Z_BUF_ERROR: + if (stream->avail_out > 0) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "possible truncated or corrupt zlib data"); + goto err; + } + offset = stream->next_out - (unsigned char*)*out; + old_size = out_size; + out_size *= 2; + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in uncompression."); + goto err; + } + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " + "not proceeding."); + goto err; + } + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); + goto err; + } + *out = tor_realloc(*out, out_size); + stream->next_out = (unsigned char*)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " + "uncompressing."); + goto err; + } + stream->avail_out = (unsigned int)(out_size - offset); + break; + default: + log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", + stream->msg ? stream->msg : "<no message>"); + goto err; + } + } + done: + *out_len = stream->next_out - (unsigned char*)*out; + r = inflateEnd(stream); + tor_free(stream); + if (r != Z_OK) { + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + } + + /* NUL-terminate output. */ + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + err: + if (stream) { + inflateEnd(stream); + tor_free(stream); + } + if (*out) { + tor_free(*out); + } + return -1; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = HIGH_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress; + bits = method_bits(method, compression_level); + memlevel = tor_compress_memory_level(compression_level); + if (compress) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel); + + total_zlib_allocation += out->allocation; + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate <b>state</b>. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + total_zlib_allocation -= state->allocation; + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return total_zlib_allocation; +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000..0862678 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +const char * +tor_zlib_get_version_str(void); + +const char * +tor_zlib_get_header_version_str(void); + +int +tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t +tor_zlib_get_total_allocation(void); + +#endif // TOR_COMPRESS_ZLIB_H. + diff --git a/src/common/include.am b/src/common/include.am index d6504c7..ea2c46a 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -105,6 +105,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ @@ -145,6 +146,7 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ diff --git a/src/common/torgzip.c b/src/common/torgzip.c index d6f6225..d760cd9 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -23,69 +23,7 @@ #include "util.h" #include "torlog.h" #include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ -static inline int -method_bits(compress_method_t method, compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - +#include "compress_zlib.h"
/** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -140,107 +78,9 @@ tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method) { - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_compress(out, out_len, in, in_len, method);
- if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); return -1; }
@@ -262,127 +102,12 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level) { - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level);
- out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } return -1; }
@@ -406,58 +131,39 @@ detect_compression_method(const char *in, size_t in_len) /** Internal state for an incremental compression/decompression. The body of * this struct is not exposed. */ struct tor_compress_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ + compress_method_t method; /**< The compression method. */
- /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; + union { + tor_zlib_compress_state_t *zlib_state; + } u; /**< Compression backend state. */ };
/** Construct and return a tor_compress_state_t object using <b>method</b>. If * <b>compress</b>, it's for compression; otherwise it's for decompression. */ tor_compress_state_t * -tor_compress_new(int compress_, compress_method_t method, +tor_compress_new(int compress, compress_method_t method, compression_level_t compression_level) { - tor_compress_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_compress_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + if (method == GZIP_METHOD || method == ZLIB_METHOD) { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + } + + return state;
err: - tor_free(out); - return NULL; + tor_free(state); + return NULL; }
/** Compress/decompress some bytes using <b>state</b>. Read up to @@ -477,112 +183,38 @@ tor_compress_process(tor_compress_state_t *state, const char **in, size_t *in_len, int finish) { - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + tor_assert(state != NULL);
- *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish);
- if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_COMPRESS_ERROR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_COMPRESS_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_COMPRESS_OK; - return TOR_COMPRESS_BUFFER_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_COMPRESS_BUFFER_FULL; - return TOR_COMPRESS_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_COMPRESS_ERROR; - } + return TOR_COMPRESS_ERROR; }
/** Deallocate <b>state</b>. */ void tor_compress_free(tor_compress_state_t *state) { - if (!state) + if (state == NULL) return;
- total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + tor_zlib_compress_free(state->u.zlib_state);
tor_free(state); }
-/** Return an approximate number of bytes used in RAM to hold a state with - * window bits <b>windowBits</b> and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - /** Return the approximate number of bytes allocated for <b>state</b>. */ size_t tor_compress_state_size(const tor_compress_state_t *state) { - return state->allocation; -} + tor_assert(state != NULL);
-/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_state_size(state->u.zlib_state); + + return 0; }
diff --git a/src/common/torgzip.h b/src/common/torgzip.h index fa56628..0fcac0c 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -38,12 +38,6 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level);
-const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - compress_method_t detect_compression_method(const char *in, size_t in_len);
int @@ -60,8 +54,10 @@ typedef enum { TOR_COMPRESS_BUFFER_FULL, TOR_COMPRESS_ERROR } tor_compress_output_t; -/** Internal state for an incremental zlib compression/decompression. */ + +/** Internal state for an incremental compression/decompression. */ typedef struct tor_compress_state_t tor_compress_state_t; + tor_compress_state_t *tor_compress_new(int compress, compress_method_t method, compression_level_t level); @@ -73,7 +69,6 @@ tor_compress_output_t tor_compress_process(tor_compress_state_t *state, void tor_compress_free(tor_compress_state_t *state);
size_t tor_compress_state_size(const tor_compress_state_t *state); -size_t tor_zlib_get_total_allocation(void);
#endif
diff --git a/src/or/config.c b/src/or/config.c index 809ff49..83e5f21 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress_zlib.h" #include "config.h" #include "connection.h" #include "connection_edge.h" diff --git a/src/or/main.c b/src/or/main.c index 4505879..ddd7b82 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,6 +58,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress_zlib.h" #include "config.h" #include "confparse.h" #include "connection.h" diff --git a/src/or/relay.c b/src/or/relay.c index 5139036..8a98c50 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,6 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "compress_zlib.h" #include "config.h" #include "connection.h" #include "connection_edge.h"