[tor-commits] [tor/master] Move zlib compression code into its own module.
nickm at torproject.org
nickm at torproject.org
Tue Apr 25 12:18:24 UTC 2017
commit 9d5bc1a9354637aa59025f61e577c6d42f8c53ba
Author: Alexander Færøy <ahf at torproject.org>
Date: Tue Apr 18 03:14:36 2017 +0200
Move zlib compression code into its own module.
This patch refactors the `torgzip` module to allow us to extend a common
compression API to support multiple compression backends.
Additionally we move the gzip/zlib code into its own module under the
name `compress_zlib`.
See https://bugs.torproject.org/21664
---
src/common/compress_zlib.c | 524 +++++++++++++++++++++++++++++++++++++++++++++
src/common/compress_zlib.h | 56 +++++
src/common/include.am | 2 +
src/common/torgzip.c | 458 ++++-----------------------------------
src/common/torgzip.h | 11 +-
src/or/config.c | 1 +
src/or/main.c | 1 +
src/or/relay.c | 1 +
8 files changed, 633 insertions(+), 421 deletions(-)
diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c
new file mode 100644
index 0000000..38e500c
--- /dev/null
+++ b/src/common/compress_zlib.c
@@ -0,0 +1,524 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.c
+ * \brief Compression backend for gzip and zlib.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "torgzip.h"
+#include "compress_zlib.h"
+
+/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of
+ saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
+ that nobody will care if the compile outputs a no-such-identifier warning.
+
+ Sorry, but we like -Werror over here, so I guess we need to define these.
+ I hope that zlib 1.2.6 doesn't break these too.
+*/
+#ifndef _LARGEFILE64_SOURCE
+#define _LARGEFILE64_SOURCE 0
+#endif
+#ifndef _LFS64_LARGEFILE
+#define _LFS64_LARGEFILE 0
+#endif
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 0
+#endif
+#ifndef off64_t
+#define off64_t int64_t
+#endif
+
+#include <zlib.h>
+
+#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
+#error "We require zlib version 1.2 or later."
+#endif
+
+static size_t tor_zlib_state_size_precalc(int inflate,
+ int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static size_t total_zlib_allocation = 0;
+
+/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
+static inline int
+method_bits(compress_method_t method, compression_level_t level)
+{
+ /* Bits+16 means "use gzip" in zlib >= 1.2 */
+ const int flag = method == GZIP_METHOD ? 16 : 0;
+ switch (level) {
+ default:
+ case HIGH_COMPRESSION: return flag + 15;
+ case MEDIUM_COMPRESSION: return flag + 13;
+ case LOW_COMPRESSION: return flag + 11;
+ }
+}
+
+/** Return a string representation of the version of the currently running
+ * version of zlib. */
+const char *
+tor_zlib_get_version_str(void)
+{
+ return zlibVersion();
+}
+
+/** Return a string representation of the version of the version of zlib
+* used at compilation. */
+const char *
+tor_zlib_get_header_version_str(void)
+{
+ return ZLIB_VERSION;
+}
+
+/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
+ * allocated buffer, using the method described in <b>method</b>. Store the
+ * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. Return
+ * 0 on success, -1 on failure.
+ */
+int
+tor_zlib_compress(char **out, size_t *out_len,
+ const char *in, size_t in_len,
+ compress_method_t method)
+{
+ struct z_stream_s *stream = NULL;
+ size_t out_size, old_size;
+ off_t offset;
+
+ tor_assert(out);
+ tor_assert(out_len);
+ tor_assert(in);
+ tor_assert(in_len < UINT_MAX);
+
+ *out = NULL;
+
+ stream = tor_malloc_zero(sizeof(struct z_stream_s));
+ stream->zalloc = Z_NULL;
+ stream->zfree = Z_NULL;
+ stream->opaque = NULL;
+ stream->next_in = (unsigned char*) in;
+ stream->avail_in = (unsigned int)in_len;
+
+ if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
+ method_bits(method, HIGH_COMPRESSION),
+ tor_compress_memory_level(HIGH_COMPRESSION),
+ Z_DEFAULT_STRATEGY) != Z_OK) {
+ //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
+ log_warn(LD_GENERAL, "Error from deflateInit2: %s",
+ stream->msg?stream->msg:"<no message>");
+ goto err;
+ //LCOV_EXCL_STOP
+ }
+
+ /* Guess 50% compression. */
+ out_size = in_len / 2;
+ if (out_size < 1024) out_size = 1024;
+ *out = tor_malloc(out_size);
+ stream->next_out = (unsigned char*)*out;
+ stream->avail_out = (unsigned int)out_size;
+
+ while (1) {
+ switch (deflate(stream, Z_FINISH))
+ {
+ case Z_STREAM_END:
+ goto done;
+ case Z_OK:
+ /* In case zlib doesn't work as I think .... */
+ if (stream->avail_out >= stream->avail_in+16)
+ break;
+ case Z_BUF_ERROR:
+ offset = stream->next_out - ((unsigned char*)*out);
+ old_size = out_size;
+ out_size *= 2;
+ if (out_size < old_size) {
+ log_warn(LD_GENERAL, "Size overflow in compression.");
+ goto err;
+ }
+ *out = tor_realloc(*out, out_size);
+ stream->next_out = (unsigned char*)(*out + offset);
+ if (out_size - offset > UINT_MAX) {
+ log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
+ "uncompressing.");
+ goto err;
+ }
+ stream->avail_out = (unsigned int)(out_size - offset);
+ break;
+ default:
+ log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
+ stream->msg ? stream->msg : "<no message>");
+ goto err;
+ }
+ }
+ done:
+ *out_len = stream->total_out;
+#if defined(OpenBSD)
+ /* "Hey Rocky! Watch me change an unsigned field to a signed field in a
+ * third-party API!"
+ * "Oh, that trick will just make people do unsafe casts to the unsigned
+ * type in their cross-platform code!"
+ * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure
+ * the newly unsigned field isn't negative." */
+ tor_assert(stream->total_out >= 0);
+#endif
+ if (deflateEnd(stream)!=Z_OK) {
+ // LCOV_EXCL_START -- unreachable if we handled the zlib structure right
+ tor_assert_nonfatal_unreached();
+ log_warn(LD_BUG, "Error freeing gzip structures");
+ goto err;
+ // LCOV_EXCL_STOP
+ }
+ tor_free(stream);
+
+ if (tor_compress_is_compression_bomb(*out_len, in_len)) {
+ log_warn(LD_BUG, "We compressed something and got an insanely high "
+ "compression factor; other Tors would think this was a zlib bomb.");
+ goto err;
+ }
+
+ return 0;
+ err:
+ if (stream) {
+ deflateEnd(stream);
+ tor_free(stream);
+ }
+ tor_free(*out);
+ return -1;
+}
+
+/** Given an Zlib/Gzip compressed string of total length <b>in_len</b> bytes
+ * at <b>in</b>, uncompress them into a newly allocated buffer. Store the
+ * uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>.
+ * Return 0 on success, -1 on failure.
+ *
+ * If <b>complete_only</b> is true, we consider a truncated input as a failure;
+ * otherwise we decompress as much as we can. Warn about truncated or corrupt
+ * inputs at <b>protocol_warn_level</b>.
+ */
+int
+tor_zlib_uncompress(char **out, size_t *out_len,
+ const char *in, size_t in_len,
+ compress_method_t method,
+ int complete_only,
+ int protocol_warn_level)
+{
+ struct z_stream_s *stream = NULL;
+ size_t out_size, old_size;
+ off_t offset;
+ int r;
+
+ tor_assert(out);
+ tor_assert(out_len);
+ tor_assert(in);
+ tor_assert(in_len < UINT_MAX);
+
+ *out = NULL;
+
+ stream = tor_malloc_zero(sizeof(struct z_stream_s));
+ stream->zalloc = Z_NULL;
+ stream->zfree = Z_NULL;
+ stream->opaque = NULL;
+ stream->next_in = (unsigned char*) in;
+ stream->avail_in = (unsigned int)in_len;
+
+ if (inflateInit2(stream,
+ method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
+ // LCOV_EXCL_START -- can only hit this if we give bad inputs.
+ log_warn(LD_GENERAL, "Error from inflateInit2: %s",
+ stream->msg?stream->msg:"<no message>");
+ goto err;
+ // LCOV_EXCL_STOP
+ }
+
+ out_size = in_len * 2; /* guess 50% compression. */
+ if (out_size < 1024) out_size = 1024;
+ if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
+ goto err;
+
+ *out = tor_malloc(out_size);
+ stream->next_out = (unsigned char*)*out;
+ stream->avail_out = (unsigned int)out_size;
+
+ while (1) {
+ switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
+ {
+ case Z_STREAM_END:
+ if (stream->avail_in == 0)
+ goto done;
+ /* There may be more compressed data here. */
+ if ((r = inflateEnd(stream)) != Z_OK) {
+ log_warn(LD_BUG, "Error freeing gzip structures");
+ goto err;
+ }
+ if (inflateInit2(stream,
+ method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
+ log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
+ stream->msg?stream->msg:"<no message>");
+ goto err;
+ }
+ break;
+ case Z_OK:
+ if (!complete_only && stream->avail_in == 0)
+ goto done;
+ /* In case zlib doesn't work as I think.... */
+ if (stream->avail_out >= stream->avail_in+16)
+ break;
+ case Z_BUF_ERROR:
+ if (stream->avail_out > 0) {
+ log_fn(protocol_warn_level, LD_PROTOCOL,
+ "possible truncated or corrupt zlib data");
+ goto err;
+ }
+ offset = stream->next_out - (unsigned char*)*out;
+ old_size = out_size;
+ out_size *= 2;
+ if (out_size < old_size) {
+ log_warn(LD_GENERAL, "Size overflow in uncompression.");
+ goto err;
+ }
+ if (tor_compress_is_compression_bomb(in_len, out_size)) {
+ log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
+ "not proceeding.");
+ goto err;
+ }
+ if (out_size >= SIZE_T_CEILING) {
+ log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
+ goto err;
+ }
+ *out = tor_realloc(*out, out_size);
+ stream->next_out = (unsigned char*)(*out + offset);
+ if (out_size - offset > UINT_MAX) {
+ log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
+ "uncompressing.");
+ goto err;
+ }
+ stream->avail_out = (unsigned int)(out_size - offset);
+ break;
+ default:
+ log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
+ stream->msg ? stream->msg : "<no message>");
+ goto err;
+ }
+ }
+ done:
+ *out_len = stream->next_out - (unsigned char*)*out;
+ r = inflateEnd(stream);
+ tor_free(stream);
+ if (r != Z_OK) {
+ log_warn(LD_BUG, "Error freeing gzip structures");
+ goto err;
+ }
+
+ /* NUL-terminate output. */
+ if (out_size == *out_len)
+ *out = tor_realloc(*out, out_size + 1);
+ (*out)[*out_len] = '\0';
+
+ return 0;
+ err:
+ if (stream) {
+ inflateEnd(stream);
+ tor_free(stream);
+ }
+ if (*out) {
+ tor_free(*out);
+ }
+ return -1;
+}
+
+/** Internal zlib state for an incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zlib_compress_state_t {
+ struct z_stream_s stream; /**< The zlib stream */
+ int compress; /**< True if we are compressing; false if we are inflating */
+
+ /** Number of bytes read so far. Used to detect zlib bombs. */
+ size_t input_so_far;
+ /** Number of bytes written so far. Used to detect zlib bombs. */
+ size_t output_so_far;
+
+ /** Approximate number of bytes allocated for this object. */
+ size_t allocation;
+};
+
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
+{
+ windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+ if (inflate_) {
+ /* From zconf.h:
+
+ "The memory requirements for inflate are (in bytes) 1 << windowBits
+ that is, 32K for windowBits=15 (default value) plus a few kilobytes
+ for small objects."
+ */
+ return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+ (1 << 15) + A_FEW_KILOBYTES;
+ } else {
+ /* Also from zconf.h:
+
+ "The memory requirements for deflate are (in bytes):
+ (1 << (windowBits+2)) + (1 << (memLevel+9))
+ ... plus a few kilobytes for small objects."
+ */
+ return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+ (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+ }
+#undef A_FEW_KILOBYTES
+}
+
+/** Construct and return a tor_zlib_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress,
+ compress_method_t method,
+ compression_level_t compression_level)
+{
+ tor_zlib_compress_state_t *out;
+ int bits, memlevel;
+
+ if (! compress) {
+ /* use this setting for decompression, since we might have the
+ * max number of window bits */
+ compression_level = HIGH_COMPRESSION;
+ }
+
+ out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t));
+ out->stream.zalloc = Z_NULL;
+ out->stream.zfree = Z_NULL;
+ out->stream.opaque = NULL;
+ out->compress = compress;
+ bits = method_bits(method, compression_level);
+ memlevel = tor_compress_memory_level(compression_level);
+ if (compress) {
+ if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
+ bits, memlevel,
+ Z_DEFAULT_STRATEGY) != Z_OK)
+ goto err; // LCOV_EXCL_LINE
+ } else {
+ if (inflateInit2(&out->stream, bits) != Z_OK)
+ goto err; // LCOV_EXCL_LINE
+ }
+ out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel);
+
+ total_zlib_allocation += out->allocation;
+
+ return out;
+
+ err:
+ tor_free(out);
+ return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>. Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+ char **out, size_t *out_len,
+ const char **in, size_t *in_len,
+ int finish)
+{
+ int err;
+ tor_assert(state != NULL);
+ tor_assert(*in_len <= UINT_MAX);
+ tor_assert(*out_len <= UINT_MAX);
+ state->stream.next_in = (unsigned char*) *in;
+ state->stream.avail_in = (unsigned int)*in_len;
+ state->stream.next_out = (unsigned char*) *out;
+ state->stream.avail_out = (unsigned int)*out_len;
+
+ if (state->compress) {
+ err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
+ } else {
+ err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
+ }
+
+ state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
+ state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
+
+ *out = (char*) state->stream.next_out;
+ *out_len = state->stream.avail_out;
+ *in = (const char *) state->stream.next_in;
+ *in_len = state->stream.avail_in;
+
+ if (! state->compress &&
+ tor_compress_is_compression_bomb(state->input_so_far,
+ state->output_so_far)) {
+ log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
+ return TOR_COMPRESS_ERROR;
+ }
+
+ switch (err)
+ {
+ case Z_STREAM_END:
+ return TOR_COMPRESS_DONE;
+ case Z_BUF_ERROR:
+ if (state->stream.avail_in == 0 && !finish)
+ return TOR_COMPRESS_OK;
+ return TOR_COMPRESS_BUFFER_FULL;
+ case Z_OK:
+ if (state->stream.avail_out == 0 || finish)
+ return TOR_COMPRESS_BUFFER_FULL;
+ return TOR_COMPRESS_OK;
+ default:
+ log_warn(LD_GENERAL, "Gzip returned an error: %s",
+ state->stream.msg ? state->stream.msg : "<no message>");
+ return TOR_COMPRESS_ERROR;
+ }
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zlib_compress_free(tor_zlib_compress_state_t *state)
+{
+ if (state == NULL)
+ return;
+
+ total_zlib_allocation -= state->allocation;
+
+ if (state->compress)
+ deflateEnd(&state->stream);
+ else
+ inflateEnd(&state->stream);
+
+ tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state)
+{
+ tor_assert(state != NULL);
+ return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+ return total_zlib_allocation;
+}
+
diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h
new file mode 100644
index 0000000..0862678
--- /dev/null
+++ b/src/common/compress_zlib.h
@@ -0,0 +1,56 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.h
+ * \brief Header for compress_zlib.c
+ **/
+
+#ifndef TOR_COMPRESS_ZLIB_H
+#define TOR_COMPRESS_ZLIB_H
+
+const char *
+tor_zlib_get_version_str(void);
+
+const char *
+tor_zlib_get_header_version_str(void);
+
+int
+tor_zlib_compress(char **out, size_t *out_len,
+ const char *in, size_t in_len,
+ compress_method_t method);
+
+int
+tor_zlib_uncompress(char **out, size_t *out_len,
+ const char *in, size_t in_len,
+ compress_method_t method,
+ int complete_only,
+ int protocol_warn_level);
+
+/** Internal state for an incremental zlib/gzip compression/decompression. */
+typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t;
+
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress,
+ compress_method_t method,
+ compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+ char **out, size_t *out_len,
+ const char **in, size_t *in_len,
+ int finish);
+
+void
+tor_zlib_compress_free(tor_zlib_compress_state_t *state);
+
+size_t
+tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state);
+
+size_t
+tor_zlib_get_total_allocation(void);
+
+#endif // TOR_COMPRESS_ZLIB_H.
+
diff --git a/src/common/include.am b/src/common/include.am
index d6504c7..ea2c46a 100644
--- a/src/common/include.am
+++ b/src/common/include.am
@@ -105,6 +105,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
LIBOR_CRYPTO_A_SRC = \
src/common/aes.c \
+ src/common/compress_zlib.c \
src/common/crypto.c \
src/common/crypto_pwbox.c \
src/common/crypto_s2k.c \
@@ -145,6 +146,7 @@ COMMONHEADERS = \
src/common/compat_openssl.h \
src/common/compat_threads.h \
src/common/compat_time.h \
+ src/common/compress_zlib.h \
src/common/confline.h \
src/common/container.h \
src/common/crypto.h \
diff --git a/src/common/torgzip.c b/src/common/torgzip.c
index d6f6225..d760cd9 100644
--- a/src/common/torgzip.c
+++ b/src/common/torgzip.c
@@ -23,69 +23,7 @@
#include "util.h"
#include "torlog.h"
#include "torgzip.h"
-
-/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of
- saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
- that nobody will care if the compile outputs a no-such-identifier warning.
-
- Sorry, but we like -Werror over here, so I guess we need to define these.
- I hope that zlib 1.2.6 doesn't break these too.
-*/
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE 0
-#endif
-#ifndef _LFS64_LARGEFILE
-#define _LFS64_LARGEFILE 0
-#endif
-#ifndef _FILE_OFFSET_BITS
-#define _FILE_OFFSET_BITS 0
-#endif
-#ifndef off64_t
-#define off64_t int64_t
-#endif
-
-#include <zlib.h>
-
-#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
-#error "We require zlib version 1.2 or later."
-#endif
-
-static size_t tor_zlib_state_size_precalc(int inflate,
- int windowbits, int memlevel);
-
-/** Total number of bytes allocated for zlib state */
-static size_t total_zlib_allocation = 0;
-
-/** Return a string representation of the version of the currently running
- * version of zlib. */
-const char *
-tor_zlib_get_version_str(void)
-{
- return zlibVersion();
-}
-
-/** Return a string representation of the version of the version of zlib
-* used at compilation. */
-const char *
-tor_zlib_get_header_version_str(void)
-{
- return ZLIB_VERSION;
-}
-
-/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
-static inline int
-method_bits(compress_method_t method, compression_level_t level)
-{
- /* Bits+16 means "use gzip" in zlib >= 1.2 */
- const int flag = method == GZIP_METHOD ? 16 : 0;
- switch (level) {
- default:
- case HIGH_COMPRESSION: return flag + 15;
- case MEDIUM_COMPRESSION: return flag + 13;
- case LOW_COMPRESSION: return flag + 11;
- }
-}
-
+#include "compress_zlib.h"
/** @{ */
/* These macros define the maximum allowable compression factor. Anything of
@@ -140,107 +78,9 @@ tor_compress(char **out, size_t *out_len,
const char *in, size_t in_len,
compress_method_t method)
{
- struct z_stream_s *stream = NULL;
- size_t out_size, old_size;
- off_t offset;
-
- tor_assert(out);
- tor_assert(out_len);
- tor_assert(in);
- tor_assert(in_len < UINT_MAX);
-
- *out = NULL;
-
- stream = tor_malloc_zero(sizeof(struct z_stream_s));
- stream->zalloc = Z_NULL;
- stream->zfree = Z_NULL;
- stream->opaque = NULL;
- stream->next_in = (unsigned char*) in;
- stream->avail_in = (unsigned int)in_len;
-
- if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
- method_bits(method, HIGH_COMPRESSION),
- get_memlevel(HIGH_COMPRESSION),
- Z_DEFAULT_STRATEGY) != Z_OK) {
- //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
- log_warn(LD_GENERAL, "Error from deflateInit2: %s",
- stream->msg?stream->msg:"<no message>");
- goto err;
- //LCOV_EXCL_STOP
- }
-
- /* Guess 50% compression. */
- out_size = in_len / 2;
- if (out_size < 1024) out_size = 1024;
- *out = tor_malloc(out_size);
- stream->next_out = (unsigned char*)*out;
- stream->avail_out = (unsigned int)out_size;
-
- while (1) {
- switch (deflate(stream, Z_FINISH))
- {
- case Z_STREAM_END:
- goto done;
- case Z_OK:
- /* In case zlib doesn't work as I think .... */
- if (stream->avail_out >= stream->avail_in+16)
- break;
- case Z_BUF_ERROR:
- offset = stream->next_out - ((unsigned char*)*out);
- old_size = out_size;
- out_size *= 2;
- if (out_size < old_size) {
- log_warn(LD_GENERAL, "Size overflow in compression.");
- goto err;
- }
- *out = tor_realloc(*out, out_size);
- stream->next_out = (unsigned char*)(*out + offset);
- if (out_size - offset > UINT_MAX) {
- log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
- "uncompressing.");
- goto err;
- }
- stream->avail_out = (unsigned int)(out_size - offset);
- break;
- default:
- log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
- stream->msg ? stream->msg : "<no message>");
- goto err;
- }
- }
- done:
- *out_len = stream->total_out;
-#if defined(OpenBSD)
- /* "Hey Rocky! Watch me change an unsigned field to a signed field in a
- * third-party API!"
- * "Oh, that trick will just make people do unsafe casts to the unsigned
- * type in their cross-platform code!"
- * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure
- * the newly unsigned field isn't negative." */
- tor_assert(stream->total_out >= 0);
-#endif
- if (deflateEnd(stream)!=Z_OK) {
- // LCOV_EXCL_START -- unreachable if we handled the zlib structure right
- tor_assert_nonfatal_unreached();
- log_warn(LD_BUG, "Error freeing gzip structures");
- goto err;
- // LCOV_EXCL_STOP
- }
- tor_free(stream);
+ if (method == GZIP_METHOD || method == ZLIB_METHOD)
+ return tor_zlib_compress(out, out_len, in, in_len, method);
- if (is_compression_bomb(*out_len, in_len)) {
- log_warn(LD_BUG, "We compressed something and got an insanely high "
- "compression factor; other Tors would think this was a zlib bomb.");
- goto err;
- }
-
- return 0;
- err:
- if (stream) {
- deflateEnd(stream);
- tor_free(stream);
- }
- tor_free(*out);
return -1;
}
@@ -262,127 +102,12 @@ tor_uncompress(char **out, size_t *out_len,
int complete_only,
int protocol_warn_level)
{
- struct z_stream_s *stream = NULL;
- size_t out_size, old_size;
- off_t offset;
- int r;
-
- tor_assert(out);
- tor_assert(out_len);
- tor_assert(in);
- tor_assert(in_len < UINT_MAX);
-
- *out = NULL;
-
- stream = tor_malloc_zero(sizeof(struct z_stream_s));
- stream->zalloc = Z_NULL;
- stream->zfree = Z_NULL;
- stream->opaque = NULL;
- stream->next_in = (unsigned char*) in;
- stream->avail_in = (unsigned int)in_len;
-
- if (inflateInit2(stream,
- method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
- // LCOV_EXCL_START -- can only hit this if we give bad inputs.
- log_warn(LD_GENERAL, "Error from inflateInit2: %s",
- stream->msg?stream->msg:"<no message>");
- goto err;
- // LCOV_EXCL_STOP
- }
+ if (method == GZIP_METHOD || method == ZLIB_METHOD)
+ return tor_zlib_uncompress(out, out_len, in, in_len,
+ method,
+ complete_only,
+ protocol_warn_level);
- out_size = in_len * 2; /* guess 50% compression. */
- if (out_size < 1024) out_size = 1024;
- if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
- goto err;
-
- *out = tor_malloc(out_size);
- stream->next_out = (unsigned char*)*out;
- stream->avail_out = (unsigned int)out_size;
-
- while (1) {
- switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
- {
- case Z_STREAM_END:
- if (stream->avail_in == 0)
- goto done;
- /* There may be more compressed data here. */
- if ((r = inflateEnd(stream)) != Z_OK) {
- log_warn(LD_BUG, "Error freeing gzip structures");
- goto err;
- }
- if (inflateInit2(stream,
- method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
- log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
- stream->msg?stream->msg:"<no message>");
- goto err;
- }
- break;
- case Z_OK:
- if (!complete_only && stream->avail_in == 0)
- goto done;
- /* In case zlib doesn't work as I think.... */
- if (stream->avail_out >= stream->avail_in+16)
- break;
- case Z_BUF_ERROR:
- if (stream->avail_out > 0) {
- log_fn(protocol_warn_level, LD_PROTOCOL,
- "possible truncated or corrupt zlib data");
- goto err;
- }
- offset = stream->next_out - (unsigned char*)*out;
- old_size = out_size;
- out_size *= 2;
- if (out_size < old_size) {
- log_warn(LD_GENERAL, "Size overflow in uncompression.");
- goto err;
- }
- if (is_compression_bomb(in_len, out_size)) {
- log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
- "not proceeding.");
- goto err;
- }
- if (out_size >= SIZE_T_CEILING) {
- log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
- goto err;
- }
- *out = tor_realloc(*out, out_size);
- stream->next_out = (unsigned char*)(*out + offset);
- if (out_size - offset > UINT_MAX) {
- log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
- "uncompressing.");
- goto err;
- }
- stream->avail_out = (unsigned int)(out_size - offset);
- break;
- default:
- log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
- stream->msg ? stream->msg : "<no message>");
- goto err;
- }
- }
- done:
- *out_len = stream->next_out - (unsigned char*)*out;
- r = inflateEnd(stream);
- tor_free(stream);
- if (r != Z_OK) {
- log_warn(LD_BUG, "Error freeing gzip structures");
- goto err;
- }
-
- /* NUL-terminate output. */
- if (out_size == *out_len)
- *out = tor_realloc(*out, out_size + 1);
- (*out)[*out_len] = '\0';
-
- return 0;
- err:
- if (stream) {
- inflateEnd(stream);
- tor_free(stream);
- }
- if (*out) {
- tor_free(*out);
- }
return -1;
}
@@ -406,58 +131,39 @@ detect_compression_method(const char *in, size_t in_len)
/** Internal state for an incremental compression/decompression. The body of
* this struct is not exposed. */
struct tor_compress_state_t {
- struct z_stream_s stream; /**< The zlib stream */
- int compress; /**< True if we are compressing; false if we are inflating */
+ compress_method_t method; /**< The compression method. */
- /** Number of bytes read so far. Used to detect zlib bombs. */
- size_t input_so_far;
- /** Number of bytes written so far. Used to detect zlib bombs. */
- size_t output_so_far;
-
- /** Approximate number of bytes allocated for this object. */
- size_t allocation;
+ union {
+ tor_zlib_compress_state_t *zlib_state;
+ } u; /**< Compression backend state. */
};
/** Construct and return a tor_compress_state_t object using <b>method</b>. If
* <b>compress</b>, it's for compression; otherwise it's for decompression. */
tor_compress_state_t *
-tor_compress_new(int compress_, compress_method_t method,
+tor_compress_new(int compress, compress_method_t method,
compression_level_t compression_level)
{
- tor_compress_state_t *out;
- int bits, memlevel;
-
- if (! compress_) {
- /* use this setting for decompression, since we might have the
- * max number of window bits */
- compression_level = HIGH_COMPRESSION;
- }
-
- out = tor_malloc_zero(sizeof(tor_compress_state_t));
- out->stream.zalloc = Z_NULL;
- out->stream.zfree = Z_NULL;
- out->stream.opaque = NULL;
- out->compress = compress_;
- bits = method_bits(method, compression_level);
- memlevel = get_memlevel(compression_level);
- if (compress_) {
- if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
- bits, memlevel,
- Z_DEFAULT_STRATEGY) != Z_OK)
- goto err; // LCOV_EXCL_LINE
- } else {
- if (inflateInit2(&out->stream, bits) != Z_OK)
- goto err; // LCOV_EXCL_LINE
- }
- out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
-
- total_zlib_allocation += out->allocation;
-
- return out;
+ tor_compress_state_t *state;
+
+ state = tor_malloc_zero(sizeof(tor_compress_state_t));
+ state->method = method;
+
+ if (method == GZIP_METHOD || method == ZLIB_METHOD) {
+ tor_zlib_compress_state_t *zlib_state =
+ tor_zlib_compress_new(compress, method, compression_level);
+
+ if (zlib_state == NULL)
+ goto err;
+
+ state->u.zlib_state = zlib_state;
+ }
+
+ return state;
err:
- tor_free(out);
- return NULL;
+ tor_free(state);
+ return NULL;
}
/** Compress/decompress some bytes using <b>state</b>. Read up to
@@ -477,112 +183,38 @@ tor_compress_process(tor_compress_state_t *state,
const char **in, size_t *in_len,
int finish)
{
- int err;
- tor_assert(*in_len <= UINT_MAX);
- tor_assert(*out_len <= UINT_MAX);
- state->stream.next_in = (unsigned char*) *in;
- state->stream.avail_in = (unsigned int)*in_len;
- state->stream.next_out = (unsigned char*) *out;
- state->stream.avail_out = (unsigned int)*out_len;
-
- if (state->compress) {
- err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
- } else {
- err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
- }
-
- state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
- state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
+ tor_assert(state != NULL);
- *out = (char*) state->stream.next_out;
- *out_len = state->stream.avail_out;
- *in = (const char *) state->stream.next_in;
- *in_len = state->stream.avail_in;
+ if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
+ return tor_zlib_compress_process(state->u.zlib_state,
+ out, out_len, in, in_len,
+ finish);
- if (! state->compress &&
- is_compression_bomb(state->input_so_far, state->output_so_far)) {
- log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
- return TOR_COMPRESS_ERROR;
- }
-
- switch (err)
- {
- case Z_STREAM_END:
- return TOR_COMPRESS_DONE;
- case Z_BUF_ERROR:
- if (state->stream.avail_in == 0 && !finish)
- return TOR_COMPRESS_OK;
- return TOR_COMPRESS_BUFFER_FULL;
- case Z_OK:
- if (state->stream.avail_out == 0 || finish)
- return TOR_COMPRESS_BUFFER_FULL;
- return TOR_COMPRESS_OK;
- default:
- log_warn(LD_GENERAL, "Gzip returned an error: %s",
- state->stream.msg ? state->stream.msg : "<no message>");
- return TOR_COMPRESS_ERROR;
- }
+ return TOR_COMPRESS_ERROR;
}
/** Deallocate <b>state</b>. */
void
tor_compress_free(tor_compress_state_t *state)
{
- if (!state)
+ if (state == NULL)
return;
- total_zlib_allocation -= state->allocation;
-
- if (state->compress)
- deflateEnd(&state->stream);
- else
- inflateEnd(&state->stream);
+ if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
+ tor_zlib_compress_free(state->u.zlib_state);
tor_free(state);
}
-/** Return an approximate number of bytes used in RAM to hold a state with
- * window bits <b>windowBits</b> and compression level 'memlevel' */
-static size_t
-tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
-{
- windowbits &= 15;
-
-#define A_FEW_KILOBYTES 2048
-
- if (inflate_) {
- /* From zconf.h:
-
- "The memory requirements for inflate are (in bytes) 1 << windowBits
- that is, 32K for windowBits=15 (default value) plus a few kilobytes
- for small objects."
- */
- return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) +
- (1 << 15) + A_FEW_KILOBYTES;
- } else {
- /* Also from zconf.h:
-
- "The memory requirements for deflate are (in bytes):
- (1 << (windowBits+2)) + (1 << (memLevel+9))
- ... plus a few kilobytes for small objects."
- */
- return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) +
- (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
- }
-#undef A_FEW_KILOBYTES
-}
-
/** Return the approximate number of bytes allocated for <b>state</b>. */
size_t
tor_compress_state_size(const tor_compress_state_t *state)
{
- return state->allocation;
-}
+ tor_assert(state != NULL);
-/** Return the approximate number of bytes allocated for all zlib states. */
-size_t
-tor_zlib_get_total_allocation(void)
-{
- return total_zlib_allocation;
+ if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
+ return tor_zlib_compress_state_size(state->u.zlib_state);
+
+ return 0;
}
diff --git a/src/common/torgzip.h b/src/common/torgzip.h
index fa56628..0fcac0c 100644
--- a/src/common/torgzip.h
+++ b/src/common/torgzip.h
@@ -38,12 +38,6 @@ tor_uncompress(char **out, size_t *out_len,
int complete_only,
int protocol_warn_level);
-const char *
-tor_zlib_get_version_str(void);
-
-const char *
-tor_zlib_get_header_version_str(void);
-
compress_method_t detect_compression_method(const char *in, size_t in_len);
int
@@ -60,8 +54,10 @@ typedef enum {
TOR_COMPRESS_BUFFER_FULL,
TOR_COMPRESS_ERROR
} tor_compress_output_t;
-/** Internal state for an incremental zlib compression/decompression. */
+
+/** Internal state for an incremental compression/decompression. */
typedef struct tor_compress_state_t tor_compress_state_t;
+
tor_compress_state_t *tor_compress_new(int compress,
compress_method_t method,
compression_level_t level);
@@ -73,7 +69,6 @@ tor_compress_output_t tor_compress_process(tor_compress_state_t *state,
void tor_compress_free(tor_compress_state_t *state);
size_t tor_compress_state_size(const tor_compress_state_t *state);
-size_t tor_zlib_get_total_allocation(void);
#endif
diff --git a/src/or/config.c b/src/or/config.c
index 809ff49..83e5f21 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -69,6 +69,7 @@
#include "circuitmux.h"
#include "circuitmux_ewma.h"
#include "circuitstats.h"
+#include "compress_zlib.h"
#include "config.h"
#include "connection.h"
#include "connection_edge.h"
diff --git a/src/or/main.c b/src/or/main.c
index 4505879..ddd7b82 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -58,6 +58,7 @@
#include "circuitlist.h"
#include "circuituse.h"
#include "command.h"
+#include "compress_zlib.h"
#include "config.h"
#include "confparse.h"
#include "connection.h"
diff --git a/src/or/relay.c b/src/or/relay.c
index 5139036..8a98c50 100644
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -54,6 +54,7 @@
#include "circuitbuild.h"
#include "circuitlist.h"
#include "circuituse.h"
+#include "compress_zlib.h"
#include "config.h"
#include "connection.h"
#include "connection_edge.h"
More information about the tor-commits
mailing list