tor-commits April 2017

tor-commits@lists.torproject.org

19 participants
966 discussions

[tor/master] Rename `zlib_compression_level_t` to `compression_level_t`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit e8b025dfc30ef0008cb44feb56ab3c740d2bae9b Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon Apr 17 14:22:13 2017 +0200 Rename `zlib_compression_level_t` to `compression_level_t`. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 6 +++--- src/common/torgzip.h | 4 ++-- src/or/directory.c | 2 +- src/or/directory.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 223cd30..a6909ce 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -74,7 +74,7 @@ tor_zlib_get_header_version_str(void) /** Return the 'bits' value to tell zlib to use method.*/ static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) +method_bits(compress_method_t method, compression_level_t level) { /* Bits+16 means "use gzip" in zlib >= 1.2 */ const int flag = method == GZIP_METHOD ? 16 : 0; @@ -87,7 +87,7 @@ method_bits(compress_method_t method, zlib_compression_level_t level) } static inline int -get_memlevel(zlib_compression_level_t level) +get_memlevel(compression_level_t level) { switch (level) { default: @@ -419,7 +419,7 @@ struct tor_zlib_state_t { * decompression. */ tor_zlib_state_t * tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) + compression_level_t compression_level) { tor_zlib_state_t *out; int bits, memlevel; diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 42b45ab..cbcbb99 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -25,7 +25,7 @@ typedef enum { **/ typedef enum { HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; +} compression_level_t; int tor_gzip_compress(char **out, size_t *out_len, @@ -54,7 +54,7 @@ typedef enum { /** Internal state for an incremental zlib compression/decompression. */ typedef struct tor_zlib_state_t tor_zlib_state_t; tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); + compression_level_t level); tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, char **out, size_t *out_len, diff --git a/src/or/directory.c b/src/or/directory.c index 1b999ee..bbe071b 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2845,7 +2845,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) /** Return the compression level we should use for sending a compressed * response of size n_bytes. */ -STATIC zlib_compression_level_t +STATIC compression_level_t choose_compression_level(ssize_t n_bytes) { if (! have_been_under_memory_pressure()) { diff --git a/src/or/directory.h b/src/or/directory.h index 0c5db3e..4c52c24 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -181,7 +181,7 @@ STATIC int handle_post_hs_descriptor(const char *url, const char *body); STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); -STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); STATIC void find_dl_min_and_max_delay(download_status_t *dls,

1 0

[tor/master] Refactor the streaming compression code.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 3c4459bcbf1d3a9da4a8b3a8bfed10d2e045af74 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon Apr 17 14:57:37 2017 +0200 Refactor the streaming compression code. This patch refactors our streaming compression code to allow us to extend it with non-zlib/non-gzip based compression schemas. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 62 ++++++++++++++++++++++++------------------------- src/common/torgzip.h | 30 +++++++++++++----------- src/or/buffers.c | 20 +++++++++------- src/or/buffers.h | 2 +- src/or/circuitlist.c | 4 ++-- src/or/connection.c | 2 +- src/or/directory.c | 14 +++++------ src/or/dirserv.c | 2 +- src/or/or.h | 4 ++-- src/test/test_buffers.c | 12 +++++----- src/test/test_util.c | 28 +++++++++++----------- 11 files changed, 93 insertions(+), 87 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index cbfcabd..af4bbc5 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -399,9 +399,9 @@ detect_compression_method(const char *in, size_t in_len) } } -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { struct z_stream_s stream; /**< The zlib stream */ int compress; /**< True if we are compressing; false if we are inflating */ @@ -414,14 +414,13 @@ struct tor_zlib_state_t { size_t allocation; }; -/** Construct and return a tor_zlib_state_t object using method. If - * compress, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - compression_level_t compression_level) +/** Construct and return a tor_compress_state_t object using method. If + * compress, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress_, compress_method_t method, + compression_level_t compression_level) { - tor_zlib_state_t *out; + tor_compress_state_t *out; int bits, memlevel; if (! compress_) { @@ -430,7 +429,7 @@ tor_zlib_new(int compress_, compress_method_t method, compression_level = HIGH_COMPRESSION; } - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); + out = tor_malloc_zero(sizeof(tor_compress_state_t)); out->stream.zalloc = Z_NULL; out->stream.zfree = Z_NULL; out->stream.opaque = NULL; @@ -462,16 +461,17 @@ tor_zlib_new(int compress_, compress_method_t method, * to *out, adjusting the values as we go. If finish is true, * we've reached the end of the input. * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on out. - * Return TOR_ZLIB_ERR if the stream is corrupt. + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) { int err; tor_assert(*in_len <= UINT_MAX); @@ -498,31 +498,31 @@ tor_zlib_process(tor_zlib_state_t *state, if (! state->compress && is_compression_bomb(state->input_so_far, state->output_so_far)) { log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; + return TOR_COMPRESS_ERROR; } switch (err) { case Z_STREAM_END: - return TOR_ZLIB_DONE; + return TOR_COMPRESS_DONE; case Z_BUF_ERROR: if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; case Z_OK: if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; default: log_warn(LD_GENERAL, "Gzip returned an error: %s", state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_ZLIB_ERR; + return TOR_COMPRESS_ERROR; } } /** Deallocate state. */ void -tor_zlib_free(tor_zlib_state_t *state) +tor_compress_free(tor_compress_state_t *state) { if (!state) return; @@ -553,7 +553,7 @@ tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) that is, 32K for windowBits=15 (default value) plus a few kilobytes for small objects." */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + (1 << 15) + A_FEW_KILOBYTES; } else { /* Also from zconf.h: @@ -562,7 +562,7 @@ tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) (1 << (windowBits+2)) + (1 << (memLevel+9)) ... plus a few kilobytes for small objects." */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; } #undef A_FEW_KILOBYTES @@ -570,7 +570,7 @@ tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) /** Return the approximate number of bytes allocated for state. */ size_t -tor_zlib_state_size(const tor_zlib_state_t *state) +tor_compress_state_size(const tor_compress_state_t *state) { return state->allocation; } diff --git a/src/common/torgzip.h b/src/common/torgzip.h index ac9763e..2b0504e 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -46,23 +46,27 @@ tor_zlib_get_header_version_str(void); compress_method_t detect_compression_method(const char *in, size_t in_len); -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ +/** Return values from tor_compress_process; see that function's documentation + * for details. */ typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; /** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - compression_level_t level); +typedef struct tor_compress_state_t tor_compress_state_t; +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); -size_t tor_zlib_state_size(const tor_zlib_state_t *state); +size_t tor_compress_state_size(const tor_compress_state_t *state); size_t tor_zlib_get_total_allocation(void); #endif diff --git a/src/or/buffers.c b/src/or/buffers.c index e559f80..3490ce4 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2088,11 +2088,11 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) } /** Compress on uncompress the data_len bytes in data using the - * zlib state state, appending the result to buf. If + * compression state state, appending the result to buf. If * done is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, +write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, const char *data, size_t data_len, int done) { @@ -2108,20 +2108,22 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, } next = CHUNK_WRITE_PTR(buf->tail); avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); - switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) { - case TOR_ZLIB_DONE: + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: over = 1; break; - case TOR_ZLIB_ERR: + case TOR_COMPRESS_ERROR: return -1; - case TOR_ZLIB_OK: + case TOR_COMPRESS_OK: if (data_len == 0) over = 1; break; - case TOR_ZLIB_BUF_FULL: + case TOR_COMPRESS_BUFFER_FULL: if (avail) { - /* Zlib says we need more room (ZLIB_BUF_FULL). Start a new chunk - * automatically, whether were going to or not. */ + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ need_new_chunk = 1; } break; diff --git a/src/or/buffers.h b/src/or/buffers.h index c6a5ffa..49a2ae4 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -36,7 +36,7 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, +int write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 80bb7f6..4f11586 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1991,8 +1991,8 @@ single_conn_free_bytes(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); if (dir_conn->zlib_state) { - result += tor_zlib_state_size(dir_conn->zlib_state); - tor_zlib_free(dir_conn->zlib_state); + result += tor_compress_state_size(dir_conn->zlib_state); + tor_compress_free(dir_conn->zlib_state); dir_conn->zlib_state = NULL; } } diff --git a/src/or/connection.c b/src/or/connection.c index 09e316d..ad97c3b 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -628,7 +628,7 @@ connection_free_(connection_t *conn) dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - tor_zlib_free(dir_conn->zlib_state); + tor_compress_free(dir_conn->zlib_state); if (dir_conn->spool) { SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled, spooled_resource_free(spooled)); diff --git a/src/or/directory.c b/src/or/directory.c index e7a71dd..5ae61e4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3178,7 +3178,7 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compressed, smartlist_len(conn->spool) == 1 ? lifetime : 0); if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + conn->zlib_state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION); /* Prime the connection with some data. */ const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3276,8 +3276,8 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(estimated_len)); + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_zlib(c, strlen(c), conn, 0)); connection_write_to_buf_zlib("", 0, conn, 1); @@ -3335,7 +3335,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3428,7 +3428,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -3519,8 +3519,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(len)); + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 87afd69..186478c 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3792,7 +3792,7 @@ connection_dirserv_flushed_some(dir_connection_t *conn) /* Flush the zlib state: there could be more bytes pending in there, and * we don't want to omit bytes. */ connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); + tor_compress_free(conn->zlib_state); conn->zlib_state = NULL; } return 0; diff --git a/src/or/or.h b/src/or/or.h index a7b3a66..6aec588 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1773,8 +1773,8 @@ typedef struct dir_connection_t { /** List of spooled_resource_t for objects that we're spooling. We use * it from back to front. */ smartlist_t *spool; - /** The zlib object doing on-the-fly compression for spooled data. */ - tor_zlib_state_t *zlib_state; + /** The compression object doing on-the-fly compression for spooled data. */ + tor_compress_state_t *zlib_state; /** What rendezvous service are we querying for? */ rend_data_t *rend_data; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index f0edd42..a6f0309 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -584,12 +584,12 @@ test_buffers_zlib_impl(int finalize_with_nil) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_zlib_state_t *zlib_state = NULL; + tor_compress_state_t *zlib_state = NULL; size_t out_len, in_len; int done; buf = buf_new_with_capacity(128); /* will round up */ - zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + zlib_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); msg = tor_malloc(512); crypto_rand(msg, 512); @@ -621,7 +621,7 @@ test_buffers_zlib_impl(int finalize_with_nil) done: buf_free(buf); - tor_zlib_free(zlib_state); + tor_compress_free(zlib_state); tor_free(contents); tor_free(expanded); tor_free(msg); @@ -647,7 +647,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_zlib_state_t *zlib_state = NULL; + tor_compress_state_t *zlib_state = NULL; size_t out_len, in_len; size_t sz, headerjunk; (void) arg; @@ -666,7 +666,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(buf->head->datalen, OP_EQ, headerjunk); tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ - zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + zlib_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); @@ -687,7 +687,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) done: buf_free(buf); - tor_zlib_free(zlib_state); + tor_compress_free(zlib_state); tor_free(contents); tor_free(expanded); tor_free(msg); diff --git a/src/test/test_util.c b/src/test/test_util.c index 7e24279..dacf56f 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2249,7 +2249,7 @@ test_util_gzip(void *arg) char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; const char *ccp2; size_t len1, len2; - tor_zlib_state_t *state = NULL; + tor_compress_state_t *state = NULL; (void)arg; buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); @@ -2320,21 +2320,21 @@ test_util_gzip(void *arg) tor_free(buf1); tor_free(buf2); tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); tt_assert(state); cp1 = buf1 = tor_malloc(1024); len1 = 1024; ccp2 = "ABCDEFGHIJABCDEFGHIJ"; len2 = 21; - tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ tt_assert(cp1 > buf1); len2 = 0; cp2 = cp1; - tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ @@ -2346,7 +2346,7 @@ test_util_gzip(void *arg) done: if (state) - tor_zlib_free(state); + tor_compress_free(state); tor_free(buf2); tor_free(buf3); tor_free(buf1); @@ -2364,7 +2364,7 @@ test_util_gzip_compression_bomb(void *arg) char *one_mb = tor_malloc_zero(one_million); char *result = NULL; size_t result_len = 0; - tor_zlib_state_t *state = NULL; + tor_compress_state_t *state = NULL; /* Make sure we can't produce a compression bomb */ setup_full_capture_of_logs(LOG_WARN); @@ -2386,22 +2386,22 @@ test_util_gzip_compression_bomb(void *arg) ZLIB_METHOD, 0, LOG_WARN)); /* Now try streaming that. */ - state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); - tor_zlib_output_t r; + state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + tor_compress_output_t r; const char *inp = compression_bomb; size_t inlen = 1039; do { char *outp = one_mb; size_t outleft = 4096; /* small on purpose */ - r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0); + r = tor_compress_process(state, &outp, &outleft, &inp, &inlen, 0); tt_int_op(inlen, OP_NE, 0); - } while (r == TOR_ZLIB_BUF_FULL); + } while (r == TOR_COMPRESS_BUFFER_FULL); - tt_int_op(r, OP_EQ, TOR_ZLIB_ERR); + tt_int_op(r, OP_EQ, TOR_COMPRESS_ERROR); done: tor_free(one_mb); - tor_zlib_free(state); + tor_compress_free(state); } /** Run unit tests for mmap() wrapper functionality. */

1 0

[tor/master] Move zlib compression code into its own module.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 9d5bc1a9354637aa59025f61e577c6d42f8c53ba Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 03:14:36 2017 +0200 Move zlib compression code into its own module. This patch refactors the `torgzip` module to allow us to extend a common compression API to support multiple compression backends. Additionally we move the gzip/zlib code into its own module under the name `compress_zlib`. See https://bugs.torproject.org/21664 --- src/common/compress_zlib.c | 524 +++++++++++++++++++++++++++++++++++++++++++++ src/common/compress_zlib.h | 56 +++++ src/common/include.am | 2 + src/common/torgzip.c | 458 ++++----------------------------------- src/common/torgzip.h | 11 +- src/or/config.c | 1 + src/or/main.c | 1 + src/or/relay.c | 1 + 8 files changed, 633 insertions(+), 421 deletions(-) diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000..38e500c --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,524 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "torgzip.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static size_t total_zlib_allocation = 0; + +/** Return the 'bits' value to tell zlib to use method.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the method described in method. Store the + * compressed string in *out, and its length in *out_len. Return + * 0 on success, -1 on failure. + */ +int +tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + struct z_stream_s *stream = NULL; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + + *out = NULL; + + stream = tor_malloc_zero(sizeof(struct z_stream_s)); + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = NULL; + stream->next_in = (unsigned char*) in; + stream->avail_in = (unsigned int)in_len; + + if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, + method_bits(method, HIGH_COMPRESSION), + tor_compress_memory_level(HIGH_COMPRESSION), + Z_DEFAULT_STRATEGY) != Z_OK) { + //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. + log_warn(LD_GENERAL, "Error from deflateInit2: %s", + stream->msg?stream->msg:"<no message>"); + goto err; + //LCOV_EXCL_STOP + } + + /* Guess 50% compression. */ + out_size = in_len / 2; + if (out_size < 1024) out_size = 1024; + *out = tor_malloc(out_size); + stream->next_out = (unsigned char*)*out; + stream->avail_out = (unsigned int)out_size; + + while (1) { + switch (deflate(stream, Z_FINISH)) + { + case Z_STREAM_END: + goto done; + case Z_OK: + /* In case zlib doesn't work as I think .... */ + if (stream->avail_out >= stream->avail_in+16) + break; + case Z_BUF_ERROR: + offset = stream->next_out - ((unsigned char*)*out); + old_size = out_size; + out_size *= 2; + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in compression."); + goto err; + } + *out = tor_realloc(*out, out_size); + stream->next_out = (unsigned char*)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " + "uncompressing."); + goto err; + } + stream->avail_out = (unsigned int)(out_size - offset); + break; + default: + log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", + stream->msg ? stream->msg : "<no message>"); + goto err; + } + } + done: + *out_len = stream->total_out; +#if defined(OpenBSD) + /* "Hey Rocky! Watch me change an unsigned field to a signed field in a + * third-party API!" + * "Oh, that trick will just make people do unsafe casts to the unsigned + * type in their cross-platform code!" + * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure + * the newly unsigned field isn't negative." */ + tor_assert(stream->total_out >= 0); +#endif + if (deflateEnd(stream)!=Z_OK) { + // LCOV_EXCL_START -- unreachable if we handled the zlib structure right + tor_assert_nonfatal_unreached(); + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + // LCOV_EXCL_STOP + } + tor_free(stream); + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a zlib bomb."); + goto err; + } + + return 0; + err: + if (stream) { + deflateEnd(stream); + tor_free(stream); + } + tor_free(*out); + return -1; +} + +/** Given an Zlib/Gzip compressed string of total length in_len bytes + * at in, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at protocol_warn_level. + */ +int +tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + struct z_stream_s *stream = NULL; + size_t out_size, old_size; + off_t offset; + int r; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + + *out = NULL; + + stream = tor_malloc_zero(sizeof(struct z_stream_s)); + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = NULL; + stream->next_in = (unsigned char*) in; + stream->avail_in = (unsigned int)in_len; + + if (inflateInit2(stream, + method_bits(method, HIGH_COMPRESSION)) != Z_OK) { + // LCOV_EXCL_START -- can only hit this if we give bad inputs. + log_warn(LD_GENERAL, "Error from inflateInit2: %s", + stream->msg?stream->msg:"<no message>"); + goto err; + // LCOV_EXCL_STOP + } + + out_size = in_len * 2; /* guess 50% compression. */ + if (out_size < 1024) out_size = 1024; + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + stream->next_out = (unsigned char*)*out; + stream->avail_out = (unsigned int)out_size; + + while (1) { + switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) + { + case Z_STREAM_END: + if (stream->avail_in == 0) + goto done; + /* There may be more compressed data here. */ + if ((r = inflateEnd(stream)) != Z_OK) { + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + } + if (inflateInit2(stream, + method_bits(method,HIGH_COMPRESSION)) != Z_OK) { + log_warn(LD_GENERAL, "Error from second inflateInit2: %s", + stream->msg?stream->msg:"<no message>"); + goto err; + } + break; + case Z_OK: + if (!complete_only && stream->avail_in == 0) + goto done; + /* In case zlib doesn't work as I think.... */ + if (stream->avail_out >= stream->avail_in+16) + break; + case Z_BUF_ERROR: + if (stream->avail_out > 0) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "possible truncated or corrupt zlib data"); + goto err; + } + offset = stream->next_out - (unsigned char*)*out; + old_size = out_size; + out_size *= 2; + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in uncompression."); + goto err; + } + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " + "not proceeding."); + goto err; + } + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); + goto err; + } + *out = tor_realloc(*out, out_size); + stream->next_out = (unsigned char*)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " + "uncompressing."); + goto err; + } + stream->avail_out = (unsigned int)(out_size - offset); + break; + default: + log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", + stream->msg ? stream->msg : "<no message>"); + goto err; + } + } + done: + *out_len = stream->next_out - (unsigned char*)*out; + r = inflateEnd(stream); + tor_free(stream); + if (r != Z_OK) { + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + } + + /* NUL-terminate output. */ + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + err: + if (stream) { + inflateEnd(stream); + tor_free(stream); + } + if (*out) { + tor_free(*out); + } + return -1; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits windowBits and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * method. If compress, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = HIGH_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress; + bits = method_bits(method, compression_level); + memlevel = tor_compress_memory_level(compression_level); + if (compress) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel); + + total_zlib_allocation += out->allocation; + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate state. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + total_zlib_allocation -= state->allocation; + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return total_zlib_allocation; +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000..0862678 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +const char * +tor_zlib_get_version_str(void); + +const char * +tor_zlib_get_header_version_str(void); + +int +tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t +tor_zlib_get_total_allocation(void); + +#endif // TOR_COMPRESS_ZLIB_H. + diff --git a/src/common/include.am b/src/common/include.am index d6504c7..ea2c46a 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -105,6 +105,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ @@ -145,6 +146,7 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ diff --git a/src/common/torgzip.c b/src/common/torgzip.c index d6f6225..d760cd9 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -23,69 +23,7 @@ #include "util.h" #include "torlog.h" #include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use method.*/ -static inline int -method_bits(compress_method_t method, compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - +#include "compress_zlib.h" /** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -140,107 +78,9 @@ tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method) { - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_compress(out, out_len, in, in_len, method); - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); return -1; } @@ -262,127 +102,12 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level) { - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } return -1; } @@ -406,58 +131,39 @@ detect_compression_method(const char *in, size_t in_len) /** Internal state for an incremental compression/decompression. The body of * this struct is not exposed. */ struct tor_compress_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ + compress_method_t method; /**< The compression method. */ - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; + union { + tor_zlib_compress_state_t *zlib_state; + } u; /**< Compression backend state. */ }; /** Construct and return a tor_compress_state_t object using method. If * compress, it's for compression; otherwise it's for decompression. */ tor_compress_state_t * -tor_compress_new(int compress_, compress_method_t method, +tor_compress_new(int compress, compress_method_t method, compression_level_t compression_level) { - tor_compress_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_compress_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + if (method == GZIP_METHOD || method == ZLIB_METHOD) { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + } + + return state; err: - tor_free(out); - return NULL; + tor_free(state); + return NULL; } /** Compress/decompress some bytes using state. Read up to @@ -477,112 +183,38 @@ tor_compress_process(tor_compress_state_t *state, const char **in, size_t *in_len, int finish) { - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + tor_assert(state != NULL); - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_COMPRESS_ERROR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_COMPRESS_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_COMPRESS_OK; - return TOR_COMPRESS_BUFFER_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_COMPRESS_BUFFER_FULL; - return TOR_COMPRESS_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_COMPRESS_ERROR; - } + return TOR_COMPRESS_ERROR; } /** Deallocate state. */ void tor_compress_free(tor_compress_state_t *state) { - if (!state) + if (state == NULL) return; - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + tor_zlib_compress_free(state->u.zlib_state); tor_free(state); } -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits windowBits and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - /** Return the approximate number of bytes allocated for state. */ size_t tor_compress_state_size(const tor_compress_state_t *state) { - return state->allocation; -} + tor_assert(state != NULL); -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_state_size(state->u.zlib_state); + + return 0; } diff --git a/src/common/torgzip.h b/src/common/torgzip.h index fa56628..0fcac0c 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -38,12 +38,6 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level); -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - compress_method_t detect_compression_method(const char *in, size_t in_len); int @@ -60,8 +54,10 @@ typedef enum { TOR_COMPRESS_BUFFER_FULL, TOR_COMPRESS_ERROR } tor_compress_output_t; -/** Internal state for an incremental zlib compression/decompression. */ + +/** Internal state for an incremental compression/decompression. */ typedef struct tor_compress_state_t tor_compress_state_t; + tor_compress_state_t *tor_compress_new(int compress, compress_method_t method, compression_level_t level); @@ -73,7 +69,6 @@ tor_compress_output_t tor_compress_process(tor_compress_state_t *state, void tor_compress_free(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -size_t tor_zlib_get_total_allocation(void); #endif diff --git a/src/or/config.c b/src/or/config.c index 809ff49..83e5f21 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress_zlib.h" #include "config.h" #include "connection.h" #include "connection_edge.h" diff --git a/src/or/main.c b/src/or/main.c index 4505879..ddd7b82 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,6 +58,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress_zlib.h" #include "config.h" #include "confparse.h" #include "connection.h" diff --git a/src/or/relay.c b/src/or/relay.c index 5139036..8a98c50 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,6 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "compress_zlib.h" #include "config.h" #include "connection.h" #include "connection_edge.h"

1 0

[tor/master] Show liblzma and libzstd versions in `tor --library-versions`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 2fa7b722ce3dcb54dca598d6b1d5a5abfb716f7c Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 22:30:36 2017 +0200 Show liblzma and libzstd versions in `tor --library-versions`. See: https://bugs.torproject.org/21662 --- src/or/config.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/or/config.c b/src/or/config.c index 9af116d..3549a1d 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -70,7 +70,9 @@ #include "circuitmux_ewma.h" #include "circuitstats.h" #include "compress.h" +#include "compress_lzma.h" #include "compress_zlib.h" +#include "compress_zstd.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -4953,6 +4955,12 @@ options_init_from_torrc(int argc, char **argv) printf("Zlib \t\t%-15s\t\t%s\n", tor_zlib_get_header_version_str(), tor_zlib_get_version_str()); + printf("Liblzma \t\t%-15s\t\t%s\n", + tor_lzma_get_header_version_str(), + tor_lzma_get_version_str()); + printf("Libzstd \t\t%-15s\t\t%s\n", + tor_zstd_get_header_version_str(), + tor_zstd_get_version_str()); //TODO: Hex versions? exit(0); }

1 0

[tor/master] Rename `tor_gzip_{compress, uncompress}` to `tor_{compress, uncompress}`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 44cb86adbe2e9b9fef7bf5fd64b52a8e44441be5 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon Apr 17 14:29:10 2017 +0200 Rename `tor_gzip_{compress,uncompress}` to `tor_{compress,uncompress}`. To allow us to use the API name `tor_compress` and `tor_uncompress` as the main entry-point for all compression/uncompression and not just gzip and zlib. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 16 +++++++------- src/common/torgzip.h | 16 +++++++------- src/or/directory.c | 8 +++---- src/or/dirserv.c | 4 ++-- src/test/test_buffers.c | 17 ++++++++------- src/test/test_dir_handle_get.c | 4 ++-- src/test/test_util.c | 48 +++++++++++++++++++++--------------------- 7 files changed, 57 insertions(+), 56 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index a6909ce..cbfcabd 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -132,9 +132,9 @@ is_compression_bomb(size_t size_in, size_t size_out) * Return 0 on success, -1 on failure. */ int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) { struct z_stream_s *stream = NULL; size_t out_size, old_size; @@ -252,11 +252,11 @@ tor_gzip_compress(char **out, size_t *out_len, * or corrupt inputs at protocol_warn_level. */ int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) { struct z_stream_s *stream = NULL; size_t out_size, old_size; diff --git a/src/common/torgzip.h b/src/common/torgzip.h index cbcbb99..ac9763e 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -28,15 +28,15 @@ typedef enum { } compression_level_t; int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); const char * tor_zlib_get_version_str(void); diff --git a/src/or/directory.c b/src/or/directory.c index bbe071b..e7a71dd 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2096,15 +2096,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } /* Try declared compression first if we can. */ if (compression == GZIP_METHOD || compression == ZLIB_METHOD) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ if (!new_body && (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && compression != guessed) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_PROTOCOL_WARN); /* If we're pretty sure that we have a compressed directory, and * we didn't manage to uncompress it, then warn and bail. */ if (!plausible && !new_body) { diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 70b0b22..87afd69 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1176,8 +1176,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, + ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index d14165c..f0edd42 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -607,10 +607,10 @@ test_buffers_zlib_impl(int finalize_with_nil) tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0); - tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len, - contents, in_len, - ZLIB_METHOD, 1, - LOG_WARN)); + tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, + contents, in_len, + ZLIB_METHOD, 1, + LOG_WARN)); tt_int_op(out_len, OP_GE, 128); tt_mem_op(msg, OP_EQ, expanded, 128); @@ -676,10 +676,11 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(in_len, OP_GT, headerjunk); - tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len, - contents + headerjunk, in_len - headerjunk, - ZLIB_METHOD, 1, - LOG_WARN)); + tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, + contents + headerjunk, + in_len - headerjunk, + ZLIB_METHOD, 1, + LOG_WARN)); tt_int_op(out_len, OP_EQ, 0); tt_assert(expanded); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index cfda314..ed8ea2f 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1832,8 +1832,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data) comp_body_used); tt_int_op(ZLIB_METHOD, OP_EQ, compression); - tor_gzip_uncompress(&body, &body_used, comp_body, comp_body_used, - compression, 0, LOG_PROTOCOL_WARN); + tor_uncompress(&body, &body_used, comp_body, comp_body_used, + compression, 0, LOG_PROTOCOL_WARN); tt_str_op(NETWORK_STATUS, OP_EQ, body); tt_int_op(strlen(NETWORK_STATUS), OP_EQ, body_used); diff --git a/src/test/test_util.c b/src/test/test_util.c index 19d1c79..7e24279 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2255,14 +2255,14 @@ test_util_gzip(void *arg) buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); tt_assert(buf2 != NULL); tt_int_op(len1, OP_LT, strlen(buf1)); tt_int_op(detect_compression_method(buf2, len1), OP_EQ, GZIP_METHOD); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); tt_int_op(strlen(buf1) + 1, OP_EQ, len2); tt_str_op(buf1, OP_EQ, buf3); @@ -2270,13 +2270,13 @@ test_util_gzip(void *arg) tor_free(buf2); tor_free(buf3); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); tt_assert(buf2); tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZLIB_METHOD); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); tt_int_op(strlen(buf1) + 1, OP_EQ, len2); tt_str_op(buf1, OP_EQ, buf3); @@ -2285,8 +2285,8 @@ test_util_gzip(void *arg) tor_free(buf3); buf2 = tor_reallocarray(buf2, len1, 2); memcpy(buf2+len1, buf2, len1); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); tt_mem_op(buf3, OP_EQ, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" @@ -2300,20 +2300,20 @@ test_util_gzip(void *arg) /* Check whether we can uncompress partial strings. */ buf1 = tor_strdup("String with low redundancy that won't be compressed much."); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); tt_int_op(len1, OP_GT, 16); /* when we allow an incomplete string, we should succeed.*/ - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); tt_assert(len2 > 5); buf3[len2]='\0'; tt_assert(!strcmpstart(buf1, buf3)); /* when we demand a complete string, this must fail. */ tor_free(buf3); - tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); tt_assert(buf3 == NULL); /* Now, try streaming compression. */ @@ -2338,8 +2338,8 @@ test_util_gzip(void *arg) tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); /* Make sure it compressed right. */ tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); tt_int_op(21, OP_EQ, len2); @@ -2368,9 +2368,9 @@ test_util_gzip_compression_bomb(void *arg) /* Make sure we can't produce a compression bomb */ setup_full_capture_of_logs(LOG_WARN); - tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len, - one_mb, one_million, - ZLIB_METHOD)); + tt_int_op(-1, OP_EQ, tor_compress(&result, &result_len, + one_mb, one_million, + ZLIB_METHOD)); expect_single_log_msg_containing( "We compressed something and got an insanely high " "compression factor; other Tors would think this " @@ -2381,9 +2381,9 @@ test_util_gzip_compression_bomb(void *arg) const char compression_bomb[1039] = { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2, 0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ }; - tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len, - compression_bomb, 1039, - ZLIB_METHOD, 0, LOG_WARN)); + tt_int_op(-1, OP_EQ, tor_uncompress(&result, &result_len, + compression_bomb, 1039, + ZLIB_METHOD, 0, LOG_WARN)); /* Now try streaming that. */ state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);

1 0

[tor/master] Display LZMA and Zstandard versions when starting Tor.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit be4dc546345831773c2e16fa3dc12749d925aa25 Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 22:31:07 2017 +0200 Display LZMA and Zstandard versions when starting Tor. See: https://bugs.torproject.org/21662 --- src/or/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/or/main.c b/src/or/main.c index ddd7b82..f3a0d84 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,7 +58,9 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress_lzma.h" #include "compress_zlib.h" +#include "compress_zstd.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -2999,11 +3001,13 @@ tor_init(int argc, char *argv[]) const char *version = get_version(); log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, " - "OpenSSL %s and Zlib %s.", version, + "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version, get_uname(), tor_libevent_get_version_str(), crypto_openssl_get_version_str(), - tor_zlib_get_version_str()); + tor_zlib_get_version_str(), + tor_lzma_get_version_str(), + tor_zstd_get_version_str()); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at "

1 0

[tor/master] Add Zstandard support.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 380736d045f85299121f044928170cff321ae852 Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 22:19:29 2017 +0200 Add Zstandard support. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 33 +++ src/common/compress.h | 3 +- src/common/compress_zstd.c | 584 +++++++++++++++++++++++++++++++++++++++++++++ src/common/compress_zstd.h | 56 +++++ src/common/include.am | 2 + src/test/test_util.c | 107 +++++++++ 6 files changed, 784 insertions(+), 1 deletion(-) diff --git a/src/common/compress.c b/src/common/compress.c index 3c7a3c6..dc86be7 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -25,6 +25,7 @@ #include "compress.h" #include "compress_lzma.h" #include "compress_zlib.h" +#include "compress_zstd.h" /** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -85,6 +86,9 @@ tor_compress(char **out, size_t *out_len, if (method == LZMA_METHOD) return tor_lzma_compress(out, out_len, in, in_len, method); + if (method == ZSTD_METHOD) + return tor_zstd_compress(out, out_len, in, in_len, method); + return -1; } @@ -118,6 +122,12 @@ tor_uncompress(char **out, size_t *out_len, complete_only, protocol_warn_level); + if (method == ZSTD_METHOD) + return tor_zstd_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + return -1; } @@ -136,6 +146,9 @@ detect_compression_method(const char *in, size_t in_len) } else if (in_len > 3 && fast_memeq(in, "\x5d\x00\x00\x00", 4)) { return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; } else { return UNKNOWN_METHOD; } @@ -149,6 +162,7 @@ struct tor_compress_state_t { union { tor_zlib_compress_state_t *zlib_state; tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; } u; /**< Compression backend state. */ }; @@ -185,6 +199,16 @@ tor_compress_new(int compress, compress_method_t method, state->u.lzma_state = lzma_state; break; } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -226,6 +250,10 @@ tor_compress_process(tor_compress_state_t *state, return tor_lzma_compress_process(state->u.lzma_state, out, out_len, in, in_len, finish); + case ZSTD_METHOD: + return tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -250,6 +278,9 @@ tor_compress_free(tor_compress_state_t *state) case LZMA_METHOD: tor_lzma_compress_free(state->u.lzma_state); break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; case NO_METHOD: case UNKNOWN_METHOD: break; @@ -270,6 +301,8 @@ tor_compress_state_size(const tor_compress_state_t *state) return tor_zlib_compress_state_size(state->u.zlib_state); case LZMA_METHOD: return tor_lzma_compress_state_size(state->u.lzma_state); + case ZSTD_METHOD: + return tor_zstd_compress_state_size(state->u.zstd_state); case NO_METHOD: case UNKNOWN_METHOD: goto err; diff --git a/src/common/compress.h b/src/common/compress.h index 3e3ab3a..59e7a79 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -19,7 +19,8 @@ typedef enum { GZIP_METHOD=1, ZLIB_METHOD=2, LZMA_METHOD=3, - UNKNOWN_METHOD=4 + ZSTD_METHOD=4, + UNKNOWN_METHOD=5 } compress_method_t; /** diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000..e2eb292 --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,584 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef HAVE_ZSTD +#include <zstd.h> +#include <zstd_errors.h> +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static size_t total_zstd_allocation = 0; + +/** Return a string representation of the version of the currently running + * version of libzstd. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[16]; + size_t version_number; + + version_number = ZSTD_versionNumber(); + tor_snprintf(version_str, sizeof(version_str), + "%lu.%lu.%lu", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); + + return version_str; +#else + return "N/A"; +#endif +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return "N/A"; +#endif +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the Zstandard method. Store the compressed string + * in *out, and its length in *out_len. Return 0 on success, -1 + * on failure. + */ +int +tor_zstd_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ +#ifdef HAVE_ZSTD + ZSTD_CStream *stream = NULL; + size_t out_size, old_size; + size_t retval; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == ZSTD_METHOD); + + *out = NULL; + + stream = ZSTD_createCStream(); + + if (stream == NULL) { + // Zstandard does not give us any useful error message to why this + // happened. See https://github.com/facebook/zstd/issues/398 + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initCStream(stream, + tor_compress_memory_level(HIGH_COMPRESSION)); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + // Assume 50% compression and update our buffer in case we need to. + out_size = in_len / 2; + if (out_size < 1024) + out_size = 1024; + + *out = tor_malloc(out_size); + *out_len = 0; + + ZSTD_inBuffer input = { in, in_len, 0 }; + ZSTD_outBuffer output = { *out, out_size, 0 }; + + while (input.pos < input.size) { + retval = ZSTD_compressStream(stream, &output, &input); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream compression error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + if (input.pos < input.size && output.pos == output.size) { + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); + goto err; + } + + if (out_size - output.pos > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " + "compressing."); + goto err; + } + + output.dst = *out = tor_realloc(*out, out_size); + output.size = out_size; + } + } + + while (1) { + retval = ZSTD_endStream(stream, &output); + + if (retval == 0) + break; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + if (output.pos == output.size) { + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); + goto err; + } + + if (out_size - output.pos > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " + "compressing."); + goto err; + } + + output.dst = *out = tor_realloc(*out, out_size); + output.size = out_size; + } + } + + *out_len = output.pos; + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tor instances would think " + "this is a compression bomb."); + goto err; + } + + if (stream != NULL) { + ZSTD_freeCStream(stream); + } + + return 0; + + err: + if (stream != NULL) { + ZSTD_freeCStream(stream); + } + + tor_free(*out); + return -1; +#else // HAVE_ZSTD. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + + return -1; +#endif // HAVE_ZSTD. +} + +/** Given a Zstandard compressed string of total length in_len bytes at + * in, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at protocol_warn_level. + */ +int +tor_zstd_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ +#ifdef HAVE_ZSTD + ZSTD_DStream *stream = NULL; + size_t retval; + size_t out_size, old_size; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == ZSTD_METHOD); + + // FIXME(ahf): Handle this? + (void)complete_only; + (void)protocol_warn_level; + + *out = NULL; + + stream = ZSTD_createDStream(); + + if (stream == NULL) { + // Zstandard does not give us any useful error message to why this + // happened. See https://github.com/facebook/zstd/issues/398 + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initDStream(stream); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + out_size = in_len * 2; + if (out_size < 1024) + out_size = 1024; + + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + *out_len = 0; + + ZSTD_inBuffer input = { in, in_len, 0 }; + ZSTD_outBuffer output = { *out, out_size, 0 }; + + while (input.pos < input.size) { + retval = ZSTD_decompressStream(stream, &output, &input); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream decompression error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + if (input.pos < input.size && output.pos == output.size) { + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); + goto err; + } + + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible Zstandard " + "compression bomb. Not proceeding."); + goto err; + } + + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " + "Zstandard data."); + goto err; + } + + if (out_size - output.pos > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " + "decompressing."); + goto err; + } + + output.dst = *out = tor_realloc(*out, out_size); + output.size = out_size; + } + } + + *out_len = output.pos; + + if (stream != NULL) { + ZSTD_freeDStream(stream); + } + + // NUL-terminate our output. + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + + err: + if (stream != NULL) { + ZSTD_freeDStream(stream); + } + + tor_free(*out); + return -1; +#else // HAVE_ZSTD. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + (void)complete_only; + (void)protocol_warn_level; + + return -1; +#endif // HAVE_ZSTD. +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when compress is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when compress is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif // HAVE_ZSTD. + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Construct and return a tor_zstd_compress_state_t object using + * method. If compress, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + + // FIXME(ahf): We should either try to do the pre-calculation that is done + // with the zlib backend or use a custom allocator here where we pass our + // tor_zstd_compress_state_t as the opaque value. + result->allocation = 0; + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initCStream(result->u.compress_stream, + tor_compress_memory_level(compression_level)); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } + + return result; + + err: + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; +#else // HAVE_ZSTD. + (void)compress; + (void)method; + (void)compression_level; + + return NULL; +#endif // HAVE_ZSTD. +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (state->compress) { + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !finish) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + } + + if (state->compress && finish) { + retval = ZSTD_endStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + } + + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; +#else // HAVE_ZSTD. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif // HAVE_ZSTD. +} + +/** Deallocate state. */ +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + total_zstd_allocation -= state->allocation; + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif // HAVE_ZSTD. + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return total_zstd_allocation; +} + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000..ec83ba9 --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +const char * +tor_zstd_get_version_str(void); + +const char * +tor_zstd_get_header_version_str(void); + +int +tor_zstd_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_zstd_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state); + +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t +tor_zstd_get_total_allocation(void); + +#endif // TOR_COMPRESS_ZSTD_H. + diff --git a/src/common/include.am b/src/common/include.am index 2682c22..e285ef5 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -108,6 +108,7 @@ LIBOR_CRYPTO_A_SRC = \ src/common/compress.c \ src/common/compress_lzma.c \ src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ @@ -150,6 +151,7 @@ COMMONHEADERS = \ src/common/compress.h \ src/common/compress_lzma.h \ src/common/compress_zlib.h \ + src/common/compress_zstd.h \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ diff --git a/src/test/test_util.c b/src/test/test_util.c index 287f7e8..6a7db31 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2510,6 +2510,112 @@ test_util_lzma(void *arg) #endif // HAVE_LZMA. } +static void +test_util_zstd(void *arg) +{ +#ifdef HAVE_ZSTD + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_compress_state_t *state = NULL; + + (void)arg; + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZSTD_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZSTD_METHOD); + + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + ZSTD_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + +#if 0 + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_reallocarray(buf2, len1, 2); + memcpy(buf2+len1, buf2, len1); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + ZSTD_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZSTD_METHOD)); + tt_int_op(len1, OP_GT, 16); + /* when we allow an incomplete string, we should succeed.*/ + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + ZSTD_METHOD, 0, LOG_INFO)); + tt_assert(len2 > 5); + buf3[len2]='\0'; + tt_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + ZSTD_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); +#endif + + /* Now, try streaming compression. */ + state = tor_compress_new(1, ZSTD_METHOD, HIGH_COMPRESSION); + tt_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ +// tt_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); + tt_assert(cp1 > cp2); /* Make sure we really added something. */ + + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + ZSTD_METHOD, 1, LOG_WARN)); + /* Make sure it compressed right. */ + tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); + tt_int_op(21, OP_EQ, len2); + + done: + if (state) + tor_compress_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +#else + (void)arg; +#endif // HAVE_ZSTD. +} + /** Run unit tests for mmap() wrapper functionality. */ static void test_util_mmap(void *arg) @@ -5824,6 +5930,7 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(gzip), UTIL_TEST(gzip_compression_bomb, TT_FORK), UTIL_LEGACY(lzma), + UTIL_LEGACY(zstd), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats),

1 0

[tor/master] Add `tor_compress_get_total_allocation()` function.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 04682d302aeeb0c2a0d9859bc0c1feee38daf16a Author: Alexander Færøy <ahf(a)torproject.org> Date: Thu Apr 20 15:33:13 2017 +0200 Add `tor_compress_get_total_allocation()` function. This patch adds the `tor_compress_get_total_allocation()` which returns an approximate number of bytes currently in use by all the different compression backends. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 10 ++++++++++ src/common/compress.h | 3 +++ src/or/relay.c | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index dc86be7..52ff132 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -154,6 +154,16 @@ detect_compression_method(const char *in, size_t in_len) } } +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + /** Internal state for an incremental compression/decompression. The body of * this struct is not exposed. */ struct tor_compress_state_t { diff --git a/src/common/compress.h b/src/common/compress.h index 59e7a79..1fd4325 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -51,6 +51,9 @@ tor_compress_memory_level(compression_level_t level); int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); +size_t +tor_compress_get_total_allocation(void); + /** Return values from tor_compress_process; see that function's documentation * for details. */ typedef enum { diff --git a/src/or/relay.c b/src/or/relay.c index 8a98c50..8524080 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,7 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" -#include "compress_zlib.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -2454,7 +2454,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); - alloc += tor_zlib_get_total_allocation(); + alloc += tor_compress_get_total_allocation(); const size_t rend_cache_total = rend_cache_get_total_allocation(); alloc += rend_cache_total; if (alloc >= get_options()->MaxMemInQueues_low_threshold) {

1 0

[tor/master] Add function to check if a given compression method is supported.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 1c77d8690caa1ad80b9d9581cac8eddae95e82d1 Author: Alexander Færøy <ahf(a)torproject.org> Date: Thu Apr 20 15:56:38 2017 +0200 Add function to check if a given compression method is supported. This patch adds support for checking if a given `compress_method_t` is supported by the currently running Tor instance using `tor_compress_supports_method()`. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 19 +++++++++++++++++++ src/common/compress.h | 6 +++++- src/common/compress_lzma.c | 11 +++++++++++ src/common/compress_lzma.h | 3 +++ src/common/compress_zlib.c | 10 ++++++++++ src/common/compress_zlib.h | 3 +++ src/common/compress_zstd.c | 11 +++++++++++ src/common/compress_zstd.h | 3 +++ src/test/test_util.c | 15 +++++++++++++++ 9 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/common/compress.c b/src/common/compress.c index 52ff132..725dde5 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -154,6 +154,25 @@ detect_compression_method(const char *in, size_t in_len) } } +/** Return 1 if a given method is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return 0; + } +} + /** Return the approximate number of bytes allocated for all * supported compression schemas. */ size_t diff --git a/src/common/compress.h b/src/common/compress.h index 1fd4325..87306f5 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -13,7 +13,8 @@ /** Enumeration of what kind of compression to use. Only ZLIB_METHOD and * GZIP_METHOD is guaranteed to be supported by the compress/uncompress - * functions here. */ + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ typedef enum { NO_METHOD=0, GZIP_METHOD=1, @@ -51,6 +52,9 @@ tor_compress_memory_level(compression_level_t level); int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); +int +tor_compress_supports_method(compress_method_t method); + size_t tor_compress_get_total_allocation(void); diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index e1a7a66..c45cb5e 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -61,6 +61,17 @@ lzma_error_str(lzma_ret error) } #endif // HAVE_LZMA. +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + /** Return a string representation of the version of the currently running * version of liblzma. */ const char * diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h index 2cc3a00..801e22d 100644 --- a/src/common/compress_lzma.h +++ b/src/common/compress_lzma.h @@ -11,6 +11,9 @@ #ifndef TOR_COMPRESS_LZMA_H #define TOR_COMPRESS_LZMA_H +int +tor_lzma_method_supported(void); + const char * tor_lzma_get_version_str(void); diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index 2c9aba3..e1a68c2 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -64,6 +64,16 @@ method_bits(compress_method_t method, compression_level_t level) } } +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + /** Return a string representation of the version of the currently running * version of zlib. */ const char * diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h index 0862678..0b1aad8 100644 --- a/src/common/compress_zlib.h +++ b/src/common/compress_zlib.h @@ -11,6 +11,9 @@ #ifndef TOR_COMPRESS_ZLIB_H #define TOR_COMPRESS_ZLIB_H +int +tor_zlib_method_supported(void); + const char * tor_zlib_get_version_str(void); diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index e2eb292..dca4dbd 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -26,6 +26,17 @@ /** Total number of bytes allocated for Zstandard state. */ static size_t total_zstd_allocation = 0; +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + /** Return a string representation of the version of the currently running * version of libzstd. */ const char * diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index ec83ba9..b2297bd 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -11,6 +11,9 @@ #ifndef TOR_COMPRESS_ZSTD_H #define TOR_COMPRESS_ZSTD_H +int +tor_zstd_method_supported(void); + const char * tor_zstd_get_version_str(void); diff --git a/src/test/test_util.c b/src/test/test_util.c index 6a7db31..1e33de8 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2252,6 +2252,9 @@ test_util_gzip(void *arg) tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(GZIP_METHOD)); + tt_assert(tor_compress_supports_method(ZLIB_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); @@ -2414,6 +2417,8 @@ test_util_lzma(void *arg) tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(LZMA_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); @@ -2507,6 +2512,10 @@ test_util_lzma(void *arg) tor_free(buf1); #else (void)arg; + tt_assert(! tor_compress_supports_method(LZMA_METHOD)); + + done: + ; #endif // HAVE_LZMA. } @@ -2520,6 +2529,8 @@ test_util_zstd(void *arg) tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(ZSTD_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); @@ -2613,6 +2624,10 @@ test_util_zstd(void *arg) tor_free(buf1); #else (void)arg; + tt_assert(! tor_compress_supports_method(ZSTD_METHOD)); + + done: + ; #endif // HAVE_ZSTD. }

1 0

[tor/master] Remove `tor_compress_memory_level()`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit cf912259ba491e51f6f211e186ff67605ff269c8 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon Apr 24 14:20:16 2017 +0200 Remove `tor_compress_memory_level()`. This patch splits up `tor_compress_memory_level()` into static functions in the individual compression backends, which allows us to tune the values per compression backend rather than globally. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 14 -------------- src/common/compress.h | 3 --- src/common/compress_lzma.c | 16 ++++++++++++++-- src/common/compress_zlib.c | 16 ++++++++++++++-- src/common/compress_zstd.c | 18 ++++++++++++++++-- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 8b49af8..38b8184 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -56,20 +56,6 @@ tor_compress_is_compression_bomb(size_t size_in, size_t size_out) return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); } -/** Given level return the memory level. The memory level is needed for - * the various compression backends used in Tor. - */ -int -tor_compress_memory_level(compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - /** Given in_len bytes at in, compress them into a newly * allocated buffer, using the method described in method. Store the * compressed string in *out, and its length in *out_len. diff --git a/src/common/compress.h b/src/common/compress.h index 182530f..1e6e9fd 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -47,9 +47,6 @@ tor_uncompress(char **out, size_t *out_len, compress_method_t detect_compression_method(const char *in, size_t in_len); int -tor_compress_memory_level(compression_level_t level); - -int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); int diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index f2952cc..ae0327f 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -26,6 +26,18 @@ static size_t total_lzma_allocation = 0; #ifdef HAVE_LZMA +/** Given level return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 6; + case LOW_COMPRESSION: return 3; + } +} + /** Convert a given error to a human readable error string. */ static const char * lzma_error_str(lzma_ret error) @@ -124,7 +136,7 @@ tor_lzma_compress(char **out, size_t *out_len, stream.avail_in = in_len; lzma_lzma_preset(&stream_options, - tor_compress_memory_level(HIGH_COMPRESSION)); + memory_level(HIGH_COMPRESSION)); retval = lzma_alone_encoder(&stream, &stream_options); @@ -432,7 +444,7 @@ tor_lzma_compress_new(int compress, if (compress) { lzma_lzma_preset(&stream_options, - tor_compress_memory_level(compression_level)); + memory_level(compression_level)); retval = lzma_alone_encoder(&result->stream, &stream_options); diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index e1a68c2..50bdf9e 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -50,6 +50,18 @@ static size_t tor_zlib_state_size_precalc(int inflate, /** Total number of bytes allocated for zlib state */ static size_t total_zlib_allocation = 0; +/** Given level return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + /** Return the 'bits' value to tell zlib to use method.*/ static inline int method_bits(compress_method_t method, compression_level_t level) @@ -120,7 +132,7 @@ tor_zlib_compress(char **out, size_t *out_len, if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, method_bits(method, HIGH_COMPRESSION), - tor_compress_memory_level(HIGH_COMPRESSION), + memory_level(HIGH_COMPRESSION), Z_DEFAULT_STRATEGY) != Z_OK) { //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. log_warn(LD_GENERAL, "Error from deflateInit2: %s", @@ -413,7 +425,7 @@ tor_zlib_compress_new(int compress, out->stream.opaque = NULL; out->compress = compress; bits = method_bits(method, compression_level); - memlevel = tor_compress_memory_level(compression_level); + memlevel = memory_level(compression_level); if (compress) { if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, bits, memlevel, diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index c838cd9..664cce1 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -26,6 +26,20 @@ /** Total number of bytes allocated for Zstandard state. */ static size_t total_zstd_allocation = 0; +#ifdef HAVE_ZSTD +/** Given level return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif // HAVE_ZSTD. + /** Return 1 if Zstandard compression is supported; otherwise 0. */ int tor_zstd_method_supported(void) @@ -104,7 +118,7 @@ tor_zstd_compress(char **out, size_t *out_len, } retval = ZSTD_initCStream(stream, - tor_compress_memory_level(HIGH_COMPRESSION)); + memory_level(HIGH_COMPRESSION)); if (ZSTD_isError(retval)) { log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", @@ -408,7 +422,7 @@ tor_zstd_compress_new(int compress, } retval = ZSTD_initCStream(result->u.compress_stream, - tor_compress_memory_level(compression_level)); + memory_level(compression_level)); if (ZSTD_isError(retval)) { log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",

1 0

← Newer
1
...
23
24
25
26
27
28
29
...
97
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits April 2017