tor-commits April 2017

tor-commits@lists.torproject.org

19 participants
966 discussions

[tor/master] Expose `tor_compress_is_compression_bomb()` as part of the public compression API.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit e6c6606a17841eaf263967254990db5e9443942d Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 03:12:19 2017 +0200 Expose `tor_compress_is_compression_bomb()` as part of the public compression API. This patch exposes the old `is_compression_bomb()` function as a public API as part of the compression module. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 8 ++++---- src/common/torgzip.h | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 09f3806..d6f6225 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -105,10 +105,10 @@ method_bits(compress_method_t method, compression_level_t level) #define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) /** @} */ -/** Return true if uncompressing an input of size in_size to an input - * of size at least size_out looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) +/** Return true if uncompressing an input of size in_size to an input of + * size at least size_out looks like a compression bomb. */ +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out) { if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) return 0; diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 8f2363a..fa56628 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -49,6 +49,9 @@ compress_method_t detect_compression_method(const char *in, size_t in_len); int tor_compress_memory_level(compression_level_t level); +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out); + /** Return values from tor_compress_process; see that function's documentation * for details. */ typedef enum {

1 0

[tor/master] Add `compress_zlib.obj` to `src/common/Makefile.nmake`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 300ac496853448eb3eeb3529d3aedaa8cc212ec3 Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 03:23:29 2017 +0200 Add `compress_zlib.obj` to `src/common/Makefile.nmake`. See https://bugs.torproject.org/21663 --- src/common/Makefile.nmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index 8667959..a1c819f 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj

1 0

[tor/master] Rename `write_to_buf_zlib()` to `write_to_buf_compress()`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit b8c9f229d747d4edc739198113dfb7e3818e23bc Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 01:23:39 2017 +0200 Rename `write_to_buf_zlib()` to `write_to_buf_compress()`. See https://bugs.torproject.org/21663 --- src/or/buffers.c | 6 +++--- src/or/buffers.h | 4 ++-- src/or/connection.c | 6 +++--- src/or/connection.h | 10 +++++----- src/or/directory.c | 13 +++++++------ src/or/dirserv.c | 13 +++++++------ src/test/test_buffers.c | 17 ++++++++++------- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 3490ce4..4f22935 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2092,9 +2092,9 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) * done is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, - const char *data, size_t data_len, - int done) +write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + int done) { char *next; size_t old_avail, avail; diff --git a/src/or/buffers.h b/src/or/buffers.h index 49a2ae4..23b58a5 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -36,8 +36,8 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, - const char *data, size_t data_len, int done); +int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); diff --git a/src/or/connection.c b/src/or/connection.c index 70b08e2..5fb2c53 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -4060,9 +4060,9 @@ connection_write_to_buf_impl_,(const char *string, size_t len, if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; - CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf, - dir_conn->compress_state, - string, len, done)); + CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf, + dir_conn->compress_state, + string, len, done)); } else { CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); } diff --git a/src/or/connection.h b/src/or/connection.h index df6fc64..36e45ae 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -141,17 +141,17 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); -/* DOCDOC connection_write_to_buf_zlib */ -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); +/* DOCDOC connection_write_to_buf_compress */ +static void connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done); static inline void connection_write_to_buf(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } static inline void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) +connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done) { connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1); } diff --git a/src/or/directory.c b/src/or/directory.c index 098683e..e0409e2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3280,8 +3280,8 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_zlib(c, strlen(c), conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress(c, strlen(c), conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf(c, strlen(c), TO_CONN(conn))); @@ -3523,10 +3523,11 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf(c->cache_info.signed_descriptor_body, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 2746d09..e76fd93 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3498,7 +3498,7 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_DONE; } if (conn->compress_state) { - connection_write_to_buf_zlib((const char*)body, bodylen, conn, 0); + connection_write_to_buf_compress((const char*)body, bodylen, conn, 0); } else { connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); } @@ -3524,8 +3524,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); if (conn->compress_state) { - connection_write_to_buf_zlib(cached->dir_z + spooled->cached_dir_offset, - bytes, conn, 0); + connection_write_to_buf_compress( + cached->dir_z + spooled->cached_dir_offset, + bytes, conn, 0); } else { connection_write_to_buf(cached->dir_z + spooled->cached_dir_offset, bytes, TO_CONN(conn)); @@ -3789,9 +3790,9 @@ connection_dirserv_flushed_some(dir_connection_t *conn) smartlist_free(conn->spool); conn->spool = NULL; if (conn->compress_state) { - /* Flush the zlib state: there could be more bytes pending in there, and - * we don't want to omit bytes. */ - connection_write_to_buf_zlib("", 0, conn, 1); + /* Flush the compression state: there could be more bytes pending in there, + * and we don't want to omit bytes. */ + connection_write_to_buf_compress("", 0, conn, 1); tor_compress_free(conn->compress_state); conn->compress_state = NULL; } diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index be757ae..43582d1 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -593,14 +593,17 @@ test_buffers_zlib_impl(int finalize_with_nil) msg = tor_malloc(512); crypto_rand(msg, 512); - tt_int_op(write_to_buf_zlib(buf, compress_state, msg, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, compress_state, msg+128, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, compress_state, msg+256, 256, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg+128, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg+256, 256, 0), OP_EQ, 0); done = !finalize_with_nil; - tt_int_op(write_to_buf_zlib(buf, compress_state, - "all done", 9, done), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + "all done", 9, done), OP_EQ, 0); if (finalize_with_nil) { - tt_int_op(write_to_buf_zlib(buf, compress_state, "", 0, 1), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); } in_len = buf_datalen(buf); @@ -668,7 +671,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ compress_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); - tt_int_op(write_to_buf_zlib(buf, compress_state, "", 0, 1), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); contents = tor_malloc(in_len);

1 0

[tor/master] Add --enable-zstd to our configure script.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit ce1feae9d9a4366e3cd93a68416841d0c6a636c5 Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 17:54:59 2017 +0200 Add --enable-zstd to our configure script. This patch adds support for enabling support for Zstandard to our configure script. By default, the --enable-zstd option is set to "auto" which means if libzstd is available we'll build Tor with Zstandard support. See: https://bugs.torproject.org/21662 --- Makefile.am | 2 +- configure.ac | 32 ++++++++++++++++++++++++++++++++ src/or/include.am | 4 ++-- src/test/fuzz/include.am | 3 ++- src/test/include.am | 8 ++++---- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/Makefile.am b/Makefile.am index 0f59ebe..be1dc7f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,7 +16,7 @@ noinst_PROGRAMS= DISTCLEANFILES= bin_SCRIPTS= AM_CPPFLAGS= -AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ +AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@ SHELL=@SHELL@ if COVERAGE_ENABLED diff --git a/configure.ac b/configure.ac index 96af8d7..c7960fa 100644 --- a/configure.ac +++ b/configure.ac @@ -764,6 +764,38 @@ fi AC_SUBST(TOR_LZMA_CFLAGS) AC_SUBST(TOR_LZMA_LIBS) +dnl ------------------------------------------------------ +dnl Where we do we find zstd? + +AC_ARG_ENABLE(zstd, + AS_HELP_STRING(--enable-zstd, [enable support for the Zstandard compression scheme.]), + [case "${enableval}" in + "yes") zstd=true ;; + "no") zstd=false ;; + * ) AC_MSG_ERROR(bad value for --enable-zstd) ;; + esac], [zstd=auto]) + +if test "x$enable_zstd" = "xno"; then + have_zstd=no; +else + PKG_CHECK_MODULES([ZSTD], + [libzstd], + have_zstd=yes, + have_zstd=no) + + if test "x$have_zstd" = "xno" ; then + AC_MSG_WARN([Unable to find libzstd.]) + fi +fi + +if test "x$have_zstd" = "xyes"; then + AC_DEFINE(HAVE_ZSTD,1,[Have Zstd]) + TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}" + TOR_ZSTD_LIBS="${ZSTD_LIBS}" +fi +AC_SUBST(TOR_ZSTD_CFLAGS) +AC_SUBST(TOR_ZSTD_LIBS) + dnl ---------------------------------------------------------------------- dnl Check if libcap is available for capabilities. diff --git a/src/or/include.am b/src/or/include.am index d4dc6a9..d001fcf 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -121,7 +121,7 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -134,7 +134,7 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ endif ORHEADERS = \ diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index abffcad..500377f 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -19,7 +19,8 @@ FUZZING_LIBS = \ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ \ + @TOR_ZSTD_LIBS@ oss-fuzz-prereqs: \ src/or/libtor-testing.a \ diff --git a/src/test/include.am b/src/test/include.am index 9b5eb14..2e02343 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -180,7 +180,7 @@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ src/common/libor-ctime-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -194,7 +194,7 @@ src_test_test_LDADD = src/or/libtor-testing.a \ src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) @@ -217,7 +217,7 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -228,7 +228,7 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS)

1 0

[tor/master] Add LZMA support.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit bf1c07cb07c27ca65ab66cf41442cfaabd9bb067 Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 14:30:44 2017 +0200 Add LZMA support. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 35 ++- src/common/compress.h | 6 +- src/common/compress_lzma.c | 582 +++++++++++++++++++++++++++++++++++++++++++++ src/common/compress_lzma.h | 56 +++++ src/common/include.am | 2 + src/test/test_util.c | 107 +++++++++ 6 files changed, 784 insertions(+), 4 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 4c87f29..3c7a3c6 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -23,6 +23,7 @@ #include "util.h" #include "torlog.h" #include "compress.h" +#include "compress_lzma.h" #include "compress_zlib.h" /** @{ */ @@ -81,6 +82,9 @@ tor_compress(char **out, size_t *out_len, if (method == GZIP_METHOD || method == ZLIB_METHOD) return tor_zlib_compress(out, out_len, in, in_len, method); + if (method == LZMA_METHOD) + return tor_lzma_compress(out, out_len, in, in_len, method); + return -1; } @@ -108,6 +112,12 @@ tor_uncompress(char **out, size_t *out_len, complete_only, protocol_warn_level); + if (method == LZMA_METHOD) + return tor_lzma_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + return -1; } @@ -123,6 +133,9 @@ detect_compression_method(const char *in, size_t in_len) } else if (in_len > 2 && (in[0] & 0x0f) == 8 && (ntohs(get_uint16(in)) % 31) == 0) { return ZLIB_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x5d\x00\x00\x00", 4)) { + return LZMA_METHOD; } else { return UNKNOWN_METHOD; } @@ -135,6 +148,7 @@ struct tor_compress_state_t { union { tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; } u; /**< Compression backend state. */ }; @@ -161,7 +175,16 @@ tor_compress_new(int compress, compress_method_t method, state->u.zlib_state = zlib_state; break; } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + + if (lzma_state == NULL) + goto err; + state->u.lzma_state = lzma_state; + break; + } case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -199,7 +222,10 @@ tor_compress_process(tor_compress_state_t *state, return tor_zlib_compress_process(state->u.zlib_state, out, out_len, in, in_len, finish); - + case LZMA_METHOD: + return tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -221,7 +247,9 @@ tor_compress_free(tor_compress_state_t *state) case ZLIB_METHOD: tor_zlib_compress_free(state->u.zlib_state); break; - + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; case NO_METHOD: case UNKNOWN_METHOD: break; @@ -240,7 +268,8 @@ tor_compress_state_size(const tor_compress_state_t *state) case GZIP_METHOD: case ZLIB_METHOD: return tor_zlib_compress_state_size(state->u.zlib_state); - + case LZMA_METHOD: + return tor_lzma_compress_state_size(state->u.lzma_state); case NO_METHOD: case UNKNOWN_METHOD: goto err; diff --git a/src/common/compress.h b/src/common/compress.h index c347e1c..3e3ab3a 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -15,7 +15,11 @@ * GZIP_METHOD is guaranteed to be supported by the compress/uncompress * functions here. */ typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 + NO_METHOD=0, + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + UNKNOWN_METHOD=4 } compress_method_t; /** diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000..e1a7a66 --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,582 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include <lzma.h> +#endif + +/** Total number of bytes allocated for LZMA state. */ +static size_t total_lzma_allocation = 0; + +#ifdef HAVE_LZMA +/** Convert a given error to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif // HAVE_LZMA. + +/** Return a string representation of the version of the currently running + * version of liblzma. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return "N/A"; +#endif +} + +/** Return a string representation of the version of the version of liblzma + * used at compilation. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return "N/A"; +#endif +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the LZMA method. Store the compressed string in + * *out, and its length in *out_len. Return 0 on success, -1 on + * failure. + */ +int +tor_lzma_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ +#ifdef HAVE_LZMA + lzma_stream stream = LZMA_STREAM_INIT; + lzma_options_lzma stream_options; + lzma_ret retval; + lzma_action action; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == LZMA_METHOD); + + stream.next_in = (unsigned char *)in; + stream.avail_in = in_len; + + lzma_lzma_preset(&stream_options, + tor_compress_memory_level(HIGH_COMPRESSION)); + + retval = lzma_alone_encoder(&stream, &stream_options); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + + out_size = in_len / 2; + if (out_size < 1024) + out_size = 1024; + + *out = tor_malloc(out_size); + + stream.next_out = (unsigned char *)*out; + stream.avail_out = out_size; + + action = LZMA_RUN; + + while (1) { + retval = lzma_code(&stream, action); + switch (retval) { + case LZMA_OK: + action = LZMA_FINISH; + break; + case LZMA_STREAM_END: + goto done; + case LZMA_BUF_ERROR: + offset = stream.next_out - ((unsigned char *)*out); + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in LZMA compression."); + goto err; + } + + *out = tor_realloc(*out, out_size); + stream.next_out = (unsigned char *)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " + "compressing."); + goto err; + } + stream.avail_out = (unsigned int)(out_size - offset); + break; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA compression didn't finish: %s.", + lzma_error_str(retval)); + goto err; + } + } + + done: + *out_len = stream.total_out; + lzma_end(&stream); + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tor instances would think " + "this is a compression bomb."); + goto err; + } + + return 0; + + err: + lzma_end(&stream); + tor_free(*out); + return -1; +#else // HAVE_LZMA. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + + return -1; +#endif // HAVE_LZMA. +} + +/** Given an LZMA compressed string of total length in_len bytes at + * in, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at protocol_warn_level. + */ +int +tor_lzma_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ +#ifdef HAVE_LZMA + lzma_stream stream = LZMA_STREAM_INIT; + lzma_ret retval; + lzma_action action; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == LZMA_METHOD); + + stream.next_in = (unsigned char *)in; + stream.avail_in = in_len; + + // FIXME(ahf): This should be something more sensible than + // UINT64_MAX: See #21665. + retval = lzma_alone_decoder(&stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + + out_size = in_len * 2; + if (out_size < 1024) + out_size = 1024; + + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + stream.next_out = (unsigned char *)*out; + stream.avail_out = out_size; + + // FIXME(ahf): We should figure out how to use LZMA_FULL_FLUSH to + // make the partial string read tests. + // action = complete_only ? LZMA_FINISH : LZMA_SYNC_FLUSH. // To do this, + // it seems like we have to use LZMA using their "xz" encoder instead of just + // regular LZMA. + (void)complete_only; + action = LZMA_FINISH; + + while (1) { + retval = lzma_code(&stream, action); + switch (retval) { + case LZMA_STREAM_END: + if (stream.avail_in == 0) + goto done; + + // We might have more data here. Reset our stream. + lzma_end(&stream); + + retval = lzma_alone_decoder(&stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + break; + case LZMA_OK: + break; + case LZMA_BUF_ERROR: + if (stream.avail_out > 0) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "possible truncated or corrupt LZMA data."); + goto err; + } + + offset = stream.next_out - (unsigned char *)*out; + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in LZMA uncompression."); + goto err; + } + + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible LZMA compression " + "bomb. Not proceeding."); + goto err; + } + + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " + "LZMA data."); + goto err; + } + + *out = tor_realloc(*out, out_size); + stream.next_out = (unsigned char *)(*out + offset); + + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " + "uncompressing."); + goto err; + } + + stream.avail_out = (unsigned int)(out_size - offset); + break; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA decompression didn't finish: %s.", + lzma_error_str(retval)); + goto err; + } + } + + done: + *out_len = stream.next_out - (unsigned char*)*out; + lzma_end(&stream); + + // NUL-terminate our output. + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + + err: + lzma_end(&stream); + tor_free(*out); + return -1; +#else // HAVE_LZMA. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + (void)complete_only; + (void)protocol_warn_level; + + return -1; +#endif // HAVE_LZMA. +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Construct and return a tor_lzma_compress_state_t object using + * method. If compress, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + + // FIXME(ahf): We should either try to do the pre-calculation that is done + // with the zlib backend or use a custom allocator here where we pass our + // tor_lzma_compress_state_t as the opaque value. + result->allocation = 0; + + if (compress) { + lzma_lzma_preset(&stream_options, + tor_compress_memory_level(compression_level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } else { + // FIXME(ahf): This should be something more sensible than + // UINT64_MAX: See #21665. + retval = lzma_alone_decoder(&result->stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } + + return result; + + err: + tor_free(result); + return NULL; +#else // HAVE_LZMA. + (void)compress; + (void)method; + (void)compression_level; + + return NULL; +#endif // HAVE_LZMA. +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else // HAVE_LZMA. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif // HAVE_LZMA. +} + +/** Deallocate state. */ +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + total_lzma_allocation -= state->allocation; + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return total_lzma_allocation; +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000..2cc3a00 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +const char * +tor_lzma_get_version_str(void); + +const char * +tor_lzma_get_header_version_str(void); + +int +tor_lzma_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_lzma_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state); + +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t +tor_lzma_get_total_allocation(void); + +#endif // TOR_COMPRESS_LZMA_H. + diff --git a/src/common/include.am b/src/common/include.am index 7ed75d9..2682c22 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -106,6 +106,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ src/common/compress.c \ + src/common/compress_lzma.c \ src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ @@ -147,6 +148,7 @@ COMMONHEADERS = \ src/common/compat_threads.h \ src/common/compat_time.h \ src/common/compress.h \ + src/common/compress_lzma.h \ src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ diff --git a/src/test/test_util.c b/src/test/test_util.c index dacf56f..287f7e8 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2404,6 +2404,112 @@ test_util_gzip_compression_bomb(void *arg) tor_compress_free(state); } +static void +test_util_lzma(void *arg) +{ +#ifdef HAVE_LZMA + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_compress_state_t *state = NULL; + + (void)arg; + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + LZMA_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, LZMA_METHOD); + + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + LZMA_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + +#if 0 + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_reallocarray(buf2, len1, 2); + memcpy(buf2+len1, buf2, len1); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + LZMA_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + LZMA_METHOD)); + tt_int_op(len1, OP_GT, 16); + /* when we allow an incomplete string, we should succeed.*/ + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + LZMA_METHOD, 0, LOG_INFO)); + tt_assert(len2 > 5); + buf3[len2]='\0'; + tt_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + LZMA_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); +#endif + + /* Now, try streaming compression. */ + state = tor_compress_new(1, LZMA_METHOD, HIGH_COMPRESSION); + tt_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ + tt_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); + tt_assert(cp1 > cp2); /* Make sure we really added something. */ + + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + LZMA_METHOD, 1, LOG_WARN)); + /* Make sure it compressed right. */ + tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); + tt_int_op(21, OP_EQ, len2); + + done: + if (state) + tor_compress_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +#else + (void)arg; +#endif // HAVE_LZMA. +} + /** Run unit tests for mmap() wrapper functionality. */ static void test_util_mmap(void *arg) @@ -5717,6 +5823,7 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(pow2), UTIL_LEGACY(gzip), UTIL_TEST(gzip_compression_bomb, TT_FORK), + UTIL_LEGACY(lzma), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats),

1 0

[tor/master] Use a switch-statement when checking for compression method.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit c171af048731cc00d77e17a1ff0be5c641024f8a Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 03:39:43 2017 +0200 Use a switch-statement when checking for compression method. This patch changes the way `tor_compress_new()`, `tor_compress_process()`, and `tor_compress_free()` handles different compression methods. This should give us compiler warnings in case an additional compression method is added, but the developer forgets to add handlers in the three aforementioned functions. See https://bugs.torproject.org/21663 --- src/common/compress.c | 60 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 74e235f..4c87f29 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -149,14 +149,22 @@ tor_compress_new(int compress, compress_method_t method, state = tor_malloc_zero(sizeof(tor_compress_state_t)); state->method = method; - if (method == GZIP_METHOD || method == ZLIB_METHOD) { - tor_zlib_compress_state_t *zlib_state = - tor_zlib_compress_new(compress, method, compression_level); + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); - if (zlib_state == NULL) - goto err; + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } - state->u.zlib_state = zlib_state; + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; } return state; @@ -185,11 +193,19 @@ tor_compress_process(tor_compress_state_t *state, { tor_assert(state != NULL); - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - return tor_zlib_compress_process(state->u.zlib_state, - out, out_len, in, in_len, - finish); + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; + } + + err: return TOR_COMPRESS_ERROR; } @@ -200,8 +216,16 @@ tor_compress_free(tor_compress_state_t *state) if (state == NULL) return; - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - tor_zlib_compress_free(state->u.zlib_state); + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } tor_free(state); } @@ -212,9 +236,17 @@ tor_compress_state_size(const tor_compress_state_t *state) { tor_assert(state != NULL); - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - return tor_zlib_compress_state_size(state->u.zlib_state); + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_compress_state_size(state->u.zlib_state); + + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; + } + err: return 0; }

1 0

[tor/master] Add --enable-lzma to our configure script.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 157af1d26e3cdfc3159c1085f35e5ddcda7ccd21 Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 13:56:02 2017 +0200 Add --enable-lzma to our configure script. This patch adds support for enabling support for LZMA to our configure script. By default, the --enable-lzma option is set to "auto" which means if liblzma is available we'll build Tor with LZMA support. See: https://bugs.torproject.org/21662 --- Makefile.am | 2 +- configure.ac | 32 ++++++++++++++++++++++++++++++++ src/or/include.am | 6 ++++-- src/test/fuzz/include.am | 3 ++- src/test/include.am | 16 ++++++++++------ 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index aeac0b8..0f59ebe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,7 +16,7 @@ noinst_PROGRAMS= DISTCLEANFILES= bin_SCRIPTS= AM_CPPFLAGS= -AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ +AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ SHELL=@SHELL@ if COVERAGE_ENABLED diff --git a/configure.ac b/configure.ac index 0d4fa98..96af8d7 100644 --- a/configure.ac +++ b/configure.ac @@ -732,6 +732,38 @@ else fi AC_SUBST(TOR_ZLIB_LIBS) +dnl ------------------------------------------------------ +dnl Where we do we find lzma? + +AC_ARG_ENABLE(lzma, + AS_HELP_STRING(--enable-lzma, [enable support for the Zstandard compression scheme.]), + [case "${enableval}" in + "yes") lzma=true ;; + "no") lzma=false ;; + * ) AC_MSG_ERROR(bad value for --enable-lzma) ;; + esac], [lzma=auto]) + +if test "x$enable_lzma" = "xno"; then + have_lzma=no; +else + PKG_CHECK_MODULES([LZMA], + [liblzma], + have_lzma=yes, + have_lzma=no) + + if test "x$have_lzma" = "xno" ; then + AC_MSG_WARN([Unable to find liblzma.]) + fi +fi + +if test "x$have_lzma" = "xyes"; then + AC_DEFINE(HAVE_LZMA,1,[Have LZMA]) + TOR_LZMA_CFLAGS="${LZMA_CFLAGS}" + TOR_LZMA_LIBS="${LZMA_LIBS}" +fi +AC_SUBST(TOR_LZMA_CFLAGS) +AC_SUBST(TOR_LZMA_LIBS) + dnl ---------------------------------------------------------------------- dnl Check if libcap is available for capabilities. diff --git a/src/or/include.am b/src/or/include.am index 1841bbf..d4dc6a9 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -120,7 +120,8 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -132,7 +133,8 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ endif ORHEADERS = \ diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 1b608c6..abffcad 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -18,7 +18,8 @@ FUZZING_LIBS = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ oss-fuzz-prereqs: \ src/or/libtor-testing.a \ diff --git a/src/test/include.am b/src/test/include.am index c92eab1..9b5eb14 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -179,7 +179,8 @@ src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ src/common/libor-ctime-testing.a \ - @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ + @TOR_LZMA_LIBS@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -193,7 +194,7 @@ src_test_test_LDADD = src/or/libtor-testing.a \ src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) @@ -216,7 +217,7 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -226,7 +227,8 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) @@ -236,7 +238,8 @@ src_test_test_timers_LDADD = \ src/common/libor-event-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS) noinst_HEADERS+= \ @@ -261,7 +264,8 @@ src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_ntor_cl_AM_CPPFLAGS = \ -I"$(top_srcdir)/src/or"

1 0

[tor/master] Fix whitespace in test_util_gzip() around `OP_*``.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 4b834e0d5e0374586823048b9c35daf079a59950 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon Apr 17 14:07:23 2017 +0200 Fix whitespace in test_util_gzip() around `OP_*``. See https://bugs.torproject.org/21663 --- src/test/test_util.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 4ad4ffa..19d1c79 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2264,8 +2264,8 @@ test_util_gzip(void *arg) tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, GZIP_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); tor_free(buf2); tor_free(buf3); @@ -2278,8 +2278,8 @@ test_util_gzip(void *arg) tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, ZLIB_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); /* Check whether we can uncompress concatenated, compressed strings. */ tor_free(buf3); @@ -2287,8 +2287,8 @@ test_util_gzip(void *arg) memcpy(buf2+len1, buf2, len1); tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, ZLIB_METHOD, 1, LOG_INFO)); - tt_int_op((strlen(buf1)+1)*2,OP_EQ, len2); - tt_mem_op(buf3,OP_EQ, + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", (strlen(buf1)+1)*2); @@ -2328,21 +2328,21 @@ test_util_gzip(void *arg) len2 = 21; tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) == TOR_ZLIB_OK); - tt_int_op(0,OP_EQ, len2); /* Make sure we compressed it all. */ + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ tt_assert(cp1 > buf1); len2 = 0; cp2 = cp1; tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) == TOR_ZLIB_DONE); - tt_int_op(0,OP_EQ, len2); + tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, ZLIB_METHOD, 1, LOG_WARN)); /* Make sure it compressed right. */ tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); - tt_int_op(21,OP_EQ, len2); + tt_int_op(21, OP_EQ, len2); done: if (state)

1 0

[tor/master] Rename the `torgzip` module to `compress`.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 04583df4520705bc171be9b720344167029292cb Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 03:21:53 2017 +0200 Rename the `torgzip` module to `compress`. See https://bugs.torproject.org/21663 --- src/common/Makefile.nmake | 2 +- src/common/compress.c | 220 +++++++++++++++++++++++++++++++++++++++++ src/common/compress.h | 74 ++++++++++++++ src/common/compress_zlib.c | 2 +- src/common/include.am | 4 +- src/common/torgzip.c | 220 ----------------------------------------- src/common/torgzip.h | 74 -------------- src/or/config.c | 2 +- src/or/or.h | 2 +- src/test/test.c | 2 +- src/test/test_dir_handle_get.c | 2 +- 11 files changed, 302 insertions(+), 302 deletions(-) diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4..8667959 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,7 +7,7 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj tortls.obj \ crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/compress.c b/src/common/compress.c new file mode 100644 index 0000000..74e235f --- /dev/null +++ b/src/common/compress.c @@ -0,0 +1,220 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.c + * \brief Common compression API. + **/ + +#include "orconfig.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include "torint.h" + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zlib.h" + +/** @{ */ +/* These macros define the maximum allowable compression factor. Anything of + * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to + * have an uncompression factor (uncompressed size:compressed size ratio) of + * any greater than MAX_UNCOMPRESSION_FACTOR. + * + * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to + * be small to limit the attack multiplier, but we also want it to be large + * enough so that no legitimate document --even ones we might invent in the + * future -- ever compresses by a factor of greater than + * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably + * large range of possible values. IMO, anything over 8 is probably safe; IMO + * anything under 50 is probably sufficient. + */ +#define MAX_UNCOMPRESSION_FACTOR 25 +#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ + +/** Return true if uncompressing an input of size in_size to an input of + * size at least size_out looks like a compression bomb. */ +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out) +{ + if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) + return 0; + + return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); +} + +/** Given level return the memory level. The memory level is needed for + * the various compression backends used in Tor. + */ +int +tor_compress_memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the method described in method. Store the + * compressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + */ +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_compress(out, out_len, in, in_len, method); + + return -1; +} + +/** Given zero or more zlib-compressed or gzip-compressed strings of + * total length + * in_len bytes at in, uncompress them into a newly allocated + * buffer, using the method described in method. Store the uncompressed + * string in *out, and its length in *out_len. Return 0 on + * success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a + * failure; otherwise we decompress as much as we can. Warn about truncated + * or corrupt inputs at protocol_warn_level. + */ +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + + return -1; +} + +/** Try to tell whether the in_len-byte string in in is likely + * to be compressed or not. If it is, return the likeliest compression method. + * Otherwise, return UNKNOWN_METHOD. + */ +compress_method_t +detect_compression_method(const char *in, size_t in_len) +{ + if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { + return GZIP_METHOD; + } else if (in_len > 2 && (in[0] & 0x0f) == 8 && + (ntohs(get_uint16(in)) % 31) == 0) { + return ZLIB_METHOD; + } else { + return UNKNOWN_METHOD; + } +} + +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { + compress_method_t method; /**< The compression method. */ + + union { + tor_zlib_compress_state_t *zlib_state; + } u; /**< Compression backend state. */ +}; + +/** Construct and return a tor_compress_state_t object using method. If + * compress, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress, compress_method_t method, + compression_level_t compression_level) +{ + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + if (method == GZIP_METHOD || method == ZLIB_METHOD) { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + } + + return state; + + err: + tor_free(state); + return NULL; +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + tor_assert(state != NULL); + + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + + return TOR_COMPRESS_ERROR; +} + +/** Deallocate state. */ +void +tor_compress_free(tor_compress_state_t *state) +{ + if (state == NULL) + return; + + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + tor_zlib_compress_free(state->u.zlib_state); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_compress_state_size(const tor_compress_state_t *state) +{ + tor_assert(state != NULL); + + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_state_size(state->u.zlib_state); + + return 0; +} + diff --git a/src/common/compress.h b/src/common/compress.h new file mode 100644 index 0000000..c347e1c --- /dev/null +++ b/src/common/compress.h @@ -0,0 +1,74 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.h + * \brief Headers for compress.c + **/ + +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H + +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. */ +typedef enum { + NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 +} compress_method_t; + +/** + * Enumeration to define tradeoffs between memory usage and compression level. + * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * memory. + **/ +typedef enum { + HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION +} compression_level_t; + +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +compress_method_t detect_compression_method(const char *in, size_t in_len); + +int +tor_compress_memory_level(compression_level_t level); + +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out); + +/** Return values from tor_compress_process; see that function's documentation + * for details. */ +typedef enum { + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; + +/** Internal state for an incremental compression/decompression. */ +typedef struct tor_compress_state_t tor_compress_state_t; + +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); + +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); + +size_t tor_compress_state_size(const tor_compress_state_t *state); + +#endif // TOR_COMPRESS_H. + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index 38e500c..2c9aba3 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -15,7 +15,7 @@ #include "util.h" #include "torlog.h" -#include "torgzip.h" +#include "compress.h" #include "compress_zlib.h" /* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of diff --git a/src/common/include.am b/src/common/include.am index ea2c46a..7ed75d9 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -105,12 +105,12 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress.c \ src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ src/common/crypto_format.c \ - src/common/torgzip.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -146,6 +146,7 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ @@ -165,7 +166,6 @@ COMMONHEADERS = \ src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/torgzip.c b/src/common/torgzip.c deleted file mode 100644 index d760cd9..0000000 --- a/src/common/torgzip.c +++ /dev/null @@ -1,220 +0,0 @@ -/* Copyright (c) 2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2017, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. - **/ - -#include "orconfig.h" - -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include "torint.h" - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#include "util.h" -#include "torlog.h" -#include "torgzip.h" -#include "compress_zlib.h" - -/** @{ */ -/* These macros define the maximum allowable compression factor. Anything of - * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to - * have an uncompression factor (uncompressed size:compressed size ratio) of - * any greater than MAX_UNCOMPRESSION_FACTOR. - * - * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to - * be small to limit the attack multiplier, but we also want it to be large - * enough so that no legitimate document --even ones we might invent in the - * future -- ever compresses by a factor of greater than - * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably - * large range of possible values. IMO, anything over 8 is probably safe; IMO - * anything under 50 is probably sufficient. - */ -#define MAX_UNCOMPRESSION_FACTOR 25 -#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) -/** @} */ - -/** Return true if uncompressing an input of size in_size to an input of - * size at least size_out looks like a compression bomb. */ -int -tor_compress_is_compression_bomb(size_t size_in, size_t size_out) -{ - if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) - return 0; - - return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); -} - -/** Given level return the memory level. The memory level is needed for - * the various compression backends used in Tor. - */ -int -tor_compress_memory_level(compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - -/** Given in_len bytes at in, compress them into a newly - * allocated buffer, using the method described in method. Store the - * compressed string in *out, and its length in *out_len. - * Return 0 on success, -1 on failure. - */ -int -tor_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - if (method == GZIP_METHOD || method == ZLIB_METHOD) - return tor_zlib_compress(out, out_len, in, in_len, method); - - return -1; -} - -/** Given zero or more zlib-compressed or gzip-compressed strings of - * total length - * in_len bytes at in, uncompress them into a newly allocated - * buffer, using the method described in method. Store the uncompressed - * string in *out, and its length in *out_len. Return 0 on - * success, -1 on failure. - * - * If complete_only is true, we consider a truncated input as a - * failure; otherwise we decompress as much as we can. Warn about truncated - * or corrupt inputs at protocol_warn_level. - */ -int -tor_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - if (method == GZIP_METHOD || method == ZLIB_METHOD) - return tor_zlib_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - - return -1; -} - -/** Try to tell whether the in_len-byte string in in is likely - * to be compressed or not. If it is, return the likeliest compression method. - * Otherwise, return UNKNOWN_METHOD. - */ -compress_method_t -detect_compression_method(const char *in, size_t in_len) -{ - if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { - return GZIP_METHOD; - } else if (in_len > 2 && (in[0] & 0x0f) == 8 && - (ntohs(get_uint16(in)) % 31) == 0) { - return ZLIB_METHOD; - } else { - return UNKNOWN_METHOD; - } -} - -/** Internal state for an incremental compression/decompression. The body of - * this struct is not exposed. */ -struct tor_compress_state_t { - compress_method_t method; /**< The compression method. */ - - union { - tor_zlib_compress_state_t *zlib_state; - } u; /**< Compression backend state. */ -}; - -/** Construct and return a tor_compress_state_t object using method. If - * compress, it's for compression; otherwise it's for decompression. */ -tor_compress_state_t * -tor_compress_new(int compress, compress_method_t method, - compression_level_t compression_level) -{ - tor_compress_state_t *state; - - state = tor_malloc_zero(sizeof(tor_compress_state_t)); - state->method = method; - - if (method == GZIP_METHOD || method == ZLIB_METHOD) { - tor_zlib_compress_state_t *zlib_state = - tor_zlib_compress_new(compress, method, compression_level); - - if (zlib_state == NULL) - goto err; - - state->u.zlib_state = zlib_state; - } - - return state; - - err: - tor_free(state); - return NULL; -} - -/** Compress/decompress some bytes using state. Read up to - * *in_len bytes from *in, and write up to *out_len bytes - * to *out, adjusting the values as we go. If finish is true, - * we've reached the end of the input. - * - * Return TOR_COMPRESS_DONE if we've finished the entire - * compression/decompression. - * Return TOR_COMPRESS_OK if we're processed everything from the input. - * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. - * Return TOR_COMPRESS_ERROR if the stream is corrupt. - */ -tor_compress_output_t -tor_compress_process(tor_compress_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) -{ - tor_assert(state != NULL); - - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - return tor_zlib_compress_process(state->u.zlib_state, - out, out_len, in, in_len, - finish); - - return TOR_COMPRESS_ERROR; -} - -/** Deallocate state. */ -void -tor_compress_free(tor_compress_state_t *state) -{ - if (state == NULL) - return; - - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - tor_zlib_compress_free(state->u.zlib_state); - - tor_free(state); -} - -/** Return the approximate number of bytes allocated for state. */ -size_t -tor_compress_state_size(const tor_compress_state_t *state) -{ - tor_assert(state != NULL); - - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - return tor_zlib_compress_state_size(state->u.zlib_state); - - return 0; -} - diff --git a/src/common/torgzip.h b/src/common/torgzip.h deleted file mode 100644 index 0fcac0c..0000000 --- a/src/common/torgzip.h +++ /dev/null @@ -1,74 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2017, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.h - * \brief Headers for torgzip.h - **/ - -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H - -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and - * GZIP_METHOD is guaranteed to be supported by the compress/uncompress - * functions here. */ -typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 -} compress_method_t; - -/** - * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most - * memory. - **/ -typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} compression_level_t; - -int -tor_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - -compress_method_t detect_compression_method(const char *in, size_t in_len); - -int -tor_compress_memory_level(compression_level_t level); - -int -tor_compress_is_compression_bomb(size_t size_in, size_t size_out); - -/** Return values from tor_compress_process; see that function's documentation - * for details. */ -typedef enum { - TOR_COMPRESS_OK, - TOR_COMPRESS_DONE, - TOR_COMPRESS_BUFFER_FULL, - TOR_COMPRESS_ERROR -} tor_compress_output_t; - -/** Internal state for an incremental compression/decompression. */ -typedef struct tor_compress_state_t tor_compress_state_t; - -tor_compress_state_t *tor_compress_new(int compress, - compress_method_t method, - compression_level_t level); - -tor_compress_output_t tor_compress_process(tor_compress_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_compress_free(tor_compress_state_t *state); - -size_t tor_compress_state_size(const tor_compress_state_t *state); - -#endif - diff --git a/src/or/config.c b/src/or/config.c index 83e5f21..9af116d 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress.h" #include "compress_zlib.h" #include "config.h" #include "connection.h" @@ -100,7 +101,6 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" -#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif diff --git a/src/or/or.h b/src/or/or.h index 20c4ebc..9e3e409 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -71,7 +71,7 @@ #include "tortls.h" #include "torlog.h" #include "container.h" -#include "torgzip.h" +#include "compress.h" #include "address.h" #include "compat_libevent.h" #include "ht.h" diff --git a/src/test/test.c b/src/test/test.c index 77ea449..e8a63e6 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -44,13 +44,13 @@ double fabs(double x); #include "buffers.h" #include "circuitlist.h" #include "circuitstats.h" +#include "compress.h" #include "config.h" #include "connection_edge.h" #include "geoip.h" #include "rendcommon.h" #include "rendcache.h" #include "test.h" -#include "torgzip.h" #include "main.h" #include "memarea.h" #include "onion.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index ed8ea2f..392fa4d 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -14,6 +14,7 @@ #include "connection.h" #include "directory.h" #include "test.h" +#include "compress.h" #include "connection.h" #include "rendcommon.h" #include "rendcache.h" @@ -28,7 +29,6 @@ #include "networkstatus.h" #include "geoip.h" #include "dirserv.h" -#include "torgzip.h" #include "dirvote.h" #include "log_test_helpers.h"

1 0

[tor/master] Expose `tor_compress_memory_level()` as a public function in the compression module.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 6b5172bcf2018f858d38f5dea13753b8238b15ae Author: Alexander Færøy <ahf(a)torproject.org> Date: Tue Apr 18 02:23:25 2017 +0200 Expose `tor_compress_memory_level()` as a public function in the compression module. This patch makes the internal `get_memlevel()` a part of the public compression API as `tor_compress_memory_level()`. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 24 ++++++++++++++---------- src/common/torgzip.h | 3 +++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index af4bbc5..09f3806 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -86,16 +86,6 @@ method_bits(compress_method_t method, compression_level_t level) } } -static inline int -get_memlevel(compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} /** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -126,6 +116,20 @@ is_compression_bomb(size_t size_in, size_t size_out) return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); } +/** Given level return the memory level. The memory level is needed for + * the various compression backends used in Tor. + */ +int +tor_compress_memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + /** Given in_len bytes at in, compress them into a newly * allocated buffer, using the method described in method. Store the * compressed string in *out, and its length in *out_len. diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 2b0504e..8f2363a 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -46,6 +46,9 @@ tor_zlib_get_header_version_str(void); compress_method_t detect_compression_method(const char *in, size_t in_len); +int +tor_compress_memory_level(compression_level_t level); + /** Return values from tor_compress_process; see that function's documentation * for details. */ typedef enum {

1 0

← Newer
1
...
22
23
24
25
26
27
28
...
97
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits April 2017