April 2017 - tor-commits - lists.torproject.org

[translation/tor-browser-manual] Update translations for tor-browser-manual
by translation＠torproject.org 25 Apr '17

25 Apr '17

commit 19a445414513c4eacf20ca28151f9cc366682d1a Author: Translation commit bot <translation(a)torproject.org> Date: Tue Apr 25 15:18:16 2017 +0000 Update translations for tor-browser-manual --- de/de.po | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/de/de.po b/de/de.po index 9599fdf..35dc48a 100644 --- a/de/de.po +++ b/de/de.po @@ -1376,6 +1376,10 @@ msgid "" "Delete Tor Browser and install it again. If updating, do not just overwrite " "your previous Tor Browser files; ensure they are fully deleted beforehand." msgstr "" +"Löschen sie den Tor-Browser und installieren sie ihn nochmals. Wenn sie den " +"Browser aktualisieren, löschen sie die alten Dateien, des Tor-Browsers. " +"Stellen sie sicher, dass sie vollständig gelöscht sind und nicht einfach " +"überschrieben werden." #: troubleshooting.page:64 msgid "Is your connection censored?" @@ -1426,6 +1430,10 @@ msgid "" "default location, however the folder will be named \"tor-browser_en-US\" if " "you are running the English Tor Browser." msgstr "" +"Platzieren sie den Ordner für den Tor-Browser. Der Standard Ort befindet " +"sich unter Windows auf dem Desktop, und auf Mac OS X im Ordner Programme. " +"Auf Linux ist kein Standard Ort festgelegt, jedoch wird der Ordner bei " +"Verwendung des englischen Tor-Browsers \"tor-browser_en-US\" genannt werden." #: uninstalling.page:32 msgid "Delete the Tor Browser folder." @@ -1456,7 +1464,7 @@ msgid "" "outdated version of the software, you may be vulnerable to serious security " "flaws that compromise your privacy and anonymity." msgstr "" -"Der Tor Browser muss immer auf dem neusten Stand gehalten werden. Wenn sie " +"Der Tor-Browser muss immer auf dem neusten Stand gehalten werden. Wenn sie " "weiterhin eine veraltete Version des Browsers verwenden, könnte ihre " "Privatsphäre und Anonymität durch ernsthafte Sicherheitslücken gefährdet " "werden." @@ -1468,6 +1476,11 @@ msgid "" "may see a written update indicator when Tor Browser opens. You can update " "either automatically or manually." msgstr "" +"Der Tor-Browser wird sie auffordern, die Software zu aktualisieren, sobald " +"eine neue Version veröffentlicht wurde. Beim öffnen des Tor-Browsers, wird " +"der Torbutton ein gelbes Dreieck anzeigen, und sie werden einen " +"schriftlichen Aktualisierungshinweis sehen können. Sie können den Tor-" +"Browser automatisch oder manuell aktualisieren. " #: updating.page:26 msgid "Updating Tor Browser automatically"

1 0

[tor/master] Teach cov-exclude to detect runaway LCOV_EXCL_START lines
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 43db91bd87a6f197a93285a87df473616cc1f6c2 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 10:59:19 2017 -0400 Teach cov-exclude to detect runaway LCOV_EXCL_START lines Also, fix two instances of runaway LCOV_EXCL_START lines. --- scripts/test/cov-exclude | 6 ++++++ src/common/address.c | 2 +- src/or/onion_tap.c | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/test/cov-exclude b/scripts/test/cov-exclude index 5117f11..5cb9b12 100755 --- a/scripts/test/cov-exclude +++ b/scripts/test/cov-exclude @@ -26,3 +26,9 @@ if ($excluding or $exclude_this) { s{^\s*\#\#+:}{ x:}; s{^ (\s*)(\d+):}{$1!!!$2:}; } + +if (eof and $excluding) { + warn "Runaway LCOV_EXCL_START in $ARGV"; + $excluding = 0; +} + diff --git a/src/common/address.c b/src/common/address.c index 701daac..c5d0e91 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1198,7 +1198,7 @@ tor_addr_hash(const tor_addr_t *addr) /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; - /* LCOV_EXCL_END */ + /* LCOV_EXCL_STOP */ } } diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 9a65fcf..294fc0d 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -159,7 +159,7 @@ onion_skin_TAP_server_handshake( * big. That should be impossible. */ log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; - /* LCOV_EXCP_STOP */ + /* LCOV_EXCL_STOP */ } key_material_len = DIGEST_LEN+key_out_len;

1 0

[tor/master] Whitespace fixes.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 7a37cbbea33896da54ea4785d8b08c3e5752dcd2 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 10:54:34 2017 -0400 Whitespace fixes. --- src/common/compat_threads.c | 1 + src/common/compress.c | 1 + src/common/compress_lzma.c | 1 + src/common/compress_zlib.c | 1 + src/common/compress_zstd.c | 1 + 5 files changed, 5 insertions(+) diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index 8d026dd..d3f64f2 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -397,3 +397,4 @@ atomic_counter_get(atomic_counter_t *counter) tor_mutex_release(&counter->mutex); return val; } + diff --git a/src/common/compress.c b/src/common/compress.c index fd2cb06..5a3359a 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -495,3 +495,4 @@ tor_compress_init(void) tor_lzma_init(); tor_zstd_init(); } + diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index 906a406..953971b 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -321,3 +321,4 @@ tor_lzma_init(void) { atomic_counter_init(&total_lzma_allocation); } + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index 3fc574c..284542e 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -301,3 +301,4 @@ tor_zlib_init(void) { atomic_counter_init(&total_zlib_allocation); } + diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 8b8aea1..deaefc1 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -332,3 +332,4 @@ tor_zstd_init(void) { atomic_counter_init(&total_zstd_allocation); } +

1 0

[tor/master] Combine all *compress/*uncompress backend function into one
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 880fb3e3a9aba21c0b36aa4aa5659432e90eb827 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 09:41:23 2017 -0400 Combine all *compress/*uncompress backend function into one Since we have a streaming API for each compression backend, we don't need a non-streaming API for each: we can build a common non-streaming API at the front-end. --- src/common/compress.c | 182 ++++++++++++++++++++++------ src/common/compress_lzma.c | 290 --------------------------------------------- src/common/compress_lzma.h | 10 -- src/common/compress_zlib.c | 254 --------------------------------------- src/common/compress_zlib.h | 10 -- src/common/compress_zstd.c | 283 ------------------------------------------- src/common/compress_zstd.h | 10 -- src/test/test_util.c | 2 +- 8 files changed, 148 insertions(+), 893 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 38b8184..2729d9f 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -56,6 +56,147 @@ tor_compress_is_compression_bomb(size_t size_in, size_t size_out) return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); } +/** Guess the size that in_len will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)method; + (void)compression_level; + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether compress is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) + return -1; + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + goto err; + } else { + goto done; + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompresing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompresing data: bad input?", + compress?"":"un"); + goto err; // bad data. + default: + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + /** Given in_len bytes at in, compress them into a newly * allocated buffer, using the method described in method. Store the * compressed string in *out, and its length in *out_len. @@ -66,19 +207,9 @@ tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method) { - switch (method) { - case GZIP_METHOD: - case ZLIB_METHOD: - return tor_zlib_compress(out, out_len, in, in_len, method); - case LZMA_METHOD: - return tor_lzma_compress(out, out_len, in, in_len, method); - case ZSTD_METHOD: - return tor_zstd_compress(out, out_len, in, in_len, method); - case NO_METHOD: - case UNKNOWN_METHOD: - default: - return -1; - } + return tor_compress_impl(1, out, out_len, in, in_len, method, + HIGH_COMPRESSION, + 1, LOG_WARN); // XXXX "best"? } /** Given zero or more zlib-compressed or gzip-compressed strings of @@ -99,28 +230,9 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level) { - switch (method) { - case GZIP_METHOD: - case ZLIB_METHOD: - return tor_zlib_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - case LZMA_METHOD: - return tor_lzma_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - case ZSTD_METHOD: - return tor_zstd_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - case NO_METHOD: - case UNKNOWN_METHOD: - default: - return -1; - } + return tor_compress_impl(0, out, out_len, in, in_len, method, + HIGH_COMPRESSION, + complete_only, protocol_warn_level); } /** Try to tell whether the in_len-byte string in in is likely diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index ae0327f..9c61d58 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -108,296 +108,6 @@ tor_lzma_get_header_version_str(void) #endif } -/** Given in_len bytes at in, compress them into a newly - * allocated buffer, using the LZMA method. Store the compressed string in - * *out, and its length in *out_len. Return 0 on success, -1 on - * failure. - */ -int -tor_lzma_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ -#ifdef HAVE_LZMA - lzma_stream stream = LZMA_STREAM_INIT; - lzma_options_lzma stream_options; - lzma_ret retval; - lzma_action action; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == LZMA_METHOD); - - stream.next_in = (unsigned char *)in; - stream.avail_in = in_len; - - lzma_lzma_preset(&stream_options, - memory_level(HIGH_COMPRESSION)); - - retval = lzma_alone_encoder(&stream, &stream_options); - - if (retval != LZMA_OK) { - log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", - lzma_error_str(retval), retval); - goto err; - } - - out_size = in_len / 2; - if (out_size < 1024) - out_size = 1024; - - *out = tor_malloc(out_size); - - stream.next_out = (unsigned char *)*out; - stream.avail_out = out_size; - - action = LZMA_RUN; - - while (1) { - retval = lzma_code(&stream, action); - switch (retval) { - case LZMA_OK: - action = LZMA_FINISH; - break; - case LZMA_STREAM_END: - goto done; - case LZMA_BUF_ERROR: - offset = stream.next_out - ((unsigned char *)*out); - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in LZMA compression."); - goto err; - } - - *out = tor_realloc(*out, out_size); - stream.next_out = (unsigned char *)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " - "compressing."); - goto err; - } - stream.avail_out = (unsigned int)(out_size - offset); - break; - - // We list all the possible values of `lzma_ret` here to silence the - // `switch-enum` warning and to detect if a new member was added. - case LZMA_NO_CHECK: - case LZMA_UNSUPPORTED_CHECK: - case LZMA_GET_CHECK: - case LZMA_MEM_ERROR: - case LZMA_MEMLIMIT_ERROR: - case LZMA_FORMAT_ERROR: - case LZMA_OPTIONS_ERROR: - case LZMA_DATA_ERROR: - case LZMA_PROG_ERROR: - default: - log_warn(LD_GENERAL, "LZMA compression didn't finish: %s.", - lzma_error_str(retval)); - goto err; - } - } - - done: - *out_len = stream.total_out; - lzma_end(&stream); - - if (tor_compress_is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tor instances would think " - "this is a compression bomb."); - goto err; - } - - return 0; - - err: - lzma_end(&stream); - tor_free(*out); - return -1; -#else // HAVE_LZMA. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - - return -1; -#endif // HAVE_LZMA. -} - -/** Given an LZMA compressed string of total length in_len bytes at - * in, uncompress them into a newly allocated buffer. Store the - * uncompressed string in *out, and its length in *out_len. - * Return 0 on success, -1 on failure. - * - * If complete_only is true, we consider a truncated input as a failure; - * otherwise we decompress as much as we can. Warn about truncated or corrupt - * inputs at protocol_warn_level. - */ -int -tor_lzma_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ -#ifdef HAVE_LZMA - lzma_stream stream = LZMA_STREAM_INIT; - lzma_ret retval; - lzma_action action; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == LZMA_METHOD); - - stream.next_in = (unsigned char *)in; - stream.avail_in = in_len; - - // FIXME(ahf): This should be something more sensible than - // UINT64_MAX: See #21665. - retval = lzma_alone_decoder(&stream, UINT64_MAX); - - if (retval != LZMA_OK) { - log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", - lzma_error_str(retval), retval); - goto err; - } - - out_size = in_len * 2; - if (out_size < 1024) - out_size = 1024; - - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream.next_out = (unsigned char *)*out; - stream.avail_out = out_size; - - // FIXME(ahf): We should figure out how to use LZMA_FULL_FLUSH to - // make the partial string read tests. - // action = complete_only ? LZMA_FINISH : LZMA_SYNC_FLUSH. // To do this, - // it seems like we have to use LZMA using their "xz" encoder instead of just - // regular LZMA. - (void)complete_only; - action = LZMA_FINISH; - - while (1) { - retval = lzma_code(&stream, action); - switch (retval) { - case LZMA_STREAM_END: - if (stream.avail_in == 0) - goto done; - - // We might have more data here. Reset our stream. - lzma_end(&stream); - - retval = lzma_alone_decoder(&stream, UINT64_MAX); - - if (retval != LZMA_OK) { - log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", - lzma_error_str(retval), retval); - goto err; - } - break; - case LZMA_OK: - break; - case LZMA_BUF_ERROR: - if (stream.avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt LZMA data."); - goto err; - } - - offset = stream.next_out - (unsigned char *)*out; - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in LZMA uncompression."); - goto err; - } - - if (tor_compress_is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible LZMA compression " - "bomb. Not proceeding."); - goto err; - } - - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " - "LZMA data."); - goto err; - } - - *out = tor_realloc(*out, out_size); - stream.next_out = (unsigned char *)(*out + offset); - - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " - "uncompressing."); - goto err; - } - - stream.avail_out = (unsigned int)(out_size - offset); - break; - - // We list all the possible values of `lzma_ret` here to silence the - // `switch-enum` warning and to detect if a new member was added. - case LZMA_NO_CHECK: - case LZMA_UNSUPPORTED_CHECK: - case LZMA_GET_CHECK: - case LZMA_MEM_ERROR: - case LZMA_MEMLIMIT_ERROR: - case LZMA_FORMAT_ERROR: - case LZMA_OPTIONS_ERROR: - case LZMA_DATA_ERROR: - case LZMA_PROG_ERROR: - default: - log_warn(LD_GENERAL, "LZMA decompression didn't finish: %s.", - lzma_error_str(retval)); - goto err; - } - } - - done: - *out_len = stream.next_out - (unsigned char*)*out; - lzma_end(&stream); - - // NUL-terminate our output. - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - - err: - lzma_end(&stream); - tor_free(*out); - return -1; -#else // HAVE_LZMA. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - (void)complete_only; - (void)protocol_warn_level; - - return -1; -#endif // HAVE_LZMA. -} - /** Internal LZMA state for incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_lzma_compress_state_t { diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h index 71de56a..79cf9dc 100644 --- a/src/common/compress_lzma.h +++ b/src/common/compress_lzma.h @@ -17,16 +17,6 @@ const char *tor_lzma_get_version_str(void); const char *tor_lzma_get_header_version_str(void); -int tor_lzma_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); - -int tor_lzma_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - /** Internal state for an incremental LZMA compression/decompression. */ typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index faa179f..37768ee 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -102,260 +102,6 @@ tor_zlib_get_header_version_str(void) return ZLIB_VERSION; } -/** Given in_len bytes at in, compress them into a newly - * allocated buffer, using the method described in method. Store the - * compressed string in *out, and its length in *out_len. Return - * 0 on success, -1 on failure. - */ -int -tor_zlib_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - memory_level(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (tor_compress_is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given an Zlib/Gzip compressed string of total length in_len bytes - * at in, uncompress them into a newly allocated buffer. Store the - * uncompressed string in *out, and its length in *out_len. - * Return 0 on success, -1 on failure. - * - * If complete_only is true, we consider a truncated input as a failure; - * otherwise we decompress as much as we can. Warn about truncated or corrupt - * inputs at protocol_warn_level. - */ -int -tor_zlib_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (tor_compress_is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - /** Internal zlib state for an incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_zlib_compress_state_t { diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h index 6e8e5c5..2dd1254 100644 --- a/src/common/compress_zlib.h +++ b/src/common/compress_zlib.h @@ -17,16 +17,6 @@ const char *tor_zlib_get_version_str(void); const char *tor_zlib_get_header_version_str(void); -int tor_zlib_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); - -int tor_zlib_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - /** Internal state for an incremental zlib/gzip compression/decompression. */ typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 664cce1..76f2991 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -85,289 +85,6 @@ tor_zstd_get_header_version_str(void) #endif } -/** Given in_len bytes at in, compress them into a newly - * allocated buffer, using the Zstandard method. Store the compressed string - * in *out, and its length in *out_len. Return 0 on success, -1 - * on failure. - */ -int -tor_zstd_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ -#ifdef HAVE_ZSTD - ZSTD_CStream *stream = NULL; - size_t out_size, old_size; - size_t retval; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == ZSTD_METHOD); - - *out = NULL; - - stream = ZSTD_createCStream(); - - if (stream == NULL) { - // Zstandard does not give us any useful error message to why this - // happened. See https://github.com/facebook/zstd/issues/398 - log_warn(LD_GENERAL, "Error while creating Zstandard stream"); - goto err; - } - - retval = ZSTD_initCStream(stream, - memory_level(HIGH_COMPRESSION)); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - // Assume 50% compression and update our buffer in case we need to. - out_size = in_len / 2; - if (out_size < 1024) - out_size = 1024; - - *out = tor_malloc(out_size); - *out_len = 0; - - ZSTD_inBuffer input = { in, in_len, 0 }; - ZSTD_outBuffer output = { *out, out_size, 0 }; - - while (input.pos < input.size) { - retval = ZSTD_compressStream(stream, &output, &input); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream compression error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - if (input.pos < input.size && output.pos == output.size) { - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); - goto err; - } - - if (out_size - output.pos > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " - "compressing."); - goto err; - } - - output.dst = *out = tor_realloc(*out, out_size); - output.size = out_size; - } - } - - while (1) { - retval = ZSTD_endStream(stream, &output); - - if (retval == 0) - break; - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - if (output.pos == output.size) { - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); - goto err; - } - - if (out_size - output.pos > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " - "compressing."); - goto err; - } - - output.dst = *out = tor_realloc(*out, out_size); - output.size = out_size; - } - } - - *out_len = output.pos; - - if (tor_compress_is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tor instances would think " - "this is a compression bomb."); - goto err; - } - - if (stream != NULL) { - ZSTD_freeCStream(stream); - } - - return 0; - - err: - if (stream != NULL) { - ZSTD_freeCStream(stream); - } - - tor_free(*out); - return -1; -#else // HAVE_ZSTD. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - - return -1; -#endif // HAVE_ZSTD. -} - -/** Given a Zstandard compressed string of total length in_len bytes at - * in, uncompress them into a newly allocated buffer. Store the - * uncompressed string in *out, and its length in *out_len. - * Return 0 on success, -1 on failure. - * - * If complete_only is true, we consider a truncated input as a failure; - * otherwise we decompress as much as we can. Warn about truncated or corrupt - * inputs at protocol_warn_level. - */ -int -tor_zstd_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ -#ifdef HAVE_ZSTD - ZSTD_DStream *stream = NULL; - size_t retval; - size_t out_size, old_size; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - tor_assert(method == ZSTD_METHOD); - - // FIXME(ahf): Handle this? - (void)complete_only; - (void)protocol_warn_level; - - *out = NULL; - - stream = ZSTD_createDStream(); - - if (stream == NULL) { - // Zstandard does not give us any useful error message to why this - // happened. See https://github.com/facebook/zstd/issues/398 - log_warn(LD_GENERAL, "Error while creating Zstandard stream"); - goto err; - } - - retval = ZSTD_initDStream(stream); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - out_size = in_len * 2; - if (out_size < 1024) - out_size = 1024; - - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - *out_len = 0; - - ZSTD_inBuffer input = { in, in_len, 0 }; - ZSTD_outBuffer output = { *out, out_size, 0 }; - - while (input.pos < input.size) { - retval = ZSTD_decompressStream(stream, &output, &input); - - if (ZSTD_isError(retval)) { - log_warn(LD_GENERAL, "Zstandard stream decompression error: %s", - ZSTD_getErrorName(retval)); - goto err; - } - - if (input.pos < input.size && output.pos == output.size) { - old_size = out_size; - out_size *= 2; - - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); - goto err; - } - - if (tor_compress_is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible Zstandard " - "compression bomb. Not proceeding."); - goto err; - } - - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " - "Zstandard data."); - goto err; - } - - if (out_size - output.pos > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " - "decompressing."); - goto err; - } - - output.dst = *out = tor_realloc(*out, out_size); - output.size = out_size; - } - } - - *out_len = output.pos; - - if (stream != NULL) { - ZSTD_freeDStream(stream); - } - - // NUL-terminate our output. - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - - err: - if (stream != NULL) { - ZSTD_freeDStream(stream); - } - - tor_free(*out); - return -1; -#else // HAVE_ZSTD. - (void)out; - (void)out_len; - (void)in; - (void)in_len; - (void)method; - (void)complete_only; - (void)protocol_warn_level; - - return -1; -#endif // HAVE_ZSTD. -} - /** Internal Zstandard state for incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_zstd_compress_state_t { diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index 663cbdd..24e287c 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -17,16 +17,6 @@ const char *tor_zstd_get_version_str(void); const char *tor_zstd_get_header_version_str(void); -int tor_zstd_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); - -int tor_zstd_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - /** Internal state for an incremental Zstandard compression/decompression. */ typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; diff --git a/src/test/test_util.c b/src/test/test_util.c index 1e33de8..d47475a 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2377,7 +2377,7 @@ test_util_gzip_compression_bomb(void *arg) expect_single_log_msg_containing( "We compressed something and got an insanely high " "compression factor; other Tors would think this " - "was a zlib bomb."); + "was a compression bomb."); teardown_capture_of_logs(); /* Here's a compression bomb that we made manually. */

1 0

[tor/master] zlib: Turn UINT_MAX overrun into an error, not an assert.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 49a5b50b31b8305e63dea5db0cc0de1cb093ff4f Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 09:55:38 2017 -0400 zlib: Turn UINT_MAX overrun into an error, not an assert. --- src/common/compress_zlib.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index d8200d5..7e848d5 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -213,8 +213,11 @@ tor_zlib_compress_process(tor_zlib_compress_state_t *state, { int err; tor_assert(state != NULL); - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); + if (*in_len > UINT_MAX || + *out_len > UINT_MAX) { + return TOR_COMPRESS_ERROR; + } + state->stream.next_in = (unsigned char*) *in; state->stream.avail_in = (unsigned int)*in_len; state->stream.next_out = (unsigned char*) *out;

1 0

[tor/master] Add a "best compression" flag.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 4b01b45ec1a2d561eab8bf7d8934857268e48701 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 09:54:02 2017 -0400 Add a "best compression" flag. --- src/common/compress.c | 6 +++--- src/common/compress.h | 4 ++-- src/common/compress_lzma.c | 1 + src/common/compress_zlib.c | 4 +++- src/common/compress_zstd.c | 1 + 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 2729d9f..e8d11b9 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -208,8 +208,8 @@ tor_compress(char **out, size_t *out_len, compress_method_t method) { return tor_compress_impl(1, out, out_len, in, in_len, method, - HIGH_COMPRESSION, - 1, LOG_WARN); // XXXX "best"? + BEST_COMPRESSION, + 1, LOG_WARN); } /** Given zero or more zlib-compressed or gzip-compressed strings of @@ -231,7 +231,7 @@ tor_uncompress(char **out, size_t *out_len, int protocol_warn_level) { return tor_compress_impl(0, out, out_len, in, in_len, method, - HIGH_COMPRESSION, + BEST_COMPRESSION, complete_only, protocol_warn_level); } diff --git a/src/common/compress.h b/src/common/compress.h index 2d812e4..8be67df 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -26,11 +26,11 @@ typedef enum { /** * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most * memory. **/ typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION + BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION } compression_level_t; int tor_compress(char **out, size_t *out_len, diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index 9c61d58..59b7df6 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -32,6 +32,7 @@ memory_level(compression_level_t level) { switch (level) { default: + case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; case MEDIUM_COMPRESSION: return 6; case LOW_COMPRESSION: return 3; diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index 37768ee..d8200d5 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -56,6 +56,7 @@ memory_level(compression_level_t level) { switch (level) { default: + case BEST_COMPRESSION: return 9; case HIGH_COMPRESSION: return 8; case MEDIUM_COMPRESSION: return 7; case LOW_COMPRESSION: return 6; @@ -70,6 +71,7 @@ method_bits(compress_method_t method, compression_level_t level) const int flag = method == GZIP_METHOD ? 16 : 0; switch (level) { default: + case BEST_COMPRESSION: case HIGH_COMPRESSION: return flag + 15; case MEDIUM_COMPRESSION: return flag + 13; case LOW_COMPRESSION: return flag + 11; @@ -162,7 +164,7 @@ tor_zlib_compress_new(int compress_, if (! compress_) { /* use this setting for decompression, since we might have the * max number of window bits */ - compression_level = HIGH_COMPRESSION; + compression_level = BEST_COMPRESSION; } out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 76f2991..58fa327 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -33,6 +33,7 @@ memory_level(compression_level_t level) { switch (level) { default: + case BEST_COMPRESSION: case HIGH_COMPRESSION: return 9; case MEDIUM_COMPRESSION: return 8; case LOW_COMPRESSION: return 7;

1 0

[tor/master] Merge branch 'unified_compress_squashed'
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 49868340f7346a7c30603b21d580a3bb54701b0a Merge: 232c9e1 49a5b50 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 10:51:13 2017 -0400 Merge branch 'unified_compress_squashed' src/common/compress.c | 182 ++++++++++++++++++++++------ src/common/compress.h | 4 +- src/common/compress_lzma.c | 291 +-------------------------------------------- src/common/compress_lzma.h | 10 -- src/common/compress_zlib.c | 265 ++--------------------------------------- src/common/compress_zlib.h | 10 -- src/common/compress_zstd.c | 284 +------------------------------------------ src/common/compress_zstd.h | 10 -- src/test/test_util.c | 2 +- 9 files changed, 160 insertions(+), 898 deletions(-)

1 0

[translation/tor-browser-manual] Update translations for tor-browser-manual
by translation＠torproject.org 25 Apr '17

25 Apr '17

commit 4d2cd6d59523968ad50105358074adda0e7d5427 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Apr 25 14:48:19 2017 +0000 Update translations for tor-browser-manual --- de/de.po | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/de/de.po b/de/de.po index 27151f0..9599fdf 100644 --- a/de/de.po +++ b/de/de.po @@ -13,12 +13,13 @@ # kan torkel <kantorkel(a)hamburg.freifunk.net>, 2017 # max weber <hfr(a)posteo.de>, 2017 # nautilusx <mail.ka(a)mailbox.org>, 2017 +# Leptopoda <rimikis.nikolas(a)gmail.com>, 2017 msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "POT-Creation-Date: 2016-12-06 16:36-0600\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: nautilusx <mail.ka(a)mailbox.org>, 2017\n" +"Last-Translator: Leptopoda <rimikis.nikolas(a)gmail.com>, 2017\n" "Language-Team: German (https://www.transifex.com/otf/teams/1519/de/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -1455,6 +1456,10 @@ msgid "" "outdated version of the software, you may be vulnerable to serious security " "flaws that compromise your privacy and anonymity." msgstr "" +"Der Tor Browser muss immer auf dem neusten Stand gehalten werden. Wenn sie " +"weiterhin eine veraltete Version des Browsers verwenden, könnte ihre " +"Privatsphäre und Anonymität durch ernsthafte Sicherheitslücken gefährdet " +"werden." #: updating.page:18 msgid ""

1 0

[tor/master] Merge branch 'atomic_counters'
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 232c9e14a8319040c59fd9da1b5d242a57d10a04 Merge: ec7c512 4266ec7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 10:46:23 2017 -0400 Merge branch 'atomic_counters' src/common/compat_threads.c | 45 ++++++++++++++++++++++++++++++++++++++++++ src/common/compat_threads.h | 14 +++++++++++++ src/common/compress.c | 8 ++++++++ src/common/compress.h | 2 ++ src/common/compress_lzma.c | 13 +++++++++--- src/common/compress_lzma.h | 2 ++ src/common/compress_zlib.c | 14 +++++++++---- src/common/compress_zlib.h | 2 ++ src/common/compress_zstd.c | 13 +++++++++--- src/common/compress_zstd.h | 2 ++ src/or/main.c | 1 + src/test/bench.c | 1 + src/test/fuzz/fuzzing_common.c | 1 + src/test/testing_common.c | 1 + 14 files changed, 109 insertions(+), 10 deletions(-)

1 0

[tor/master] Use atomic counters for compressor allocation.
by nickm＠torproject.org 25 Apr '17

25 Apr '17

commit 4266ec766af8e210ac52f597699b44818960ee39 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 25 10:29:07 2017 -0400 Use atomic counters for compressor allocation. --- src/common/compat_threads.c | 45 ++++++++++++++++++++++++++++++++++++++++++ src/common/compat_threads.h | 14 +++++++++++++ src/common/compress.c | 8 ++++++++ src/common/compress.h | 2 ++ src/common/compress_lzma.c | 13 +++++++++--- src/common/compress_lzma.h | 2 ++ src/common/compress_zlib.c | 14 +++++++++---- src/common/compress_zlib.h | 2 ++ src/common/compress_zstd.c | 13 +++++++++--- src/common/compress_zstd.h | 2 ++ src/or/main.c | 1 + src/test/bench.c | 1 + src/test/fuzz/fuzzing_common.c | 1 + src/test/testing_common.c | 1 + 14 files changed, 109 insertions(+), 10 deletions(-) diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index 7b28a94..8d026dd 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -352,3 +352,48 @@ alert_sockets_close(alert_sockets_t *socks) socks->read_fd = socks->write_fd = -1; } +/* + * XXXX We might be smart to move to compiler intrinsics or real atomic + * XXXX operations at some point. But not yet. + * + */ + +/** Initialize a new atomic counter with the value 0 */ +void +atomic_counter_init(atomic_counter_t *counter) +{ + tor_mutex_init_nonrecursive(&counter->mutex); + counter->val = 0; +} +/** Clean up all resources held by an atomic counter. */ +void +atomic_counter_destroy(atomic_counter_t *counter) +{ + tor_mutex_uninit(&counter->mutex); + memset(counter, 0, sizeof(*counter)); +} +/** Add a value to an atomic counter. */ +void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + tor_mutex_acquire(&counter->mutex); + counter->val += add; + tor_mutex_release(&counter->mutex); +} +/** Subtract a value from an atomic counter. */ +void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + // this relies on unsigned overflow, but that's fine. + atomic_counter_add(counter, -sub); +} +/** Return the current value of an atomic counter */ +size_t +atomic_counter_get(atomic_counter_t *counter) +{ + size_t val; + tor_mutex_acquire(&counter->mutex); + val = counter->val; + tor_mutex_release(&counter->mutex); + return val; +} diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 2943573..9fa3d0d 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -147,5 +147,19 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal); */ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); +/** + * Atomic counter type; holds a size_t value. + */ +typedef struct atomic_counter_t { + tor_mutex_t mutex; + size_t val; +} atomic_counter_t; + +void atomic_counter_init(atomic_counter_t *counter); +void atomic_counter_destroy(atomic_counter_t *counter); +void atomic_counter_add(atomic_counter_t *counter, size_t add); +void atomic_counter_sub(atomic_counter_t *counter, size_t sub); +size_t atomic_counter_get(atomic_counter_t *counter); + #endif diff --git a/src/common/compress.c b/src/common/compress.c index 38b8184..9332c00 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -375,3 +375,11 @@ tor_compress_state_size(const tor_compress_state_t *state) return 0; } +/** Initialize all compression modules. */ +void +tor_compress_init(void) +{ + tor_zlib_init(); + tor_lzma_init(); + tor_zstd_init(); +} diff --git a/src/common/compress.h b/src/common/compress.h index 2d812e4..5dd916b 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -79,5 +79,7 @@ void tor_compress_free(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); +void tor_compress_init(void); + #endif // TOR_COMPRESS_H. diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index ae0327f..b0ba4b6 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -23,7 +23,7 @@ #endif /** Total number of bytes allocated for LZMA state. */ -static size_t total_lzma_allocation = 0; +static atomic_counter_t total_lzma_allocation; #ifdef HAVE_LZMA /** Given level return the memory level. */ @@ -465,6 +465,7 @@ tor_lzma_compress_new(int compress, } } + atomic_counter_add(&total_lzma_allocation, result->allocation); return result; err: @@ -579,7 +580,7 @@ tor_lzma_compress_free(tor_lzma_compress_state_t *state) if (state == NULL) return; - total_lzma_allocation -= state->allocation; + atomic_counter_sub(&total_lzma_allocation, state->allocation); #ifdef HAVE_LZMA lzma_end(&state->stream); @@ -600,6 +601,12 @@ tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) size_t tor_lzma_get_total_allocation(void) { - return total_lzma_allocation; + return atomic_counter_get(&total_lzma_allocation); } +/** Initialize the lzma module */ +void +tor_lzma_init(void) +{ + atomic_counter_init(&total_lzma_allocation); +} diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h index 71de56a..60a5f9c 100644 --- a/src/common/compress_lzma.h +++ b/src/common/compress_lzma.h @@ -47,5 +47,7 @@ size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); size_t tor_lzma_get_total_allocation(void); +void tor_lzma_init(void); + #endif // TOR_COMPRESS_LZMA_H. diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index faa179f..5425f20 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -48,7 +48,7 @@ static size_t tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel); /** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; +static atomic_counter_t total_zlib_allocation; /** Given level return the memory level. */ static int @@ -437,7 +437,7 @@ tor_zlib_compress_new(int compress_, } out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - total_zlib_allocation += out->allocation; + atomic_counter_add(&total_zlib_allocation, out->allocation); return out; @@ -519,7 +519,7 @@ tor_zlib_compress_free(tor_zlib_compress_state_t *state) if (state == NULL) return; - total_zlib_allocation -= state->allocation; + atomic_counter_sub(&total_zlib_allocation, state->allocation); if (state->compress) deflateEnd(&state->stream); @@ -541,6 +541,12 @@ tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) size_t tor_zlib_get_total_allocation(void) { - return total_zlib_allocation; + return atomic_counter_get(&total_zlib_allocation); } +/** Set up global state for the zlib module */ +void +tor_zlib_init(void) +{ + atomic_counter_init(&total_zlib_allocation); +} diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h index 6e8e5c5..817ca27 100644 --- a/src/common/compress_zlib.h +++ b/src/common/compress_zlib.h @@ -47,5 +47,7 @@ size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); size_t tor_zlib_get_total_allocation(void); +void tor_zlib_init(void); + #endif // TOR_COMPRESS_ZLIB_H. diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index 664cce1..8725c10 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -24,7 +24,7 @@ #endif /** Total number of bytes allocated for Zstandard state. */ -static size_t total_zstd_allocation = 0; +static atomic_counter_t total_zstd_allocation; #ifdef HAVE_ZSTD /** Given level return the memory level. */ @@ -446,6 +446,7 @@ tor_zstd_compress_new(int compress, } } + atomic_counter_add(&total_zstd_allocation, result->allocation); return result; err: @@ -578,7 +579,7 @@ tor_zstd_compress_free(tor_zstd_compress_state_t *state) if (state == NULL) return; - total_zstd_allocation -= state->allocation; + atomic_counter_sub(&total_zstd_allocation, state->allocation); #ifdef HAVE_ZSTD if (state->compress) { @@ -604,6 +605,12 @@ tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) size_t tor_zstd_get_total_allocation(void) { - return total_zstd_allocation; + return atomic_counter_get(&total_zstd_allocation); } +/** Initialize the zstd module */ +void +tor_zstd_init(void) +{ + atomic_counter_init(&total_zstd_allocation); +} diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index 663cbdd..186dc3b 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -47,5 +47,7 @@ size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); size_t tor_zstd_get_total_allocation(void); +void tor_zstd_init(void); + #endif // TOR_COMPRESS_ZSTD_H. diff --git a/src/or/main.c b/src/or/main.c index 1ba6554..5fec7e4 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -3616,6 +3616,7 @@ tor_main(int argc, char *argv[]) update_approx_time(time(NULL)); tor_threads_init(); + tor_compress_init(); init_logging(0); monotime_init(); #ifdef USE_DMALLOC diff --git a/src/test/bench.c b/src/test/bench.c index 5da9168..a44dc94 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -674,6 +674,7 @@ main(int argc, const char **argv) or_options_t *options; tor_threads_init(); + tor_compress_init(); if (argc == 4 && !strcmp(argv[1], "diff")) { init_logging(1); diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index f540176..7aee92d 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -96,6 +96,7 @@ static void global_init(void) { tor_threads_init(); + tor_compress_init(); { struct sipkey sipkey = { 1337, 7331 }; siphash_set_global_key(&sipkey); diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 0e37e0d..d3dc761 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -245,6 +245,7 @@ main(int c, const char **v) update_approx_time(time(NULL)); options = options_new(); tor_threads_init(); + tor_compress_init(); network_init();

1 0