June 2017 - tor-commits - lists.torproject.org

[metrics-lib/release] Add a test for the encoding of microdescriptor digests in votes
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit 47a69b342faae83fcd70909c3f03129f7b133364 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Jun 17 21:18:11 2017 +0200 Add a test for the encoding of microdescriptor digests in votes --- .../descriptor/impl/RelayNetworkStatusVoteImplTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index 1152626..8a85145 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -9,6 +9,7 @@ import static org.junit.Assert.assertTrue; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DirectorySignature; +import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusVote; import org.junit.Rule; @@ -717,9 +718,12 @@ public class RelayNetworkStatusVoteImplTest { assertTrue(vote.getKnownFlags().contains("Running")); assertEquals(30000, (int) vote.getConsensusParams().get( "CircuitPriorityHalflifeMsec")); + NetworkStatusEntry nse = vote.getStatusEntry( + "00343A8024F70E214728F0C5AF7ACE0C1508F073"); assertEquals("Tor 0.2.1.29 (r8e9b25e6c7a2e70c)", - vote.getStatusEntry("00343A8024F70E214728F0C5AF7ACE0C1508F073") - .getVersion()); + nse.getVersion()); + assertTrue(nse.getMicrodescriptorDigestsSha256Base64().contains( + "9ciEx9t0McXk9A06I7qwN7pxuNOdpCP64RV/6cx2Zkc")); assertEquals(3, vote.getDirKeyCertificateVersion()); assertEquals("80550987E1D626E3EBA5E5E75A458DE0626D088C", vote.getIdentity());

1 0

[metrics-lib/master] Prepare for 1.9.0 release.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit 3c98457777aa4d96b2e5da6cd640c2704dd42e9f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jun 20 22:36:58 2017 +0200 Prepare for 1.9.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95cd851..98728b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.9.0 - 2017-06-?? +# Changes in version 1.9.0 - 2017-06-21 * Major changes - Simplify DescriptorReader by returning Descriptor instances diff --git a/build.xml b/build.xml index d5aaa8f..18ba365 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="descriptor" basedir="."> - <property name="release.version" value="1.8.2-dev" /> + <property name="release.version" value="1.9.0" /> <property name="javadoc-title" value="DescripTor API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="DescripTor" />

1 0

[tor/master] Send the correct content-encoding when serving cached_dir_t objects
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit 782eb02b79363bf8ed609495d516bc59972f4352 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 11:26:51 2017 -0400 Send the correct content-encoding when serving cached_dir_t objects A cached_dir_t object (for now) is always compressed with DEFLATE_METHOD, but in handle_get_status_vote() to we were using the general compression-negotiation code decide what compression to claim we were using. This was one of the reasons behind 22502. Fixes bug 22669; bugfix on 0.3.1.1-alpha --- changes/bug22669 | 4 ++++ src/or/directory.c | 25 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/changes/bug22669 b/changes/bug22669 new file mode 100644 index 0000000..804a39e --- /dev/null +++ b/changes/bug22669 @@ -0,0 +1,4 @@ + o Minor bugfixes (compression): + - When serving directory votes compressed with zlib, + do not claim to have compressed them with zstd. Fixes bug 22669; + bugfix on 0.3.1.1-alpha. diff --git a/src/or/directory.c b/src/or/directory.c index ecd9811..b680b13 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -4190,13 +4190,14 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const compress_method_t compress_method = - find_best_compression_method(args->compression_supported, 1); { int current; ssize_t body_len = 0; ssize_t estimated_len = 0; + /* This smartlist holds strings that we can compress on the fly. */ smartlist_t *items = smartlist_new(); + /* This smartlist holds cached_dir_t objects that have a precompressed + * deflated version. */ smartlist_t *dir_items = smartlist_new(); int lifetime = 60; /* XXXX?? should actually use vote intervals. */ url += strlen("/tor/status-vote/"); @@ -4247,6 +4248,26 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) write_http_status_line(conn, 404, "Not found"); goto vote_done; } + + /* We're sending items from at most one kind of source */ + tor_assert_nonfatal(smartlist_len(items) == 0 || + smartlist_len(dir_items) == 0); + + int streaming; + unsigned mask; + if (smartlist_len(items)) { + /* We're taking strings and compressing them on the fly. */ + streaming = 1; + mask = ~0u; + } else { + /* We're taking cached_dir_t objects. We only have them uncompressed + * or deflated. */ + streaming = 0; + mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD); + } + const compress_method_t compress_method = find_best_compression_method( + args->compression_supported&mask, streaming); + SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, body_len += compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len);

1 0

[tor/maint-0.3.1] Send the correct content-encoding when serving cached_dir_t objects
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit 782eb02b79363bf8ed609495d516bc59972f4352 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 11:26:51 2017 -0400 Send the correct content-encoding when serving cached_dir_t objects A cached_dir_t object (for now) is always compressed with DEFLATE_METHOD, but in handle_get_status_vote() to we were using the general compression-negotiation code decide what compression to claim we were using. This was one of the reasons behind 22502. Fixes bug 22669; bugfix on 0.3.1.1-alpha --- changes/bug22669 | 4 ++++ src/or/directory.c | 25 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/changes/bug22669 b/changes/bug22669 new file mode 100644 index 0000000..804a39e --- /dev/null +++ b/changes/bug22669 @@ -0,0 +1,4 @@ + o Minor bugfixes (compression): + - When serving directory votes compressed with zlib, + do not claim to have compressed them with zstd. Fixes bug 22669; + bugfix on 0.3.1.1-alpha. diff --git a/src/or/directory.c b/src/or/directory.c index ecd9811..b680b13 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -4190,13 +4190,14 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const compress_method_t compress_method = - find_best_compression_method(args->compression_supported, 1); { int current; ssize_t body_len = 0; ssize_t estimated_len = 0; + /* This smartlist holds strings that we can compress on the fly. */ smartlist_t *items = smartlist_new(); + /* This smartlist holds cached_dir_t objects that have a precompressed + * deflated version. */ smartlist_t *dir_items = smartlist_new(); int lifetime = 60; /* XXXX?? should actually use vote intervals. */ url += strlen("/tor/status-vote/"); @@ -4247,6 +4248,26 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) write_http_status_line(conn, 404, "Not found"); goto vote_done; } + + /* We're sending items from at most one kind of source */ + tor_assert_nonfatal(smartlist_len(items) == 0 || + smartlist_len(dir_items) == 0); + + int streaming; + unsigned mask; + if (smartlist_len(items)) { + /* We're taking strings and compressing them on the fly. */ + streaming = 1; + mask = ~0u; + } else { + /* We're taking cached_dir_t objects. We only have them uncompressed + * or deflated. */ + streaming = 0; + mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD); + } + const compress_method_t compress_method = find_best_compression_method( + args->compression_supported&mask, streaming); + SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, body_len += compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len);

1 0

[tor/maint-0.3.1] Merge branch 'bug22672_031' into maint-0.3.1
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit c999e84436a5f7ba1964abd4ae502212f0995bcb Merge: c4152a2 9328bd5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 20:26:45 2017 -0400 Merge branch 'bug22672_031' into maint-0.3.1 changes/bug22672 | 5 +++++ src/common/compress.c | 36 +++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-)

1 0

[tor/maint-0.3.1] Enforce the rule that COMPRESS_OK means progress was made.
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit 9328bd524e38befe240fd729a9febb4dfe8b5be4 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 12:26:57 2017 -0400 Enforce the rule that COMPRESS_OK means progress was made. If COMPRESS_OK occurs but data is neither consumed nor generated, treat it as a BUG and a COMPRESS_ERROR. This change is meant to prevent infinite loops in the case where we've made a mistake in one of our compression backends. Closes ticket 22672. --- changes/bug22672 | 5 +++++ src/common/compress.c | 36 +++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/changes/bug22672 b/changes/bug22672 new file mode 100644 index 0000000..ec66811 --- /dev/null +++ b/changes/bug22672 @@ -0,0 +1,5 @@ + o Minor features (compression, defensive programming): + - Detect and break out of infinite loops in our compression code. + We don't think that any such loops exist now, but it's best to be + safe. Closes ticket 22672. + diff --git a/src/common/compress.c b/src/common/compress.c index c16d1b5..46aa1d5 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -548,28 +548,42 @@ tor_compress_process(tor_compress_state_t *state, int finish) { tor_assert(state != NULL); + const size_t in_len_orig = *in_len; + const size_t out_len_orig = *out_len; + tor_compress_output_t rv; switch (state->method) { case GZIP_METHOD: case ZLIB_METHOD: - return tor_zlib_compress_process(state->u.zlib_state, - out, out_len, in, in_len, - finish); + rv = tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + break; case LZMA_METHOD: - return tor_lzma_compress_process(state->u.lzma_state, - out, out_len, in, in_len, - finish); + rv =tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + break; case ZSTD_METHOD: - return tor_zstd_compress_process(state->u.zstd_state, - out, out_len, in, in_len, - finish); + rv = tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + break; case NO_METHOD: - return tor_cnone_compress_process(out, out_len, in, in_len, - finish); + rv = tor_cnone_compress_process(out, out_len, in, in_len, + finish); + break; + default: case UNKNOWN_METHOD: goto err; } + if (BUG((rv == TOR_COMPRESS_OK) && + *in_len == in_len_orig && + *out_len == out_len_orig)) { + return TOR_COMPRESS_ERROR; + } + return rv; err: return TOR_COMPRESS_ERROR; }

1 0

[tor/master] Enforce the rule that COMPRESS_OK means progress was made.
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit 9328bd524e38befe240fd729a9febb4dfe8b5be4 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 12:26:57 2017 -0400 Enforce the rule that COMPRESS_OK means progress was made. If COMPRESS_OK occurs but data is neither consumed nor generated, treat it as a BUG and a COMPRESS_ERROR. This change is meant to prevent infinite loops in the case where we've made a mistake in one of our compression backends. Closes ticket 22672. --- changes/bug22672 | 5 +++++ src/common/compress.c | 36 +++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/changes/bug22672 b/changes/bug22672 new file mode 100644 index 0000000..ec66811 --- /dev/null +++ b/changes/bug22672 @@ -0,0 +1,5 @@ + o Minor features (compression, defensive programming): + - Detect and break out of infinite loops in our compression code. + We don't think that any such loops exist now, but it's best to be + safe. Closes ticket 22672. + diff --git a/src/common/compress.c b/src/common/compress.c index c16d1b5..46aa1d5 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -548,28 +548,42 @@ tor_compress_process(tor_compress_state_t *state, int finish) { tor_assert(state != NULL); + const size_t in_len_orig = *in_len; + const size_t out_len_orig = *out_len; + tor_compress_output_t rv; switch (state->method) { case GZIP_METHOD: case ZLIB_METHOD: - return tor_zlib_compress_process(state->u.zlib_state, - out, out_len, in, in_len, - finish); + rv = tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + break; case LZMA_METHOD: - return tor_lzma_compress_process(state->u.lzma_state, - out, out_len, in, in_len, - finish); + rv =tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + break; case ZSTD_METHOD: - return tor_zstd_compress_process(state->u.zstd_state, - out, out_len, in, in_len, - finish); + rv = tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + break; case NO_METHOD: - return tor_cnone_compress_process(out, out_len, in, in_len, - finish); + rv = tor_cnone_compress_process(out, out_len, in, in_len, + finish); + break; + default: case UNKNOWN_METHOD: goto err; } + if (BUG((rv == TOR_COMPRESS_OK) && + *in_len == in_len_orig && + *out_len == out_len_orig)) { + return TOR_COMPRESS_ERROR; + } + return rv; err: return TOR_COMPRESS_ERROR; }

1 0

[tor/maint-0.3.1] Correct the fix to bug 22629 to permit trailing non-garbage
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit eb632afb17aad58a23f3c8edce865ddf9a338f20 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 10:24:22 2017 -0400 Correct the fix to bug 22629 to permit trailing non-garbage This change makes it so that we can decompress concatenated zstd outputs. --- src/common/compress.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index e65894d..c6c3897 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -150,13 +150,7 @@ tor_compress_impl(int compress, method, compression_level, in_len); goto err; } else { - if (in_len != 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "Unexpected extra input while decompressing"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %zd", - method, compression_level, in_len); - goto err; - } else { + if (in_len == 0) { goto done; } }

1 0

[tor/maint-0.3.1] Merge branch 'bug22502_redux_031' into maint-0.3.1
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit e51e7bd38b758d3f71a2e31184dde3771f897af8 Merge: c999e84 782eb02 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 20:27:48 2017 -0400 Merge branch 'bug22502_redux_031' into maint-0.3.1 changes/bug22669 | 4 ++++ src/common/compress.c | 8 +------ src/or/directory.c | 25 ++++++++++++++++++-- src/test/test_util.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 9 deletions(-) diff --cc src/common/compress.c index 46aa1d5,c6c3897..7926faa --- a/src/common/compress.c +++ b/src/common/compress.c @@@ -146,17 -146,11 +146,11 @@@ tor_compress_impl(int compress "Unexpected %s while %scompressing", complete_only?"end of input":"result", compress?"":"de"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %zd", - method, compression_level, in_len); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); goto err; } else { - if (in_len != 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "Unexpected extra input while decompressing"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", - method, compression_level, (unsigned long)in_len); - goto err; - } else { + if (in_len == 0) { goto done; } }

1 0

[tor/master] Correct the fix to bug 22629 to permit trailing non-garbage
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit eb632afb17aad58a23f3c8edce865ddf9a338f20 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 20 10:24:22 2017 -0400 Correct the fix to bug 22629 to permit trailing non-garbage This change makes it so that we can decompress concatenated zstd outputs. --- src/common/compress.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index e65894d..c6c3897 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -150,13 +150,7 @@ tor_compress_impl(int compress, method, compression_level, in_len); goto err; } else { - if (in_len != 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "Unexpected extra input while decompressing"); - log_debug(LD_GENERAL, "method: %d level: %d at len: %zd", - method, compression_level, in_len); - goto err; - } else { + if (in_len == 0) { goto done; } }

1 0