April 2019 - tor-commits - lists.torproject.org

[tor/master] Merge branch 'tor-github/pr/938'
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 78223ab0fc279abe0c575f191f5d0f42ef1bda59 Merge: 2db9bb02c 733234600 Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Apr 18 13:23:32 2019 +0300 Merge branch 'tor-github/pr/938' changes/ticket30149 | 3 +++ scripts/maint/practracker/exceptions.txt | 2 +- src/feature/client/circpathbias.c | 4 +++- src/feature/nodelist/node_select.c | 1 + src/feature/relay/onion_queue.c | 10 ++++++---- src/lib/math/prob_distr.h | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-)

1 0

[tor/master] Merge branch 'maint-0.4.0'
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit d867b7ae1dcd77ca7af64c3e5489644670b136f5 Merge: ef97e6c55 6a179b107 Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Apr 18 13:22:23 2019 +0300 Merge branch 'maint-0.4.0' changes/diagnostic_28223_redux | 4 ++++ src/feature/dirparse/microdesc_parse.c | 17 ++++++++++++++++- src/feature/nodelist/microdesc.c | 28 ++++++++++++++++++++++++++++ src/lib/fdio/fdio.c | 4 ++++ 4 files changed, 52 insertions(+), 1 deletion(-)

1 0

[tor/release-0.4.0] Even more diagnostic messages for bug 28223.
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 99b87d7ca48c5ed80aec5f24902843f72975bfea Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 3 13:53:06 2019 -0400 Even more diagnostic messages for bug 28223. Try to figure out _where exactly_ we are first encountering NULs in microdescriptors, and what we are doing when that happens. --- changes/diagnostic_28223_redux | 4 ++++ src/feature/dirparse/microdesc_parse.c | 17 ++++++++++++++++- src/feature/nodelist/microdesc.c | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/changes/diagnostic_28223_redux b/changes/diagnostic_28223_redux new file mode 100644 index 000000000..0d7499832 --- /dev/null +++ b/changes/diagnostic_28223_redux @@ -0,0 +1,4 @@ + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve + the issue of NUL bytes appearing in a microdescriptor cache. + Related to ticket 28223. diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 5a75af399..1e26901ce 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -159,7 +159,22 @@ microdescs_parse_from_string(const char *s, const char *eos, if (tokenize_string(area, s, start_of_next_microdesc, tokens, microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); goto next; } diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index dafaabb5e..af4f608d2 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -221,6 +221,13 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); + const char *nulpos = memchr(md->body, 0, md->bodylen); + if (BUG(nulpos)) { + log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " + "offset %"PRId64", bodylen %zu, nul position %zu", + (int64_t)md->off, md->bodylen, + (size_t)(nulpos - md->body)); + } written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -480,6 +487,17 @@ microdesc_cache_clear(microdesc_cache_t *cache) cache->bytes_dropped = 0; } +static void +warn_if_nul_found(const char *inp, size_t len, const char *description) +{ + const char *nul_found = memchr(inp, 0, len); + if (BUG(nul_found)) { + log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " + "position %zu/%zu.", + description, (nul_found - inp), len); + } +} + /** Reload the contents of cache from disk. If it is empty, load it * for the first time. Return 0 on success, -1 on failure. */ int @@ -497,6 +515,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { + warn_if_nul_found(mm->data, mm->size, "microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -509,6 +528,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; + warn_if_nul_found(journal_content, cache->journal_len, + "microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL);

1 0

[tor/master] Merge branch 'tor-github/pr/925'
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 2db9bb02c79e023128e7b6fc89f4601b88e8ef4b Merge: d867b7ae1 5722c6d12 Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Apr 18 13:22:54 2019 +0300 Merge branch 'tor-github/pr/925' changes/ticket30114 | 3 +++ scripts/git/git-pull-all.sh | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-)

1 0

[tor/release-0.4.0] fixup! Even more diagnostic messages for bug 28223.
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 0c42ddf28c330280f66feb7cc92b63f3998468e1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Apr 15 15:21:18 2019 -0400 fixup! Even more diagnostic messages for bug 28223. Use TOR_PRIuSZ in place of %zu. --- src/feature/nodelist/microdesc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index af4f608d2..eaa7f0caf 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -224,7 +224,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) const char *nulpos = memchr(md->body, 0, md->bodylen); if (BUG(nulpos)) { log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " - "offset %"PRId64", bodylen %zu, nul position %zu", + "offset %"PRId64", bodylen %"TOR_PRIuSZ", " + "nul position %"TOR_PRIuSZ".", (int64_t)md->off, md->bodylen, (size_t)(nulpos - md->body)); } @@ -493,7 +494,7 @@ warn_if_nul_found(const char *inp, size_t len, const char *description) const char *nul_found = memchr(inp, 0, len); if (BUG(nul_found)) { log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " - "position %zu/%zu.", + "position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", description, (nul_found - inp), len); } }

1 0

[tor/release-0.4.0] fdio.c: add more includes.
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 821d29e42040789c59d271ba2f1d87a54d6d1435 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 3 13:53:36 2019 -0400 fdio.c: add more includes. This is just in case there is some rogue platform that uses a nonstandard value for SEEK_*, and does not define that macro in unistd.h. I think that's unlikely, but it's conceivable. --- src/lib/fdio/fdio.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index 6c87af791..078af6a9b 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -17,12 +17,16 @@ #ifdef _WIN32 #include <windows.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include <stdlib.h> +#include <stdio.h> /** @{ */ /** Some old versions of Unix didn't define constants for these values,

1 0

[tor/release-0.4.0] Merge branch 'tor-github/pr/891' into maint-0.4.0
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 6a179b1072887c5b8e9a2f679a6faf5504bbf3e3 Merge: c72f0cd5c 950d890f7 Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Apr 18 13:21:59 2019 +0300 Merge branch 'tor-github/pr/891' into maint-0.4.0 changes/diagnostic_28223_redux | 4 ++++ src/feature/dirparse/microdesc_parse.c | 17 ++++++++++++++++- src/feature/nodelist/microdesc.c | 28 ++++++++++++++++++++++++++++ src/lib/fdio/fdio.c | 4 ++++ 4 files changed, 52 insertions(+), 1 deletion(-)

1 0

[tor/release-0.4.0] In warn_if_nul_found, log surrounding context.
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 950d890f77d8f060702b100e6075d5083d536577 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Apr 15 15:33:09 2019 -0400 In warn_if_nul_found, log surrounding context. We need to encode here instead of doing escaped(), since fwict escaped() does not currently handle NUL bytes. Also, use warn_if_nul_found in more cases to avoid duplication. --- src/feature/nodelist/microdesc.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index eaa7f0caf..f4fa8eeae 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -69,6 +69,8 @@ struct microdesc_cache_t { }; static microdesc_cache_t *get_microdesc_cache_noload(void); +static void warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity); /** Helper: computes a hash of md to place it in a hash table. */ static inline unsigned int @@ -221,14 +223,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); - const char *nulpos = memchr(md->body, 0, md->bodylen); - if (BUG(nulpos)) { - log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " - "offset %"PRId64", bodylen %"TOR_PRIuSZ", " - "nul position %"TOR_PRIuSZ".", - (int64_t)md->off, md->bodylen, - (size_t)(nulpos - md->body)); - } + warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off, + "dumping a microdescriptor"); written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -489,13 +485,23 @@ microdesc_cache_clear(microdesc_cache_t *cache) } static void -warn_if_nul_found(const char *inp, size_t len, const char *description) +warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity) { const char *nul_found = memchr(inp, 0, len); if (BUG(nul_found)) { - log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " - "position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", - description, (nul_found - inp), len); + log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64 + "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", + activity, offset, (nul_found - inp), len); + const char *start_excerpt_at, *eos = inp + len; + if ((nul_found - inp) >= 16) + start_excerpt_at = nul_found - 16; + else + start_excerpt_at = inp; + size_t excerpt_len = MIN(32, eos - start_excerpt_at); + char tmp[65]; + base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len); + log_warn(LD_BUG, " surrounding string: %s", tmp); } } @@ -516,7 +522,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { - warn_if_nul_found(mm->data, mm->size, "microdesc cache"); + warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -529,8 +535,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; - warn_if_nul_found(journal_content, cache->journal_len, - "microdesc journal"); + warn_if_nul_found(journal_content, cache->journal_len, 0, + "reading microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL);

1 0

[tor/release-0.4.0] Merge branch 'maint-0.4.0' into release-0.4.0
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 3444371dd9386dc37e6dc7df4d7d151429626c91 Merge: ba15b2f23 6a179b107 Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Apr 18 13:22:23 2019 +0300 Merge branch 'maint-0.4.0' into release-0.4.0 changes/diagnostic_28223_redux | 4 ++++ src/feature/dirparse/microdesc_parse.c | 17 ++++++++++++++++- src/feature/nodelist/microdesc.c | 28 ++++++++++++++++++++++++++++ src/lib/fdio/fdio.c | 4 ++++ 4 files changed, 52 insertions(+), 1 deletion(-)

1 0

[tor/maint-0.4.0] Even more diagnostic messages for bug 28223.
by asn＠torproject.org 18 Apr '19

18 Apr '19

commit 99b87d7ca48c5ed80aec5f24902843f72975bfea Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 3 13:53:06 2019 -0400 Even more diagnostic messages for bug 28223. Try to figure out _where exactly_ we are first encountering NULs in microdescriptors, and what we are doing when that happens. --- changes/diagnostic_28223_redux | 4 ++++ src/feature/dirparse/microdesc_parse.c | 17 ++++++++++++++++- src/feature/nodelist/microdesc.c | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/changes/diagnostic_28223_redux b/changes/diagnostic_28223_redux new file mode 100644 index 000000000..0d7499832 --- /dev/null +++ b/changes/diagnostic_28223_redux @@ -0,0 +1,4 @@ + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve + the issue of NUL bytes appearing in a microdescriptor cache. + Related to ticket 28223. diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 5a75af399..1e26901ce 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -159,7 +159,22 @@ microdescs_parse_from_string(const char *s, const char *eos, if (tokenize_string(area, s, start_of_next_microdesc, tokens, microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); goto next; } diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index dafaabb5e..af4f608d2 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -221,6 +221,13 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); + const char *nulpos = memchr(md->body, 0, md->bodylen); + if (BUG(nulpos)) { + log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " + "offset %"PRId64", bodylen %zu, nul position %zu", + (int64_t)md->off, md->bodylen, + (size_t)(nulpos - md->body)); + } written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -480,6 +487,17 @@ microdesc_cache_clear(microdesc_cache_t *cache) cache->bytes_dropped = 0; } +static void +warn_if_nul_found(const char *inp, size_t len, const char *description) +{ + const char *nul_found = memchr(inp, 0, len); + if (BUG(nul_found)) { + log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " + "position %zu/%zu.", + description, (nul_found - inp), len); + } +} + /** Reload the contents of cache from disk. If it is empty, load it * for the first time. Return 0 on success, -1 on failure. */ int @@ -497,6 +515,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { + warn_if_nul_found(mm->data, mm->size, "microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -509,6 +528,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; + warn_if_nul_found(journal_content, cache->journal_len, + "microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL);

1 0