- tor-commits - lists.torproject.org

[tor/master] Change who calls microdesc_cache_rebuild().
by nickm＠torproject.org 03 May '11

03 May '11

commit 68ae5afa5a172fbc485eda7e6bfd4750fa6a69f5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 17:28:28 2011 -0400 Change who calls microdesc_cache_rebuild(). Previously we ensured that it would get called periodically by doing it from inside the code that added microdescriptors. That won't work though: it would interfere with our code that tried to read microdescs from disk initially. Instead, we should consider rebuilding the cache periodically, and on startup. --- changes/bug2230_part4 | 6 ++++++ src/or/main.c | 2 ++ src/or/microdesc.c | 11 ++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/changes/bug2230_part4 b/changes/bug2230_part4 new file mode 100644 index 0000000..f7721fa --- /dev/null +++ b/changes/bug2230_part4 @@ -0,0 +1,6 @@ + o Minor bugfixes: + - Authorities now clean their microdesc cache periodically and when + reading from disk initially, not only when adding new descriptors. + This prevents a bug where we could lose microdescriptors. Bugfix + on 0.2.2.6-alpha. + diff --git a/src/or/main.c b/src/or/main.c index a26be39..462b51e 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1075,6 +1075,8 @@ run_scheduled_events(time_t now) rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(); rend_cache_clean_v2_descs_as_dir(); + if (authdir_mode_v3(options)) + microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; } diff --git a/src/or/microdesc.c b/src/or/microdesc.c index c6bb3c6..73d2285 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -208,8 +208,6 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - microdesc_cache_rebuild(cache, 0/* only as needed */); - return added; } @@ -230,6 +228,7 @@ microdesc_cache_clear(microdesc_cache_t *cache) } cache->total_len_seen = 0; cache->n_seen = 0; + cache->bytes_dropped = 0; } /** Reload the contents of cache from disk. If it is empty, load it @@ -271,7 +270,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", total); - microdesc_cache_clean(cache, 0, 0); + microdesc_cache_rebuild(cache, 0 /* don't force */); return 0; } @@ -354,6 +353,12 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) off_t off = 0; int orig_size, new_size; + if (cache == NULL) { + cache = the_microdesc_cache; + if (cache == NULL) + return 0; + } + /* Remove dead descriptors */ microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);

1 0

[tor/maint-0.2.2] Change who calls microdesc_cache_rebuild().
by nickm＠torproject.org 03 May '11

03 May '11

commit 68ae5afa5a172fbc485eda7e6bfd4750fa6a69f5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 17:28:28 2011 -0400 Change who calls microdesc_cache_rebuild(). Previously we ensured that it would get called periodically by doing it from inside the code that added microdescriptors. That won't work though: it would interfere with our code that tried to read microdescs from disk initially. Instead, we should consider rebuilding the cache periodically, and on startup. --- changes/bug2230_part4 | 6 ++++++ src/or/main.c | 2 ++ src/or/microdesc.c | 11 ++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/changes/bug2230_part4 b/changes/bug2230_part4 new file mode 100644 index 0000000..f7721fa --- /dev/null +++ b/changes/bug2230_part4 @@ -0,0 +1,6 @@ + o Minor bugfixes: + - Authorities now clean their microdesc cache periodically and when + reading from disk initially, not only when adding new descriptors. + This prevents a bug where we could lose microdescriptors. Bugfix + on 0.2.2.6-alpha. + diff --git a/src/or/main.c b/src/or/main.c index a26be39..462b51e 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1075,6 +1075,8 @@ run_scheduled_events(time_t now) rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(); rend_cache_clean_v2_descs_as_dir(); + if (authdir_mode_v3(options)) + microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; } diff --git a/src/or/microdesc.c b/src/or/microdesc.c index c6bb3c6..73d2285 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -208,8 +208,6 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - microdesc_cache_rebuild(cache, 0/* only as needed */); - return added; } @@ -230,6 +228,7 @@ microdesc_cache_clear(microdesc_cache_t *cache) } cache->total_len_seen = 0; cache->n_seen = 0; + cache->bytes_dropped = 0; } /** Reload the contents of cache from disk. If it is empty, load it @@ -271,7 +270,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", total); - microdesc_cache_clean(cache, 0, 0); + microdesc_cache_rebuild(cache, 0 /* don't force */); return 0; } @@ -354,6 +353,12 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) off_t off = 0; int orig_size, new_size; + if (cache == NULL) { + cache = the_microdesc_cache; + if (cache == NULL) + return 0; + } + /* Remove dead descriptors */ microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 03 May '11

03 May '11

commit 865d53be5484ef06ca4a5103d9ebca343e19f7d4 Merge: 97a55b8 cb6c909 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 17:19:35 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' Conflicts: src/or/microdesc.c src/or/microdesc.c | 45 ++++++++++++++++++++++++++++++++------------- src/or/microdesc.h | 2 +- src/test/test_microdesc.c | 4 ++-- 3 files changed, 35 insertions(+), 16 deletions(-) diff --cc src/or/microdesc.c index 521ee07,c6bb3c6..5d3f1c3 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@@ -243,21 -208,8 +245,14 @@@ microdescs_add_list_to_cache(microdesc_ if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - { - size_t old_content_len = - cache->cache_content ? cache->cache_content->size : 0; - if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len / 2) { - microdesc_cache_rebuild(cache); - } - } + microdesc_cache_rebuild(cache, 0/* only as needed */); + { + networkstatus_t *ns = networkstatus_get_latest_consensus(); + if (ns && ns->flavor == FLAV_MICRODESC) + SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md)); + } + return added; } diff --cc src/test/test_microdesc.c index 5b14cdb,0000000..5a61c5d mode 100644,000000..100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@@ -1,233 -1,0 +1,233 @@@ +/* Copyright (c) 2010-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" + +#include "config.h" +#include "microdesc.h" + +#include "test.h" + +#ifdef MS_WINDOWS +/* For mkdir() */ +#include <direct.h> +#else +#include <dirent.h> +#endif + +static const char test_md1[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMjlHH/daN43cSVRaHBwgUfnszzAhg98EvivJ9Qxfv51mvQUxPjQ07es\n" + "gV/3n8fyh3Kqr/ehi9jxkdgSRfSnmF7giaHL1SLZ29kA7KtST+pBvmTpDtHa3ykX\n" + "Xorc7hJvIyTZoc1HU+5XSynj3gsBE5IGK1ZRzrNS688LnuZMVp1tAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + +static const char test_md2[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMIixIowh2DyPmDNMDwBX2DHcYcqdcH1zdIQJZkyV6c6rQHnvbcaDoSg\n" + "jgFSLJKpnGmh71FVRqep+yVB0zI1JY43kuEnXry2HbZCD9UDo3d3n7t015X5S7ON\n" + "bSSYtQGPwOr6Epf96IF6DoQxy4iDnPUAlejuhAG51s1y6/rZQ3zxAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + +static const char test_md3[] = + "@last-listed 2009-06-22\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMH3340d4ENNGrqx7UxT+lB7x6DNUKOdPEOn4teceE11xlMyZ9TPv41c\n" + "qj2fRZzfxlc88G/tmiaHshmdtEpklZ740OFqaaJVj4LjPMKFNE+J7Xc1142BE9Ci\n" + "KgsbjGYe2RY261aADRWLetJ8T9QDMm+JngL4288hc8pq1uB/3TAbAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "p accept 1-700,800-1000\n" + "family nodeX nodeY nodeZ\n"; + +static void +test_md_cache(void *data) +{ + or_options_t *options = NULL; + microdesc_cache_t *mc = NULL ; + smartlist_t *added = NULL, *wanted = NULL; + microdesc_t *md1, *md2, *md3; + char d1[DIGEST256_LEN], d2[DIGEST256_LEN], d3[DIGEST256_LEN]; + const char *test_md3_noannotation = strchr(test_md3, '\n')+1; + time_t time1, time2, time3; + char *fn = NULL, *s = NULL; + (void)data; + + options = get_options(); + tt_assert(options); + + time1 = time(NULL); + time2 = time(NULL) - 2*24*60*60; + time3 = time(NULL) - 15*24*60*60; + + /* Possibly, turn this into a test setup/cleanup pair */ + tor_free(options->DataDirectory); + options->DataDirectory = tor_strdup(get_fname("md_datadir_test")); +#ifdef MS_WINDOWS + tt_int_op(0, ==, mkdir(options->DataDirectory)); +#else + tt_int_op(0, ==, mkdir(options->DataDirectory, 0700)); +#endif + + tt_assert(!strcmpstart(test_md3_noannotation, "onion-key")); + + crypto_digest256(d1, test_md1, strlen(test_md1), DIGEST_SHA256); + crypto_digest256(d2, test_md2, strlen(test_md1), DIGEST_SHA256); + crypto_digest256(d3, test_md3_noannotation, strlen(test_md3_noannotation), + DIGEST_SHA256); + + mc = get_microdesc_cache(); + + added = microdescs_add_to_cache(mc, test_md1, NULL, SAVED_NOWHERE, 0, + time1, NULL); + tt_int_op(1, ==, smartlist_len(added)); + md1 = smartlist_get(added, 0); + smartlist_free(added); + added = NULL; + + wanted = smartlist_create(); + added = microdescs_add_to_cache(mc, test_md2, NULL, SAVED_NOWHERE, 0, + time2, wanted); + /* Should fail, since we didn't list test_md2's digest in wanted */ + tt_int_op(0, ==, smartlist_len(added)); + smartlist_free(added); + added = NULL; + + smartlist_add(wanted, tor_memdup(d2, DIGEST256_LEN)); + smartlist_add(wanted, tor_memdup(d3, DIGEST256_LEN)); + added = microdescs_add_to_cache(mc, test_md2, NULL, SAVED_NOWHERE, 0, + time2, wanted); + /* Now it can work. md2 should have been added */ + tt_int_op(1, ==, smartlist_len(added)); + md2 = smartlist_get(added, 0); + /* And it should have gotten removed from 'wanted' */ + tt_int_op(smartlist_len(wanted), ==, 1); + test_mem_op(smartlist_get(wanted, 0), ==, d3, DIGEST256_LEN); + smartlist_free(added); + added = NULL; + + added = microdescs_add_to_cache(mc, test_md3, NULL, + SAVED_NOWHERE, 0, -1, NULL); + /* Must fail, since SAVED_NOWHERE precludes annotations */ + tt_int_op(0, ==, smartlist_len(added)); + smartlist_free(added); + added = NULL; + + added = microdescs_add_to_cache(mc, test_md3_noannotation, NULL, + SAVED_NOWHERE, 0, time3, NULL); + /* Now it can work */ + tt_int_op(1, ==, smartlist_len(added)); + md3 = smartlist_get(added, 0); + smartlist_free(added); + added = NULL; + + /* Okay. We added 1...3. Let's poke them to see how they look, and make + * sure they're really in the journal. */ + tt_ptr_op(md1, ==, microdesc_cache_lookup_by_digest256(mc, d1)); + tt_ptr_op(md2, ==, microdesc_cache_lookup_by_digest256(mc, d2)); + tt_ptr_op(md3, ==, microdesc_cache_lookup_by_digest256(mc, d3)); + + tt_int_op(md1->last_listed, ==, time1); + tt_int_op(md2->last_listed, ==, time2); + tt_int_op(md3->last_listed, ==, time3); + + tt_int_op(md1->saved_location, ==, SAVED_IN_JOURNAL); + tt_int_op(md2->saved_location, ==, SAVED_IN_JOURNAL); + tt_int_op(md3->saved_location, ==, SAVED_IN_JOURNAL); + + tt_int_op(md1->bodylen, ==, strlen(test_md1)); + tt_int_op(md2->bodylen, ==, strlen(test_md2)); + tt_int_op(md3->bodylen, ==, strlen(test_md3_noannotation)); + test_mem_op(md1->body, ==, test_md1, strlen(test_md1)); + test_mem_op(md2->body, ==, test_md2, strlen(test_md2)); + test_mem_op(md3->body, ==, test_md3_noannotation, + strlen(test_md3_noannotation)); + + tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs.new", + options->DataDirectory); + s = read_file_to_str(fn, RFTS_BIN, NULL); + tt_assert(s); + test_mem_op(md1->body, ==, s + md1->off, md1->bodylen); + test_mem_op(md2->body, ==, s + md2->off, md2->bodylen); + test_mem_op(md3->body, ==, s + md3->off, md3->bodylen); + + tt_ptr_op(md1->family, ==, NULL); + tt_ptr_op(md3->family, !=, NULL); + tt_int_op(smartlist_len(md3->family), ==, 3); + tt_str_op(smartlist_get(md3->family, 0), ==, "nodeX"); + + /* Now rebuild the cache! */ - tt_int_op(microdesc_cache_rebuild(mc), ==, 0); ++ tt_int_op(microdesc_cache_rebuild(mc, 1), ==, 0); + + tt_int_op(md1->saved_location, ==, SAVED_IN_CACHE); + tt_int_op(md2->saved_location, ==, SAVED_IN_CACHE); + tt_int_op(md3->saved_location, ==, SAVED_IN_CACHE); + + /* The journal should be empty now */ + tor_free(s); + s = read_file_to_str(fn, RFTS_BIN, NULL); + tt_str_op(s, ==, ""); + tor_free(s); + tor_free(fn); + + /* read the cache. */ + tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs", + options->DataDirectory); + s = read_file_to_str(fn, RFTS_BIN, NULL); + test_mem_op(md1->body, ==, s + md1->off, strlen(test_md1)); + test_mem_op(md2->body, ==, s + md2->off, strlen(test_md2)); + test_mem_op(md3->body, ==, s + md3->off, strlen(test_md3_noannotation)); + + /* Okay, now we are going to forget about the cache entirely, and reload it + * from the disk. */ + microdesc_free_all(); + mc = get_microdesc_cache(); + md1 = microdesc_cache_lookup_by_digest256(mc, d1); + md2 = microdesc_cache_lookup_by_digest256(mc, d2); + md3 = microdesc_cache_lookup_by_digest256(mc, d3); + test_assert(md1); + test_assert(md2); + test_assert(md3); + test_mem_op(md1->body, ==, s + md1->off, strlen(test_md1)); + test_mem_op(md2->body, ==, s + md2->off, strlen(test_md2)); + test_mem_op(md3->body, ==, s + md3->off, strlen(test_md3_noannotation)); + + tt_int_op(md1->last_listed, ==, time1); + tt_int_op(md2->last_listed, ==, time2); + tt_int_op(md3->last_listed, ==, time3); + + /* Okay, now we are going to clear out everything older than a week old. + * In practice, that means md3 */ + microdesc_cache_clean(mc, time(NULL)-7*24*60*60, 1/*force*/); + tt_ptr_op(md1, ==, microdesc_cache_lookup_by_digest256(mc, d1)); + tt_ptr_op(md2, ==, microdesc_cache_lookup_by_digest256(mc, d2)); + tt_ptr_op(NULL, ==, microdesc_cache_lookup_by_digest256(mc, d3)); + md3 = NULL; /* it's history now! */ + + /* rebuild again, make sure it stays gone. */ - microdesc_cache_rebuild(mc); ++ microdesc_cache_rebuild(mc, 1); + tt_ptr_op(md1, ==, microdesc_cache_lookup_by_digest256(mc, d1)); + tt_ptr_op(md2, ==, microdesc_cache_lookup_by_digest256(mc, d2)); + tt_ptr_op(NULL, ==, microdesc_cache_lookup_by_digest256(mc, d3)); + + done: + if (options) + tor_free(options->DataDirectory); + microdesc_free_all(); + + smartlist_free(added); + if (wanted) + SMARTLIST_FOREACH(wanted, char *, cp, tor_free(cp)); + smartlist_free(wanted); + tor_free(s); + tor_free(fn); +} + +struct testcase_t microdesc_tests[] = { + { "cache", test_md_cache, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; +

1 0

[tor/master] Rebuild the microdesc cache when a sufficient number of bytes are dropped
by nickm＠torproject.org 03 May '11

03 May '11

commit cb6c909664f97f751fe78c0aa3205a9042760c53 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 17:03:49 2011 -0400 Rebuild the microdesc cache when a sufficient number of bytes are dropped Previously on 0.2.2, we'd never clean the cache. Now that we can clean it, we want to add a condition to rebuild it: that should happen whenever we have dropped enough microdescriptors that we could save a lot of space. No changes file, since 0.2.3 doesn't need one and 0.2.2 already has some changes files for the backport of the microdesc_clean_cahce() function. --- src/or/microdesc.c | 45 +++++++++++++++++++++++++++++++++------------ src/or/microdesc.h | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 58fbe37..c6bb3c6 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -177,6 +177,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (md2->last_listed < md->last_listed) md2->last_listed = md->last_listed; microdesc_free(md); + if (where != SAVED_NOWHERE) + cache->bytes_dropped += size; continue; } @@ -206,14 +208,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - { - size_t old_content_len = - cache->cache_content ? cache->cache_content->size : 0; - if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len / 2) { - microdesc_cache_rebuild(cache); - } - } + microdesc_cache_rebuild(cache, 0/* only as needed */); return added; } @@ -275,6 +270,9 @@ microdesc_cache_reload(microdesc_cache_t *cache) } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", total); + + microdesc_cache_clean(cache, 0, 0); + return 0; } @@ -323,11 +321,30 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) } } +static int +should_rebuild_md_cache(microdesc_cache_t *cache) +{ + const size_t old_len = + cache->cache_content ? cache->cache_content->size : 0; + const size_t journal_len = cache->journal_len; + const size_t dropped = cache->bytes_dropped; + + if (journal_len < 16384) + return 0; /* Don't bother, not enough has happened yet. */ + if (dropped > (journal_len + old_len) / 3) + return 1; /* We could save 1/3 or more of the currently used space. */ + if (journal_len > old_len / 2) + return 1; /* We should append to the regular file */ + + return 0; +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new - * location. */ + * location. If force is true, do this unconditionally. If + * force is false, do it only if we expect to save space on disk. */ int -microdesc_cache_rebuild(microdesc_cache_t *cache) +microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; FILE *f; @@ -337,11 +354,14 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) off_t off = 0; int orig_size, new_size; - log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); - /* Remove dead descriptors */ microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + if (!force && !should_rebuild_md_cache(cache)) + return 0; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; @@ -396,6 +416,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) write_str_to_file(cache->journal_fname, "", 1); cache->journal_len = 0; + cache->bytes_dropped = 0; new_size = (int)cache->cache_content->size; log_info(LD_DIR, "Done rebuilding microdesc cache. " diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 30cb25d..77ce853 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -22,7 +22,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, int no_save); void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); -int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_rebuild(microdesc_cache_t *cache, int force); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/maint-0.2.2] Rebuild the microdesc cache when a sufficient number of bytes are dropped
by nickm＠torproject.org 03 May '11

03 May '11

commit cb6c909664f97f751fe78c0aa3205a9042760c53 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 17:03:49 2011 -0400 Rebuild the microdesc cache when a sufficient number of bytes are dropped Previously on 0.2.2, we'd never clean the cache. Now that we can clean it, we want to add a condition to rebuild it: that should happen whenever we have dropped enough microdescriptors that we could save a lot of space. No changes file, since 0.2.3 doesn't need one and 0.2.2 already has some changes files for the backport of the microdesc_clean_cahce() function. --- src/or/microdesc.c | 45 +++++++++++++++++++++++++++++++++------------ src/or/microdesc.h | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 58fbe37..c6bb3c6 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -177,6 +177,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (md2->last_listed < md->last_listed) md2->last_listed = md->last_listed; microdesc_free(md); + if (where != SAVED_NOWHERE) + cache->bytes_dropped += size; continue; } @@ -206,14 +208,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, if (f) finish_writing_to_file(open_file); /*XXX Check me.*/ - { - size_t old_content_len = - cache->cache_content ? cache->cache_content->size : 0; - if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len / 2) { - microdesc_cache_rebuild(cache); - } - } + microdesc_cache_rebuild(cache, 0/* only as needed */); return added; } @@ -275,6 +270,9 @@ microdesc_cache_reload(microdesc_cache_t *cache) } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", total); + + microdesc_cache_clean(cache, 0, 0); + return 0; } @@ -323,11 +321,30 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) } } +static int +should_rebuild_md_cache(microdesc_cache_t *cache) +{ + const size_t old_len = + cache->cache_content ? cache->cache_content->size : 0; + const size_t journal_len = cache->journal_len; + const size_t dropped = cache->bytes_dropped; + + if (journal_len < 16384) + return 0; /* Don't bother, not enough has happened yet. */ + if (dropped > (journal_len + old_len) / 3) + return 1; /* We could save 1/3 or more of the currently used space. */ + if (journal_len > old_len / 2) + return 1; /* We should append to the regular file */ + + return 0; +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new - * location. */ + * location. If force is true, do this unconditionally. If + * force is false, do it only if we expect to save space on disk. */ int -microdesc_cache_rebuild(microdesc_cache_t *cache) +microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; FILE *f; @@ -337,11 +354,14 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) off_t off = 0; int orig_size, new_size; - log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); - /* Remove dead descriptors */ microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + if (!force && !should_rebuild_md_cache(cache)) + return 0; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; @@ -396,6 +416,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) write_str_to_file(cache->journal_fname, "", 1); cache->journal_len = 0; + cache->bytes_dropped = 0; new_size = (int)cache->cache_content->size; log_info(LD_DIR, "Done rebuilding microdesc cache. " diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 30cb25d..77ce853 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -22,7 +22,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, int no_save); void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); -int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_rebuild(microdesc_cache_t *cache, int force); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/master] Backport microdesc_cache_clean to 0.2.2
by nickm＠torproject.org 03 May '11

03 May '11

commit 56fbd728c2df7f189390b9a417b4c3ecc3690ede Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:45:15 2011 -0400 Backport microdesc_cache_clean to 0.2.2 Otherwise we have no way to keep authorities' microdesc caches in 0.2.2 from growing without bound. --- changes/bug2230_clean_1 | 4 +++ src/or/microdesc.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ src/or/microdesc.h | 1 + 3 files changed, 56 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_clean_1 b/changes/bug2230_clean_1 new file mode 100644 index 0000000..a4edf94 --- /dev/null +++ b/changes/bug2230_clean_1 @@ -0,0 +1,4 @@ + o Minor features + - Backport code from 0.2.3.x to allow directory authorities to clean + their microdescriptor caches. + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0ceb134..58fbe37 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -23,6 +23,8 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; /** Total bytes of microdescriptor bodies we have added to this cache */ uint64_t total_len_seen; @@ -276,6 +278,51 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from cache that haven't been listed for + * a long time. Does not rebuild the cache on disk. If cutoff is + * positive, specifically remove microdescriptors that have been unlisted + * since cutoff. If force is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new * location. */ @@ -291,6 +338,10 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) int orig_size, new_size; log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; diff --git a/src/or/microdesc.h b/src/or/microdesc.h index b3e12f8..30cb25d 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -21,6 +21,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save); +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 03 May '11

03 May '11

commit 97a55b80894b2816f435a1ee06a2549f0da72939 Merge: 1f97e83 56fbd72 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:46:28 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2'

1 0

[tor/maint-0.2.2] Backport microdesc_cache_clean to 0.2.2
by nickm＠torproject.org 03 May '11

03 May '11

commit 56fbd728c2df7f189390b9a417b4c3ecc3690ede Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:45:15 2011 -0400 Backport microdesc_cache_clean to 0.2.2 Otherwise we have no way to keep authorities' microdesc caches in 0.2.2 from growing without bound. --- changes/bug2230_clean_1 | 4 +++ src/or/microdesc.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ src/or/microdesc.h | 1 + 3 files changed, 56 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_clean_1 b/changes/bug2230_clean_1 new file mode 100644 index 0000000..a4edf94 --- /dev/null +++ b/changes/bug2230_clean_1 @@ -0,0 +1,4 @@ + o Minor features + - Backport code from 0.2.3.x to allow directory authorities to clean + their microdescriptor caches. + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0ceb134..58fbe37 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -23,6 +23,8 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; /** Total bytes of microdescriptor bodies we have added to this cache */ uint64_t total_len_seen; @@ -276,6 +278,51 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from cache that haven't been listed for + * a long time. Does not rebuild the cache on disk. If cutoff is + * positive, specifically remove microdescriptors that have been unlisted + * since cutoff. If force is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + /** Regenerate the main cache file for cache, clear the journal file, * and update every microdesc_t in the cache with pointers to its new * location. */ @@ -291,6 +338,10 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) int orig_size, new_size; log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; diff --git a/src/or/microdesc.h b/src/or/microdesc.h index b3e12f8..30cb25d 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -21,6 +21,7 @@ smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save); +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache);

1 0

[tor/master] Add missing code to set cache->journal_len when reading microdesc journal
by nickm＠torproject.org 03 May '11

03 May '11

commit 698fa0fc67118bea2ea7720455ebbdee705c8c1a Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:22:31 2011 -0400 Add missing code to set cache->journal_len when reading microdesc journal This could be one reason that authorities' journals would grow without bound; related to bug 2230. Bugfix on 0.2.2.6-alpha. Fix by "cypherpunks". --- changes/bug2230_part1 | 7 +++++++ src/or/microdesc.c | 1 + 2 files changed, 8 insertions(+), 0 deletions(-) diff --git a/changes/bug2230_part1 b/changes/bug2230_part1 new file mode 100644 index 0000000..79f7254 --- /dev/null +++ b/changes/bug2230_part1 @@ -0,0 +1,7 @@ + o Minor bugfixes + - When loading the microdesc journal, remember its current size. + In 0.2.2, this helps prevent the microdesc journal from growing + without limit on authorities (who are the only ones to use it in + 0.2.2). Fixes a part of bug 2230; bugfix on 0.2.2.6-alpha. + Fix posted by "cypherpunks." + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 2c4b343..3566277 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -261,6 +261,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) journal_content = read_file_to_str(cache->journal_fname, RFTS_IGNORE_MISSING, &st); if (journal_content) { + cache->journal_len = (size_t) st.st_size; added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0);

1 0

[tor/master] Fix a check for when to rebuild the microdesc cache. (Backport from 0.2.3.
by nickm＠torproject.org 03 May '11

03 May '11

commit 970715dd8f52a79bbaddfcfa5af1f11608ddacf9 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue May 3 16:29:39 2011 -0400 Fix a check for when to rebuild the microdesc cache. (Backport from 0.2.3. --- changes/bug2230_part2 | 5 +++++ src/or/microdesc.c | 2 +- 2 files changed, 6 insertions(+), 1 deletions(-) diff --git a/changes/bug2230_part2 b/changes/bug2230_part2 new file mode 100644 index 0000000..2664ecc --- /dev/null +++ b/changes/bug2230_part2 @@ -0,0 +1,5 @@ + o Minor bugfixes + - The microdesc journal is supposed to get rebuilt only if it is + at least _half_ the length of the store, not _twice_ the length + of the store. Bugfix on 0.2.2.6-alpha; fixes part of bug 2230. + diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 3566277..0ceb134 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -208,7 +208,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, size_t old_content_len = cache->cache_content ? cache->cache_content->size : 0; if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len * 2) { + cache->journal_len > old_content_len / 2) { microdesc_cache_rebuild(cache); } }

1 0