[or-cvs] [tor/master] Fix various bugs in microdescriptor caching.
Nick Mathewson
nickm at seul.org
Mon Oct 19 04:48:30 UTC 2009
Author: Nick Mathewson <nickm at torproject.org>
Date: Wed, 14 Oct 2009 16:05:08 -0400
Subject: Fix various bugs in microdescriptor caching.
Commit: d61b5df9c1bed57cb39888a1f256cf6c234c29eb
---
src/or/main.c | 1 +
src/or/microdesc.c | 105 ++++++++++++++++++++++++++++++++++++++----------
src/or/networkstatus.c | 2 +-
src/or/or.h | 1 +
4 files changed, 86 insertions(+), 23 deletions(-)
diff --git a/src/or/main.c b/src/or/main.c
index 2518291..9605b2b 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -1985,6 +1985,7 @@ tor_free_all(int postfork)
connection_free_all();
buf_shrink_freelists(1);
memarea_clear_freelist();
+ microdesc_free_all();
if (!postfork) {
config_free_all();
router_free_all();
diff --git a/src/or/microdesc.c b/src/or/microdesc.c
index 0128fbb..2533564 100644
--- a/src/or/microdesc.c
+++ b/src/or/microdesc.c
@@ -40,19 +40,28 @@ HT_GENERATE(microdesc_map, microdesc_t, node,
_microdesc_hash, _microdesc_eq, 0.6,
_tor_malloc, _tor_realloc, _tor_free);
+/* returns n bytes written */
static int
-dump_microdescriptor(FILE *f, microdesc_t *md)
+dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out)
{
+ int r = 0;
/* XXXX drops unkown annotations. */
if (md->last_listed) {
char buf[ISO_TIME_LEN+1];
+ char annotation[ISO_TIME_LEN+32];
format_iso_time(buf, md->last_listed);
- fprintf(f, "@last-listed %s\n", buf);
+ tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
+ fputs(annotation, f);
+ r += strlen(annotation);
+ *annotation_len_out = r;
+ } else {
+ *annotation_len_out = 0;
}
md->off = (off_t) ftell(f);
fwrite(md->body, 1, md->bodylen, f);
- return 0;
+ r += md->bodylen;
+ return r;
}
static microdesc_cache_t *the_microdesc_cache = NULL;
@@ -72,7 +81,7 @@ get_microdesc_cache(void)
}
/* There are three sources of microdescriptors:
- 1) Generated us while acting as a directory authority.
+ 1) Generated by us while acting as a directory authority.
2) Loaded from the cache on disk.
3) Downloaded.
*/
@@ -98,7 +107,8 @@ microdescs_add_to_cache(microdesc_cache_t *cache,
return added;
}
-/* Returns list of added microdesc_t. Frees any not added. */
+/* Returns list of added microdesc_t. Frees any not added. Updates last_listed.
+ */
smartlist_t *
microdescs_add_list_to_cache(microdesc_cache_t *cache,
smartlist_t *descriptors, saved_location_t where,
@@ -108,13 +118,17 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache,
open_file_t *open_file = NULL;
FILE *f = NULL;
// int n_added = 0;
+ size_t size = 0;
if (where == SAVED_NOWHERE && !no_save) {
- f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND,
+ f = start_writing_to_stdio_file(cache->journal_fname,
+ OPEN_FLAGS_APPEND|O_BINARY,
0600, &open_file);
- if (!f)
- log_warn(LD_DIR, "Couldn't append to journal in %s",
- cache->journal_fname);
+ if (!f) {
+ log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
+ cache->journal_fname, strerror(errno));
+ return NULL;
+ }
}
added = smartlist_create();
@@ -131,8 +145,10 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache,
/* Okay, it's a new one. */
if (f) {
- dump_microdescriptor(f, md);
+ int annotation_len;
+ size = dump_microdescriptor(f, md, &annotation_len);
md->saved_location = SAVED_IN_JOURNAL;
+ cache->journal_len += size;
} else {
md->saved_location = where;
}
@@ -143,7 +159,18 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache,
smartlist_add(added, md);
} SMARTLIST_FOREACH_END(md);
- finish_writing_to_file(open_file); /*XXX Check me.*/
+ if (f)
+ finish_writing_to_file(open_file); /*XXX Check me.*/
+
+ {
+ size_t old_content_len =
+ cache->cache_content ? cache->cache_content->size : 0;
+ if (cache->journal_len > 16384 + old_content_len &&
+ cache->journal_len > old_content_len * 2) {
+ microdesc_cache_rebuild(cache);
+ }
+ }
+
return added;
}
@@ -152,9 +179,11 @@ microdesc_cache_clear(microdesc_cache_t *cache)
{
microdesc_t **entry, **next;
for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
+ microdesc_t *md = *entry;
next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
- microdesc_free(*entry);
+ microdesc_free(md);
}
+ HT_CLEAR(microdesc_map, &cache->map);
if (cache->cache_content) {
tor_munmap_file(cache->cache_content);
cache->cache_content = NULL;
@@ -176,8 +205,10 @@ microdesc_cache_reload(microdesc_cache_t *cache)
if (mm) {
added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
SAVED_IN_CACHE, 0);
- total += smartlist_len(added);
- smartlist_free(added);
+ if (added) {
+ total += smartlist_len(added);
+ smartlist_free(added);
+ }
}
journal_content = read_file_to_str(cache->journal_fname,
@@ -186,8 +217,10 @@ microdesc_cache_reload(microdesc_cache_t *cache)
added = microdescs_add_to_cache(cache, journal_content,
journal_content+st.st_size,
SAVED_IN_JOURNAL, 0);
- total += smartlist_len(added);
- smartlist_free(added);
+ if (added) {
+ total += smartlist_len(added);
+ smartlist_free(added);
+ }
tor_free(journal_content);
}
log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
@@ -202,8 +235,16 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
FILE *f;
microdesc_t **mdp;
smartlist_t *wrote;
+ int size;
+ off_t off = 0;
+ int orig_size, new_size;
- f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE,
+ log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+ orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
+ orig_size += (int)cache->journal_len;
+
+ f = start_writing_to_stdio_file(cache->cache_fname,
+ OPEN_FLAGS_REPLACE|O_BINARY,
0600, &open_file);
if (!f)
return -1;
@@ -212,15 +253,17 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
HT_FOREACH(mdp, microdesc_map, &cache->map) {
microdesc_t *md = *mdp;
+ int annotation_len;
if (md->no_save)
continue;
- dump_microdescriptor(f, md);
+ size = dump_microdescriptor(f, md, &annotation_len);
+ md->off = off + annotation_len;
+ off += size;
if (md->saved_location != SAVED_IN_CACHE) {
tor_free(md->body);
md->saved_location = SAVED_IN_CACHE;
}
-
smartlist_add(wrote, md);
}
@@ -229,21 +272,29 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
if (cache->cache_content)
tor_munmap_file(cache->cache_content);
cache->cache_content = tor_mmap_file(cache->cache_fname);
+
if (!cache->cache_content && smartlist_len(wrote)) {
log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
cache->cache_fname);
+ smartlist_free(wrote);
return -1;
}
SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
- if (md->no_save)
- continue;
tor_assert(md->saved_location == SAVED_IN_CACHE);
md->body = (char*)cache->cache_content->data + md->off;
tor_assert(!memcmp(md->body, "onion-key", 9));
- } SMARTLIST_FOREACH_END(wrote);
+ } SMARTLIST_FOREACH_END(md);
smartlist_free(wrote);
+ write_str_to_file(cache->journal_fname, "", 1);
+ cache->journal_len = 0;
+
+ new_size = (int)cache->cache_content->size;
+ log_info(LD_DIR, "Done rebuilding microdesc cache. "
+ "Saved %d bytes; %d still used.",
+ orig_size-new_size, new_size);
+
return 0;
}
@@ -265,3 +316,13 @@ microdesc_free(microdesc_t *md)
tor_free(md);
}
+void
+microdesc_free_all(void)
+{
+ if (the_microdesc_cache) {
+ microdesc_cache_clear(the_microdesc_cache);
+ tor_free(the_microdesc_cache->cache_fname);
+ tor_free(the_microdesc_cache->journal_fname);
+ tor_free(the_microdesc_cache);
+ }
+}
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index 6481499..37838f4 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -1664,7 +1664,7 @@ networkstatus_set_current_consensus(const char *consensus,
dirserv_set_cached_consensus_networkstatus(consensus,
flavor,
&c->digests,
- current_valid_after);
+ c->valid_after);
}
if (flav == USABLE_CONSENSUS_FLAVOR) {
diff --git a/src/or/or.h b/src/or/or.h
index b2e16a3..63d2eee 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -4123,6 +4123,7 @@ int microdesc_cache_reload(microdesc_cache_t *cache);
void microdesc_cache_clear(microdesc_cache_t *cache);
void microdesc_free(microdesc_t *md);
+void microdesc_free_all(void);
/********************************* networkstatus.c *********************/
--
1.5.6.5
More information about the tor-commits
mailing list