[tor-commits] [tor/master] Replace "read consensus from disk" with "map consensus from disk".
nickm at torproject.org
nickm at torproject.org
Wed Oct 31 14:13:23 UTC 2018
commit 7e3005af30b94fd1925b0be475d72875272b9044
Author: Nick Mathewson <nickm at torproject.org>
Date: Fri Sep 7 19:38:21 2018 -0400
Replace "read consensus from disk" with "map consensus from disk".
Implements 27244, and should save a bunch of RAM on clients.
---
src/feature/dirauth/dirvote.c | 4 +-
src/feature/dircache/directory.c | 18 ++++----
src/feature/nodelist/networkstatus.c | 89 +++++++++++++++++++-----------------
src/feature/nodelist/networkstatus.h | 4 +-
src/test/test_routerlist.c | 3 +-
5 files changed, 65 insertions(+), 53 deletions(-)
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index 42821760c..771e0363a 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -3648,7 +3648,9 @@ dirvote_publish_consensus(void)
continue;
}
- if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+ if (networkstatus_set_current_consensus(pending->body,
+ strlen(pending->body),
+ name, 0, NULL))
log_warn(LD_DIR, "Error publishing %s consensus", name);
else
log_notice(LD_DIR, "Published %s consensus", name);
diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c
index 8e5fc8683..147329902 100644
--- a/src/feature/dircache/directory.c
+++ b/src/feature/dircache/directory.c
@@ -2606,17 +2606,17 @@ handle_response_fetch_consensus(dir_connection_t *conn,
if (looks_like_a_consensus_diff(body, body_len)) {
/* First find our previous consensus. Maybe it's in ram, maybe not. */
cached_dir_t *cd = dirserv_get_consensus(flavname);
- const char *consensus_body;
+ const char *consensus_body = NULL;
size_t consensus_body_len;
- char *owned_consensus = NULL;
+ tor_mmap_t *mapped_consensus = NULL;
if (cd) {
consensus_body = cd->dir;
consensus_body_len = cd->dir_len;
} else {
- owned_consensus = networkstatus_read_cached_consensus(flavname);
- if (owned_consensus) {
- consensus_body = owned_consensus;
- consensus_body_len = strlen(consensus_body);
+ mapped_consensus = networkstatus_map_cached_consensus(flavname);
+ if (mapped_consensus) {
+ consensus_body = mapped_consensus->data;
+ consensus_body_len = mapped_consensus->size;
}
}
if (!consensus_body) {
@@ -2629,7 +2629,7 @@ handle_response_fetch_consensus(dir_connection_t *conn,
new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
body, body_len);
- tor_free(owned_consensus);
+ tor_munmap_file(mapped_consensus);
if (new_consensus == NULL) {
log_warn(LD_DIR, "Could not apply consensus diff received from server "
"'%s:%d'", conn->base_.address, conn->base_.port);
@@ -2651,7 +2651,9 @@ handle_response_fetch_consensus(dir_connection_t *conn,
sourcename = "downloaded";
}
- if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+ if ((r=networkstatus_set_current_consensus(consensus,
+ strlen(consensus),
+ flavname, 0,
conn->identity_digest))<0) {
log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
"Unable to load %s consensus directory %s from "
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index 4338fde59..847ca0cdf 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -105,8 +105,6 @@ STATIC networkstatus_t *current_md_consensus = NULL;
typedef struct consensus_waiting_for_certs_t {
/** The consensus itself. */
networkstatus_t *consensus;
- /** The encoded version of the consensus, nul-terminated. */
- char *body;
/** When did we set the current value of consensus_waiting_for_certs? If
* this is too recent, we shouldn't try to fetch a new consensus for a
* little while, to give ourselves time to get certificates for this one. */
@@ -199,14 +197,11 @@ networkstatus_reset_download_failures(void)
download_status_reset(&consensus_bootstrap_dl_status[i]);
}
-/**
- * Read and and return the cached consensus of type <b>flavorname</b>. If
- * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
- * the file isn't there. */
+/** Return the filename used to cache the consensus of a given flavor */
static char *
-networkstatus_read_cached_consensus_impl(int flav,
- const char *flavorname,
- int unverified_consensus)
+networkstatus_get_cache_fname(int flav,
+ const char *flavorname,
+ int unverified_consensus)
{
char buf[128];
const char *prefix;
@@ -221,21 +216,35 @@ networkstatus_read_cached_consensus_impl(int flav,
tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
}
- char *filename = get_cachedir_fname(buf);
- char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+ return get_cachedir_fname(buf);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>. If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static tor_mmap_t *
+networkstatus_map_cached_consensus_impl(int flav,
+ const char *flavorname,
+ int unverified_consensus)
+{
+ char *filename = networkstatus_get_cache_fname(flav,
+ flavorname,
+ unverified_consensus);
+ tor_mmap_t *result = tor_mmap_file(filename);
tor_free(filename);
return result;
}
-/** Return a new string containing the current cached consensus of flavor
- * <b>flavorname</b>. */
-char *
-networkstatus_read_cached_consensus(const char *flavorname)
- {
+/** Map the file containing the current cached consensus of flavor
+ * <b>flavorname</b> */
+tor_mmap_t *
+networkstatus_map_cached_consensus(const char *flavorname)
+{
int flav = networkstatus_parse_flavor_name(flavorname);
if (flav < 0)
return NULL;
- return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+ return networkstatus_map_cached_consensus_impl(flav, flavorname, 0);
}
/** Read every cached v3 consensus networkstatus from the disk. */
@@ -248,24 +257,26 @@ router_reload_consensus_networkstatus(void)
/* FFFF Suppress warnings if cached consensus is bad? */
for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
const char *flavor = networkstatus_get_flavor_name(flav);
- char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
- if (s) {
- if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
+ tor_mmap_t *m = networkstatus_map_cached_consensus_impl(flav, flavor, 0);
+ if (m) {
+ if (networkstatus_set_current_consensus(m->data, m->size,
+ flavor, flags, NULL) < -1) {
log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
flavor);
}
- tor_free(s);
+ tor_munmap_file(m);
}
- s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
- if (s) {
- if (networkstatus_set_current_consensus(s, flavor,
+ m = networkstatus_map_cached_consensus_impl(flav, flavor, 1);
+ if (m) {
+ if (networkstatus_set_current_consensus(m->data, m->size,
+ flavor,
flags | NSSET_WAS_WAITING_FOR_CERTS,
NULL)) {
log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
"from cache", flavor);
}
- tor_free(s);
+ tor_munmap_file(m);
}
}
@@ -1833,6 +1844,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor,
*/
int
networkstatus_set_current_consensus(const char *consensus,
+ size_t consensus_len,
const char *flavor,
unsigned flags,
const char *source_dir)
@@ -1862,7 +1874,7 @@ networkstatus_set_current_consensus(const char *consensus,
/* Make sure it's parseable. */
c = networkstatus_parse_vote_from_string(consensus,
- strlen(consensus),
+ consensus_len,
NULL, NS_TYPE_CONSENSUS);
if (!c) {
log_warn(LD_DIR, "Unable to parse networkstatus consensus");
@@ -1951,14 +1963,12 @@ networkstatus_set_current_consensus(const char *consensus,
c->valid_after > current_valid_after) {
waiting = &consensus_waiting_for_certs[flav];
networkstatus_vote_free(waiting->consensus);
- tor_free(waiting->body);
waiting->consensus = c;
free_consensus = 0;
- waiting->body = tor_strdup(consensus);
waiting->set_at = now;
waiting->dl_failed = 0;
if (!from_cache) {
- write_str_to_file(unverified_fname, consensus, 0);
+ write_bytes_to_file(unverified_fname, consensus, consensus_len, 0);
}
if (dl_certs)
authority_certs_fetch_missing(c, now, source_dir);
@@ -2049,10 +2059,6 @@ networkstatus_set_current_consensus(const char *consensus,
waiting->consensus->valid_after <= c->valid_after) {
networkstatus_vote_free(waiting->consensus);
waiting->consensus = NULL;
- if (consensus != waiting->body)
- tor_free(waiting->body);
- else
- waiting->body = NULL;
waiting->set_at = 0;
waiting->dl_failed = 0;
if (unlink(unverified_fname) != 0) {
@@ -2147,14 +2153,16 @@ networkstatus_note_certs_arrived(const char *source_dir)
if (!waiting->consensus)
continue;
if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
- char *waiting_body = waiting->body;
- if (!networkstatus_set_current_consensus(
- waiting_body,
- flavor_name,
- NSSET_WAS_WAITING_FOR_CERTS,
- source_dir)) {
- tor_free(waiting_body);
+ tor_mmap_t *mapping = networkstatus_map_cached_consensus_impl(
+ i, flavor_name, 1);
+ if (mapping) {
+ networkstatus_set_current_consensus(mapping->data,
+ mapping->size,
+ flavor_name,
+ NSSET_WAS_WAITING_FOR_CERTS,
+ source_dir);
}
+ tor_munmap_file(mapping);
}
}
}
@@ -2730,6 +2738,5 @@ networkstatus_free_all(void)
networkstatus_vote_free(waiting->consensus);
waiting->consensus = NULL;
}
- tor_free(waiting->body);
}
}
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
index cc6badf0b..997d7cfa3 100644
--- a/src/feature/nodelist/networkstatus.h
+++ b/src/feature/nodelist/networkstatus.h
@@ -16,7 +16,7 @@
void networkstatus_reset_warnings(void);
void networkstatus_reset_download_failures(void);
-char *networkstatus_read_cached_consensus(const char *flavorname);
+tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname);
int router_reload_consensus_networkstatus(void);
void routerstatus_free_(routerstatus_t *rs);
#define routerstatus_free(rs) \
@@ -108,6 +108,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options);
#define NSSET_ACCEPT_OBSOLETE 8
#define NSSET_REQUIRE_FLAVOR 16
int networkstatus_set_current_consensus(const char *consensus,
+ size_t consensus_len,
const char *flavor,
unsigned flags,
const char *source_dir);
@@ -159,4 +160,3 @@ extern networkstatus_t *current_md_consensus;
#endif /* defined(NETWORKSTATUS_PRIVATE) */
#endif /* !defined(TOR_NETWORKSTATUS_H) */
-
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 7fe4c15b1..bf76570b3 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -659,7 +659,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
MOCK(clock_skew_warning, mock_clock_skew_warning);
/* Caller will call teardown_capture_of_logs() */
setup_capture_of_logs(LOG_WARN);
- retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+ retval = networkstatus_set_current_consensus(consensus, strlen(consensus),
+ "microdesc", 0,
NULL);
done:
More information about the tor-commits
mailing list