commit a1172b6774bc95ff951ac58051ea42bc0a62e32b Author: Nick Mathewson nickm@torproject.org Date: Tue Apr 25 14:52:40 2017 -0400

Store archived consensuses compressed on disk.

I'm just using gzip compression now, for performance. --- src/or/consdiffmgr.c | 74 +++++++++++++++++++++++++++++++++++++++------ src/or/consdiffmgr.h | 2 ++ src/test/test_consdiffmgr.c | 15 +++++---- 3 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index adb3fc2..2deeab5 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -110,6 +110,9 @@ static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); static void consdiffmgr_set_cache_flags(void);

+/* Just gzip consensuses for now. */ +#define COMPRESS_CONSENSUS_WITH GZIP_METHOD + /* ===== * Hashtable setup * ===== */ @@ -420,18 +423,28 @@ consdiffmgr_add_consensus(const char *consensus, format_iso_time_nospace(formatted_time, valid_after); const char *flavname = networkstatus_get_flavor_name(flavor);

- cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST, - (const uint8_t *)consensus, bodylen); cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, (const uint8_t *)consensus, bodylen); + + char *body_compressed = NULL; + size_t size_compressed = 0; + if (tor_compress(&body_compressed, &size_compressed, + consensus, bodylen, COMPRESS_CONSENSUS_WITH) < 0) { + config_free_lines(labels); + return -1; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST, + (const uint8_t *)body_compressed, size_compressed); + config_line_prepend(&labels, LABEL_COMPRESSION_TYPE, + compression_method_get_name(COMPRESS_CONSENSUS_WITH)); config_line_prepend(&labels, LABEL_FLAVOR, flavname); config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);

entry = consensus_cache_add(cdm_cache_get(), labels, - (const uint8_t *)consensus, - bodylen); + (const uint8_t *)body_compressed, + size_compressed); config_free_lines(labels); }

@@ -915,6 +928,40 @@ typedef struct consensus_diff_worker_job_t { size_t bodylen_out; } consensus_diff_worker_job_t;

+/** Given a consensus_cache_entry_t, check whether it has a label claiming + * that it was compressed. If so, uncompress its contents into <b>out</b> and + * set <b>outlen</b> to hold their size. If not, just copy the body into + * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success, + * -1 on failure. + * + * In all cases, the output is nul-terminated. */ +STATIC int +uncompress_or_copy(char **out, size_t *outlen, + consensus_cache_entry_t *ent) +{ + const uint8_t *body; + size_t bodylen; + + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) + return -1; + + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = NO_METHOD; + + if (lv_compression) + method = compression_method_get_by_name(lv_compression); + + if (method == NO_METHOD) { + *out = tor_memdup_nulterm(body, bodylen); + *outlen = bodylen; + return 0; + } else { + return tor_uncompress(out, outlen, (const char *)body, bodylen, + method, 1, LOG_WARN); + } +} + /** * Worker function. This function runs inside a worker thread and receives * a consensus_diff_worker_job_t as its input. @@ -966,11 +1013,20 @@ consensus_diff_worker_threadfn(void *state_, void *work_)

char *consensus_diff; { - // XXXX the input might not be nul-terminated. And also we wanted to - // XXXX support compression later I guess. So, we need to copy here. - char *diff_from_nt, *diff_to_nt; - diff_from_nt = tor_memdup_nulterm(diff_from, len_from); - diff_to_nt = tor_memdup_nulterm(diff_to, len_to); + char *diff_from_nt = NULL, *diff_to_nt = NULL; + size_t diff_from_nt_len, diff_to_nt_len; + + if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, + job->diff_from) < 0) { + return WQ_RPL_REPLY; + } + if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, + job->diff_to) < 0) { + tor_free(diff_from_nt); + return WQ_RPL_REPLY; + } + tor_assert(diff_from_nt); + tor_assert(diff_to_nt);

// XXXX ugh; this is going to calculate the SHA3 of both its // XXXX inputs again, even though we already have that. Maybe it's time diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index 6932b2f..982e0df 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -42,6 +42,8 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label); +STATIC int uncompress_or_copy(char **out, size_t *outlen, + consensus_cache_entry_t *ent); #endif

#endif diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 2fb7dd2..b3fa388 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -149,7 +149,6 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, const char *str1, const char *str2) { - char *diff_string = NULL; consensus_cache_entry_t *ent = NULL; consdiff_status_t status = lookup_diff_from(&ent, flav, str1); if (ent == NULL || status != CONSDIFF_AVAILABLE) @@ -157,12 +156,10 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav,

consensus_cache_entry_incref(ent); size_t size; - const uint8_t *body; - int r = consensus_cache_entry_get_body(ent, &body, &size); - if (r == 0) - diff_string = tor_memdup_nulterm(body, size); + char *diff_string = NULL; + int r = uncompress_or_copy(&diff_string, &size, ent); consensus_cache_entry_decref(ent); - if (diff_string == NULL) + if (diff_string == NULL || r < 0) return -1;

char *applied = consensus_diff_apply(str1, diff_string); @@ -266,6 +263,8 @@ test_consdiffmgr_add(void *arg) (void) arg; time_t now = approx_time();

+ char *body = NULL; + consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); const char *dummy = "foo"; @@ -306,8 +305,7 @@ test_consdiffmgr_add(void *arg) tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; - const uint8_t *body; - r = consensus_cache_entry_get_body(ent, &body, &s); + r = uncompress_or_copy(&body, &s, ent); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); @@ -320,6 +318,7 @@ test_consdiffmgr_add(void *arg) networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); + tor_free(body); }

static void