commit 3af9704e459fab0120694a46924b27460237cb1a Author: Nick Mathewson nickm@torproject.org Date: Wed May 3 12:56:16 2017 -0400

bug#22143/prop#140: in consdiffmgr, store and use digest-as-signed

We need to index diffs by the digest-as-signed of their source consensus, so that we can find them even from consensuses whose signatures are encoded differently. --- src/or/consdiffmgr.c | 35 ++++++++++++++++++++++++++++------- src/test/test_consdiffmgr.c | 23 ++++++++++++++++------- 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 29807e7..96b0bba 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -19,6 +19,7 @@ #include "consdiffmgr.h" #include "cpuworker.h" #include "networkstatus.h" +#include "routerparse.h" #include "workqueue.h"

/** @@ -35,11 +36,13 @@ #define LABEL_SHA3_DIGEST "sha3-digest" /* A hex encoded SHA3 digest of the object before compression. */ #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed" +/* A hex encoded SHA3 digest-as-signed of a consensus */ +#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed" /* The flavor of the consensus or consensuses diff */ #define LABEL_FLAVOR "consensus-flavor" -/* Diff only: the SHA3 digest of the source consensus. */ +/* Diff only: the SHA3 digest-as-signed of the source consensus. */ #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest" -/* Diff only: the SHA3 digest of the target consensus. */ +/* Diff only: the SHA3 digest-in-full of the target consensus. */ #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest" /* Diff only: the valid-after date of the source consensus. */ #define LABEL_FROM_VALID_AFTER "from-valid-after" @@ -466,6 +469,17 @@ consdiffmgr_add_consensus(const char *consensus,

cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, (const uint8_t *)consensus, bodylen); + { + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(consensus, + &start, &end) < 0) { + start = consensus; + end = consensus+bodylen; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, + (const uint8_t *)start, + end - start); + }

char *body_compressed = NULL; size_t size_compressed = 0; @@ -845,7 +859,7 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)

uint8_t this_sha3[DIGEST256_LEN]; if (BUG(cdm_entry_get_sha3_value(this_sha3, c, - LABEL_SHA3_DIGEST_UNCOMPRESSED)<0)) + LABEL_SHA3_DIGEST_AS_SIGNED)<0)) continue; // LCOV_EXCL_LINE if (cdm_diff_ht_check_and_note_pending(flavor, this_sha3, most_recent_sha3)) { @@ -1131,7 +1145,7 @@ consensus_diff_worker_threadfn(void *state_, void *work_) consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); const char *lv_from_digest = consensus_cache_entry_get_value(job->diff_from, - LABEL_SHA3_DIGEST_UNCOMPRESSED); + LABEL_SHA3_DIGEST_AS_SIGNED); const char *lv_from_flavor = consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR); const char *lv_to_flavor = @@ -1140,10 +1154,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_) consensus_cache_entry_get_value(job->diff_to, LABEL_SHA3_DIGEST_UNCOMPRESSED);

+ if (! lv_from_digest) { + /* This isn't a bug right now, since it can happen if you're migrating + * from an older version of master to a newer one. The older ones didn't + * annotate their stored consensus objects with sha3-digest-as-signed. + */ + return WQ_RPL_REPLY; // LCOV_EXCL_LINE + } + /* All these values are mandatory on the input */ if (BUG(!lv_to_valid_after) || BUG(!lv_from_valid_after) || - BUG(!lv_from_digest) || BUG(!lv_from_flavor) || BUG(!lv_to_flavor)) { return WQ_RPL_REPLY; // LCOV_EXCL_LINE @@ -1267,7 +1288,7 @@ consensus_diff_worker_replyfn(void *work_)

const char *lv_from_digest = consensus_cache_entry_get_value(job->diff_from, - LABEL_SHA3_DIGEST_UNCOMPRESSED); + LABEL_SHA3_DIGEST_AS_SIGNED); const char *lv_to_digest = consensus_cache_entry_get_value(job->diff_to, LABEL_SHA3_DIGEST_UNCOMPRESSED); @@ -1283,7 +1304,7 @@ consensus_diff_worker_replyfn(void *work_) int flav = -1; int cache = 1; if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from, - LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0)) + LABEL_SHA3_DIGEST_AS_SIGNED) < 0)) cache = 0; if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to, LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0)) diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 31ce6ce..0e44ed2 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -10,6 +10,7 @@ #include "consdiffmgr.h" #include "cpuworker.h" #include "networkstatus.h" +#include "routerparse.h" #include "workqueue.h"

#include "test.h" @@ -66,6 +67,7 @@ fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)

format_iso_time(valid_after_string, valid_after); char *random_stuff = crypto_random_hostname(3, 25, "junk ", ""); + char *random_stuff2 = crypto_random_hostname(3, 10, "", "");

char *consensus; tor_asprintf(&consensus, @@ -74,11 +76,15 @@ fake_ns_body_new(consensus_flavor_t flav, time_t valid_after) "valid-after %s\n" "r name ccccccccccccccccc etc\nsample\n" "r name eeeeeeeeeeeeeeeee etc\nbar\n" - "%s\n", + "%s\n" + "directory-signature hello-there\n" + "directory-signature %s\n", flavor_string, valid_after_string, - random_stuff); + random_stuff, + random_stuff2); tor_free(random_stuff); + tor_free(random_stuff2); return consensus; }

@@ -139,7 +145,10 @@ lookup_diff_from(consensus_cache_entry_t **out, const char *str1) { uint8_t digest[DIGEST256_LEN]; - crypto_digest256((char*)digest, str1, strlen(str1), DIGEST_SHA3_256); + if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) { + TT_FAIL(("Unable to compute sha3-as-signed")); + return CONSDIFF_NOT_FOUND; + } return consdiffmgr_find_diff_from(out, flav, DIGEST_SHA3_256, digest, sizeof(digest), NO_METHOD); @@ -152,8 +161,9 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, { consensus_cache_entry_t *ent = NULL; consdiff_status_t status = lookup_diff_from(&ent, flav, str1); - if (ent == NULL || status != CONSDIFF_AVAILABLE) + if (ent == NULL || status != CONSDIFF_AVAILABLE) { return -1; + }

consensus_cache_entry_incref(ent); size_t size; @@ -299,7 +309,7 @@ test_consdiffmgr_add(void *arg) ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */ r = consdiffmgr_add_consensus(dummy, ns_tmp); tt_int_op(r, OP_EQ, -1); - expect_single_log_msg_containing("it's too old."); + expect_log_msg_containing("it's too old.");

/* Try looking up a consensuses. */ ent = cdm_cache_lookup_consensus(FLAV_NS, now-60); @@ -352,8 +362,7 @@ test_consdiffmgr_make_diffs(void *arg) ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); - crypto_digest256((char*)md_ns_sha3, md_ns_body, strlen(md_ns_body), - DIGEST_SHA3_256); + router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0);