May 2017 - tor-commits - lists.torproject.org

[tor/master] Serve consensus diffs on request.
by nickm＠torproject.org 04 May '17

04 May '17

commit 0418357ffd575ff4a3ec95937f596776c3b9ecec Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 28 14:36:24 2017 -0400 Serve consensus diffs on request. --- src/or/directory.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 4d03c12..cf272f7 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -13,6 +13,7 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "consdiffmgr.h" #include "control.h" #include "compat.h" #define DIRECTORY_PRIVATE @@ -3180,6 +3181,16 @@ write_http_response_header(dir_connection_t *conn, ssize_t length, cache_lifetime); } +/** Array of compression methods to use (if supported) for serving + * precompressed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_precompressed[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + /** Parse the compression methods listed in an Accept-Encoding header h, * and convert them to a bitfield where compression method x is supported if * and only if 1 << x is set in the bitfield. */ @@ -3483,6 +3494,69 @@ warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) } } +/** If there is an X-Or-Diff-From-Consensus header included in headers, + * set digest_out to a new smartlist containing every 256-bit + * hex-encoded digest listed in that header and return 0. Otherwise return + * -1. */ +static int +parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) +{ + char *hdr = http_get_header(headers, "X-Or-Diff-From-Consensus"); + if (hdr == NULL) { + return -1; + } + smartlist_t *hex_digests = smartlist_new(); + *digests_out = smartlist_new(); + smartlist_split_string(hex_digests, hdr, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { + uint8_t digest[DIGEST256_LEN]; + if (base16_decode((char*)digest, sizeof(digest), hex, strlen(hex)) == + DIGEST256_LEN) { + smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "X-Or-Diff-From-Consensus header contained bogus digest %s; " + "ignoring.", escaped(hex)); + } + } SMARTLIST_FOREACH_END(hex); + SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); + smartlist_free(hex_digests); + return 0; +} + +/** + * Try to find the best consensus diff possible in order to serve a client + * request for a diff from one of the consensuses in digests to the + * current consensus of flavor flav. The client supports the + * compression methods listed in the compression_methods bitfield: + * place the method chosen (if any) into compression_used_out. + */ +static struct consensus_cache_entry_t * +find_best_diff(const smartlist_t *digests, int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (0 == (compression_methods & (1u<<method))) + continue; // client doesn't like this one, or we don't have it. + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, + diff_from, DIGEST256_LEN, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; + } + } + } SMARTLIST_FOREACH_END(diff_from); + return NULL; +} + /** Helper function for GET /tor/status-vote/current/consensus */ static int @@ -3542,16 +3616,33 @@ handle_get_current_consensus(dir_connection_t *conn, goto done; } + struct consensus_cache_entry_t *cached_diff = NULL; + smartlist_t *diff_from_digests = NULL; + compress_method_t compression_used = NO_METHOD; + if (!parse_or_diff_from_header(&diff_from_digests, args->headers)) { + tor_assert(diff_from_digests); + cached_diff = find_best_diff(diff_from_digests, flav, + args->compression_supported, + &compression_used); + SMARTLIST_FOREACH(diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(diff_from_digests); + } + conn->spool = smartlist_new(); clear_spool = 1; { spooled_resource_t *spooled; - if (flavor) + if (cached_diff) { + spooled = spooled_resource_new_from_cache_entry(cached_diff); + } else if (flavor) { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, (uint8_t*)flavor, strlen(flavor)); - else + compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + } else { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, NULL, 0); + compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + } tor_free(flavor); smartlist_add(conn->spool, spooled); } @@ -3606,7 +3697,7 @@ handle_get_current_consensus(dir_connection_t *conn, clear_spool = 0; write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compression_used, smartlist_len(conn->spool) == 1 ? lifetime : 0); if (! compressed) conn->compress_state = tor_compress_new(0, ZLIB_METHOD,

1 0

[tor/master] Request (and try to use) consensus diffs.
by nickm＠torproject.org 04 May '17

04 May '17

commit c12d2cb2dcfe52aadf528a70433b468327472eaf Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 3 15:08:59 2017 -0400 Request (and try to use) consensus diffs. --- src/or/consdiff.c | 9 +++++++ src/or/consdiff.h | 2 ++ src/or/directory.c | 58 +++++++++++++++++++++++++++++++++++++---- src/or/networkstatus.c | 71 ++++++++++++++++++++++++++++++++------------------ src/or/networkstatus.h | 1 + 5 files changed, 111 insertions(+), 30 deletions(-) diff --git a/src/or/consdiff.c b/src/or/consdiff.c index 3c2140b..1baa118 100644 --- a/src/or/consdiff.c +++ b/src/or/consdiff.c @@ -1401,3 +1401,12 @@ consensus_diff_apply(const char *consensus, return result; } +/** Return true iff, based on its header, document is likely + * to be a consensus diff. */ +int +looks_like_a_consensus_diff(const char *document, size_t len) +{ + return (len >= strlen(ns_diff_version) && + fast_memeq(document, ns_diff_version, strlen(ns_diff_version))); +} + diff --git a/src/or/consdiff.h b/src/or/consdiff.h index 284a576..d05df74 100644 --- a/src/or/consdiff.h +++ b/src/or/consdiff.h @@ -12,6 +12,8 @@ char *consensus_diff_generate(const char *cons1, char *consensus_diff_apply(const char *consensus, const char *diff); +int looks_like_a_consensus_diff(const char *document, size_t len); + #ifdef CONSDIFF_PRIVATE struct memarea_t; diff --git a/src/or/directory.c b/src/or/directory.c index 76fc121..bf963f1 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -13,6 +13,7 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "consdiff.h" #include "consdiffmgr.h" #include "control.h" #include "compat.h" @@ -2417,6 +2418,10 @@ handle_response_fetch_consensus(dir_connection_t *conn, const char *reason = args->reason; const time_t now = approx_time(); + const char *consensus; + char *new_consensus = NULL; + const char *sourcename; + int r; const char *flavname = conn->requested_resource; if (status_code != 200) { @@ -2429,15 +2434,57 @@ handle_response_fetch_consensus(dir_connection_t *conn, networkstatus_consensus_download_failed(status_code, flavname); return -1; } - log_info(LD_DIR,"Received consensus directory (body size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if ((r=networkstatus_set_current_consensus(body, flavname, 0, + + if (looks_like_a_consensus_diff(body, body_len)) { + /* First find our previous consensus. Maybe it's in ram, maybe not. */ + cached_dir_t *cd = dirserv_get_consensus(flavname); + const char *consensus_body; + char *owned_consensus = NULL; + if (cd) { + consensus_body = cd->dir; + } else { + owned_consensus = networkstatus_read_cached_consensus(flavname); + consensus_body = owned_consensus; + } + if (!consensus_body) { + log_warn(LD_DIR, "Received a consensus diff, but we can't find " + "any %s-flavored consensus in our current cache.",flavname); + networkstatus_consensus_download_failed(0, flavname); + // XXXX if this happens too much, see below + return -1; + } + + new_consensus = consensus_diff_apply(consensus_body, body); + tor_free(owned_consensus); + if (new_consensus == NULL) { + log_warn(LD_DIR, "Could not apply consensus diff received from server " + "'%s:%d'", conn->base_.address, conn->base_.port); + // XXXX If this happens too many times, we should maybe not use + // XXXX this directory for diffs any more? + networkstatus_consensus_download_failed(0, flavname); + return -1; + } + log_info(LD_DIR, "Applied consensus diff (body size %d) from server " + "'%s:%d' resulted in a new consensus document (size %d).", + (int)body_len, conn->base_.address, conn->base_.port, + (int)strlen(new_consensus)); + consensus = new_consensus; + sourcename = "generated based on a diff"; + } else { + log_info(LD_DIR,"Received consensus directory (body size %d) from server " + "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); + consensus = body; + sourcename = "downloaded"; + } + + if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, - "Unable to load %s consensus directory downloaded from " + "Unable to load %s consensus directory %s from " "server '%s:%d'. I'll try again soon.", - flavname, conn->base_.address, conn->base_.port); + flavname, sourcename, conn->base_.address, conn->base_.port); networkstatus_consensus_download_failed(0, flavname); + tor_free(new_consensus); return -1; } @@ -2455,6 +2502,7 @@ handle_response_fetch_consensus(dir_connection_t *conn, } log_info(LD_DIR, "Successfully loaded consensus."); + tor_free(new_consensus); return 0; } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 8a5cdaf..18537a3 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -179,53 +179,74 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } +/** + * Read and and return the cached consensus of type flavorname. If + * unverified is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static char * +networkstatus_read_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char buf[128]; + const char *prefix; + if (unverified_consensus) { + prefix = "unverified"; + } else { + prefix = "cached"; + } + if (flav == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix); + } else { + tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); + } + + char *filename = get_datadir_fname(buf); + char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + tor_free(filename); + return result; +} + +/** Return a new string containing the current cached consensus of flavor + * flavorname. */ +char * +networkstatus_read_cached_consensus(const char *flavorname) + { + int flav = networkstatus_parse_flavor_name(flavorname); + if (flav < 0) + return NULL; + return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); +} + /** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { - char *filename; - char *s; const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; int flav; /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { - char buf[128]; const char *flavor = networkstatus_get_flavor_name(flav); - if (flav == FLAV_NS) { - filename = get_datadir_fname("cached-consensus"); - } else { - tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); - filename = get_datadir_fname(buf); - } - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); if (s) { if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { - log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", - flavor, filename); + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", + flavor); } tor_free(s); } - tor_free(filename); - if (flav == FLAV_NS) { - filename = get_datadir_fname("unverified-consensus"); - } else { - tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); - filename = get_datadir_fname(buf); - } - - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); if (s) { if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS, NULL)) { - log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", - flavor, filename); - } + log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " + "from cache", flavor); + } tor_free(s); } - tor_free(filename); } if (!networkstatus_get_latest_consensus()) { diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 8a9f5f0..6a90d70 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -16,6 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); +char *networkstatus_read_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free(routerstatus_t *rs); void networkstatus_vote_free(networkstatus_t *ns);

1 0

[tor/master] Add consensus_cache_entry spooling support to spooled_resource_t
by nickm＠torproject.org 04 May '17

04 May '17

commit a32083bd03625c1d99a32971c1c1a574d7e6e014 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun Apr 16 17:47:28 2017 -0400 Add consensus_cache_entry spooling support to spooled_resource_t --- src/or/dirserv.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++------ src/or/dirserv.h | 14 +++++++++++-- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e76fd93..bdf40b6 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -13,6 +13,7 @@ #include "command.h" #include "connection.h" #include "connection_or.h" +#include "conscache.h" #include "control.h" #include "directory.h" #include "dirserv.h" @@ -3392,6 +3393,9 @@ spooled_resource_new(dir_spool_source_t source, default: spooled->spool_eagerly = 1; break; + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: + tor_assert_unreached(); + break; } tor_assert(digestlen <= sizeof(spooled->digest)); if (digest) @@ -3399,6 +3403,33 @@ spooled_resource_new(dir_spool_source_t source, return spooled; } +/** + * Create a new spooled_resource_t to spool the contents of entry to + * the user. Return the spooled object on success, or NULL on failure (which + * is probably caused by a failure to map the body of the item from disk). + * + * Adds a reference to entry's reference counter. + */ +spooled_resource_t * +spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry) +{ + spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t)); + spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY; + spooled->spool_eagerly = 0; + consensus_cache_entry_incref(entry); + spooled->consensus_cache_entry = entry; + + int r = consensus_cache_entry_get_body(entry, + &spooled->cce_body, + &spooled->cce_len); + if (r == 0) { + return spooled; + } else { + spooled_resource_free(spooled); + return NULL; + } +} + /** Release all storage held by spooled. */ void spooled_resource_free(spooled_resource_t *spooled) @@ -3410,6 +3441,10 @@ spooled_resource_free(spooled_resource_t *spooled) cached_dir_decref(spooled->cached_dir_ref); } + if (spooled->consensus_cache_entry) { + consensus_cache_entry_decref(spooled->consensus_cache_entry); + } + tor_free(spooled); } @@ -3456,6 +3491,9 @@ spooled_resource_estimate_size(const spooled_resource_t *spooled, return bodylen; } else { cached_dir_t *cached; + if (spooled->consensus_cache_entry) { + return spooled->cce_len; + } if (spooled->cached_dir_ref) { cached = spooled->cached_dir_ref; } else { @@ -3505,7 +3543,8 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_DONE; } else { cached_dir_t *cached = spooled->cached_dir_ref; - if (cached == NULL) { + consensus_cache_entry_t *cce = spooled->consensus_cache_entry; + if (cached == NULL && cce == NULL) { /* The cached_dir_t hasn't been materialized yet. So let's look it up. */ cached = spooled->cached_dir_ref = spooled_resource_lookup_cached_dir(spooled, NULL); @@ -3517,22 +3556,34 @@ spooled_resource_flush_some(spooled_resource_t *spooled, tor_assert_nonfatal(spooled->cached_dir_offset == 0); } + if (BUG(!cached && !cce)) + return SRFS_DONE; + + int64_t total_len; + const char *ptr; + if (cached) { + total_len = cached->dir_z_len; + ptr = cached->dir_z; + } else { + total_len = spooled->cce_len; + ptr = (const char *)spooled->cce_body; + } /* How many bytes left to flush? */ - int64_t remaining = 0; - remaining = cached->dir_z_len - spooled->cached_dir_offset; + int64_t remaining; + remaining = total_len - spooled->cached_dir_offset; if (BUG(remaining < 0)) return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); if (conn->compress_state) { connection_write_to_buf_compress( - cached->dir_z + spooled->cached_dir_offset, + ptr + spooled->cached_dir_offset, bytes, conn, 0); } else { - connection_write_to_buf(cached->dir_z + spooled->cached_dir_offset, + connection_write_to_buf(ptr + spooled->cached_dir_offset, bytes, TO_CONN(conn)); } spooled->cached_dir_offset += bytes; - if (spooled->cached_dir_offset >= (off_t)cached->dir_z_len) { + if (spooled->cached_dir_offset >= (off_t)total_len) { return SRFS_DONE; } else { return SRFS_MORE; @@ -3608,6 +3659,7 @@ spooled_resource_lookup_body(const spooled_resource_t *spooled, return 0; } case DIR_SPOOL_NETWORKSTATUS: + case DIR_SPOOL_CONSENSUS_CACHE_ENTRY: default: /* LCOV_EXCL_START */ tor_assert_nonfatal_unreached(); diff --git a/src/or/dirserv.h b/src/or/dirserv.h index f707237..4b09f87 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -38,6 +38,7 @@ typedef enum dir_spool_source_t { DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, DIR_SPOOL_MICRODESC, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_CONSENSUS_CACHE_ENTRY, } dir_spool_source_t; #define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) @@ -74,8 +75,15 @@ typedef struct spooled_resource_t { */ struct cached_dir_t *cached_dir_ref; /** - * The current offset into cached_dir. Only used when spool_eagerly is - * false */ + * A different kind of large object that we might be spooling. Also + * reference-counted. Also only used when spool_eagerly is false. + */ + struct consensus_cache_entry_t *consensus_cache_entry; + const uint8_t *cce_body; + size_t cce_len; + /** + * The current offset into cached_dir or cce_body. Only used when + * spool_eagerly is false */ off_t cached_dir_offset; } spooled_resource_t; @@ -184,6 +192,8 @@ int dirserv_read_guardfraction_file(const char *fname, spooled_resource_t *spooled_resource_new(dir_spool_source_t source, const uint8_t *digest, size_t digestlen); +spooled_resource_t *spooled_resource_new_from_cache_entry( + struct consensus_cache_entry_t *entry); void spooled_resource_free(spooled_resource_t *spooled); void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn, time_t cutoff,

1 0

[tor/master] Update the consdiff directory code based on #22143 fixes
by nickm＠torproject.org 04 May '17

04 May '17

commit e1d31f2a2f5c7810c9c34ffa57e4b86ec8fd860b Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 4 08:49:02 2017 -0400 Update the consdiff directory code based on #22143 fixes These are mostly just identifier renames, except for one place in routerparse.c where we switch to using a correct hash. --- src/or/directory.c | 4 ++-- src/or/dirserv.c | 5 ++--- src/or/networkstatus.c | 2 +- src/or/or.h | 7 ++++--- src/or/routerparse.c | 8 +++----- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 1b37743..917dfee 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -517,7 +517,7 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, ims_delay = (v->fresh_until - v->valid_after)/2; } if_modified_since = v->valid_after + ims_delay; - memcpy(or_diff_from, v->digest_full_sha3, DIGEST256_LEN); + memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN); or_diff_from_is_set = 1; } } else { @@ -528,7 +528,7 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, * unparsed consensus, so we use the default. */ if (cd) { if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; - memcpy(or_diff_from, cd->digest_sha3_full, DIGEST256_LEN); + memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN); or_diff_from_is_set = 1; } } diff --git a/src/or/dirserv.c b/src/or/dirserv.c index af7ec97..7de72df 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1212,8 +1212,7 @@ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, const char *flavor_name, const common_digests_t *digests, - // XXXX rename after 22143 - const uint8_t *sha3_full_digest, + const uint8_t *sha3_as_signed, time_t published) { cached_dir_t *new_networkstatus; @@ -1223,7 +1222,7 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); - memcpy(&new_networkstatus->digest_sha3_full, sha3_full_digest, + memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, DIGEST256_LEN); old_networkstatus = strmap_set(cached_consensuses, flavor_name, new_networkstatus); diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 18537a3..1b21dd7 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -2002,7 +2002,7 @@ networkstatus_set_current_consensus(const char *consensus, dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, - c->digest_full_sha3, + c->digest_sha3_as_signed, c->valid_after); if (server_mode(get_options())) { consdiffmgr_add_consensus(consensus, c); diff --git a/src/or/or.h b/src/or/or.h index b69fcf1..423f66c 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1938,7 +1938,8 @@ typedef struct cached_dir_t { size_t dir_z_len; /**< Length of dir_z. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ - uint8_t digest_sha3_full[DIGEST256_LEN]; /**< sha3 digest (also ns only) */ + /** Sha3 digest (also ns only) */ + uint8_t digest_sha3_as_signed[DIGEST256_LEN]; int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -2639,9 +2640,9 @@ typedef struct networkstatus_t { /** Digests of this document, as signed. */ common_digests_t digests; - /** A SHA3-256 digest of the document, including signatures: used for + /** A SHA3-256 digest of the document, not including signatures: used for * consensus diffs */ - uint8_t digest_full_sha3[DIGEST256_LEN]; + uint8_t digest_sha3_as_signed[DIGEST256_LEN]; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 4def75a..fa79cf7 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -3384,7 +3384,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; common_digests_t ns_digests; - uint8_t sha3_full[DIGEST256_LEN]; + uint8_t sha3_as_signed[DIGEST256_LEN]; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; struct in_addr in; @@ -3398,10 +3398,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - // XXXX replace SHA3_full with as_signed digest once #22143 is merged. - // XXXX Merge #22143 before this! if (router_get_networkstatus_v3_hashes(s, &ns_digests) || - crypto_digest256((char *)sha3_full, s, strlen(s), DIGEST_SHA3_256)<0) { + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -3418,7 +3416,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns = tor_malloc_zero(sizeof(networkstatus_t)); memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); - memcpy(&ns->digest_full_sha3, sha3_full, sizeof(sha3_full)); + memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed)); tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); tor_assert(tok);

1 0

[tor/master] Generate X-Or-Diff-From-Consensus headers correctly.
by nickm＠torproject.org 04 May '17

04 May '17

commit 912b0641e9e9922ad2da50973579de38cb4be6ba Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 3 14:29:06 2017 -0400 Generate X-Or-Diff-From-Consensus headers correctly. --- src/or/directory.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 9c007e3..76fc121 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -130,6 +130,7 @@ static void directory_request_set_guard_state(directory_request_t *req, #define ALLOW_DIRECTORY_TIME_SKEW (30*60) #define X_ADDRESS_HEADER "X-Your-Address-Is: " +#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus" /** HTTP cache control: how long do we tell proxies they can cache each * kind of document we serve? */ @@ -480,15 +481,18 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, /** * Set the extra fields in req that are used when requesting a * consensus of type resource. + * + * Right now, these fields are if-modified-since and x-or-diff-from-consensus. */ static void dir_consensus_request_set_additional_headers(directory_request_t *req, const char *resource) { time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; + int or_diff_from_is_set = 0; int flav = FLAV_NS; - networkstatus_t *v; if (resource) flav = networkstatus_parse_flavor_name(resource); @@ -499,6 +503,7 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */ + networkstatus_t *v; v = networkstatus_get_latest_consensus_by_flavor(flav); if (v) { /* In networks with particularly short V3AuthVotingIntervals, @@ -510,6 +515,8 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, ims_delay = (v->fresh_until - v->valid_after)/2; } if_modified_since = v->valid_after + ims_delay; + memcpy(or_diff_from, v->digest_full_sha3, DIGEST256_LEN); + or_diff_from_is_set = 1; } } else { /* Otherwise it might be a consensus we don't parse, but which we @@ -517,11 +524,21 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, cached_dir_t *cd = dirserv_get_consensus(resource); /* We have no method of determining the voting interval from an * unparsed consensus, so we use the default. */ - if (cd) + if (cd) { if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; + memcpy(or_diff_from, cd->digest_sha3_full, DIGEST256_LEN); + or_diff_from_is_set = 1; + } } - directory_request_set_if_modified_since(req, if_modified_since); + if (if_modified_since > 0) + directory_request_set_if_modified_since(req, if_modified_since); + if (or_diff_from_is_set) { + char hex[HEX_DIGEST256_LEN + 1]; + base16_encode(hex, sizeof(hex), + (const char*)or_diff_from, sizeof(or_diff_from)); + directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex); + } } /** Start a connection to a random running directory server, using @@ -3537,7 +3554,7 @@ warn_consensus_is_too_old(networkstatus_t *v, const char *flavor, time_t now) static int parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) { - char *hdr = http_get_header(headers, "X-Or-Diff-From-Consensus"); + char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER); if (hdr == NULL) { return -1; }

1 0

[tor/master] Turn DEFAULT_IF_MODIFIED_SINCE_DELAY into a const
by nickm＠torproject.org 04 May '17

04 May '17

commit a8eccb6363ff02d4c59526a4daace96f71ff01c8 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 4 08:24:14 2017 -0400 Turn DEFAULT_IF_MODIFIED_SINCE_DELAY into a const --- src/or/directory.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index bf963f1..1b37743 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -493,14 +493,15 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, uint8_t or_diff_from[DIGEST256_LEN]; int or_diff_from_is_set = 0; + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ + const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180; + int flav = FLAV_NS; if (resource) flav = networkstatus_parse_flavor_name(resource); - /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus - * period of 1 hour. - */ -#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */

1 0

[tor/master] Store the sha3 of a networkstatus as part of the networkstatus_t
by nickm＠torproject.org 04 May '17

04 May '17

commit 112286338b61ed747572cfa67ac38e2052e7c807 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 3 10:17:37 2017 -0400 Store the sha3 of a networkstatus as part of the networkstatus_t Also store it in the cached_dir_t. --- src/or/dirserv.c | 4 ++++ src/or/dirserv.h | 1 + src/or/networkstatus.c | 1 + src/or/or.h | 4 ++++ src/or/routerparse.c | 7 ++++++- src/test/test_dir_handle_get.c | 4 ++++ 6 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index bdf40b6..af7ec97 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1212,6 +1212,8 @@ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, const char *flavor_name, const common_digests_t *digests, + // XXXX rename after 22143 + const uint8_t *sha3_full_digest, time_t published) { cached_dir_t *new_networkstatus; @@ -1221,6 +1223,8 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); + memcpy(&new_networkstatus->digest_sha3_full, sha3_full_digest, + DIGEST256_LEN); old_networkstatus = strmap_set(cached_consensuses, flavor_name, new_networkstatus); if (old_networkstatus) diff --git a/src/or/dirserv.h b/src/or/dirserv.h index 4b09f87..480174d 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -118,6 +118,7 @@ cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, const char *flavor_name, const common_digests_t *digests, + const uint8_t *sha3_as_signed, time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key, diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 188e753..8a5cdaf 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1981,6 +1981,7 @@ networkstatus_set_current_consensus(const char *consensus, dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, + c->digest_full_sha3, c->valid_after); if (server_mode(get_options())) { consdiffmgr_add_consensus(consensus, c); diff --git a/src/or/or.h b/src/or/or.h index e30a1d2..b69fcf1 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1938,6 +1938,7 @@ typedef struct cached_dir_t { size_t dir_z_len; /**< Length of dir_z. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ + uint8_t digest_sha3_full[DIGEST256_LEN]; /**< sha3 digest (also ns only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -2638,6 +2639,9 @@ typedef struct networkstatus_t { /** Digests of this document, as signed. */ common_digests_t digests; + /** A SHA3-256 digest of the document, including signatures: used for + * consensus diffs */ + uint8_t digest_full_sha3[DIGEST256_LEN]; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements diff --git a/src/or/routerparse.c b/src/or/routerparse.c index f39c332..4def75a 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -3384,6 +3384,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; common_digests_t ns_digests; + uint8_t sha3_full[DIGEST256_LEN]; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; struct in_addr in; @@ -3397,7 +3398,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { + // XXXX replace SHA3_full with as_signed digest once #22143 is merged. + // XXXX Merge #22143 before this! + if (router_get_networkstatus_v3_hashes(s, &ns_digests) || + crypto_digest256((char *)sha3_full, s, strlen(s), DIGEST_SHA3_256)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -3414,6 +3418,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns = tor_malloc_zero(sizeof(networkstatus_t)); memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + memcpy(&ns->digest_full_sha3, sha3_full, sizeof(sha3_full)); tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); tor_assert(tok); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 6e96391..c98938b 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1773,10 +1773,14 @@ status_vote_current_consensus_ns_test(char **header, char **body, size_t *body_len) { common_digests_t digests; + uint8_t sha3[DIGEST256_LEN]; dir_connection_t *conn = NULL; #define NETWORK_STATUS "some network status string" + memset(&digests, 0x60, sizeof(digests)); + memset(sha3, 0x06, sizeof(sha3)); dirserv_set_cached_consensus_networkstatus(NETWORK_STATUS, "ns", &digests, + sha3, time(NULL)); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);

1 0

[tor/master] Changes file for prop140
by nickm＠torproject.org 04 May '17

04 May '17

commit 8dd1028aa2788cbd3bbb960ed8eaee7cda7c22c7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 3 15:27:18 2017 -0400 Changes file for prop140 --- changes/prop140 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/changes/prop140 b/changes/prop140 new file mode 100644 index 0000000..04a30dc --- /dev/null +++ b/changes/prop140 @@ -0,0 +1,9 @@ + o Major features (directory protocol): + - Tor relays and authorities are now able to serve clients an + abbreviated version of the networkstatus consensus document, + containing only the changes since the an older consensus document that + the client holds. Clients now request these documents when + available. When this new protocol is in use by both client and server, + they will use far less bandwidth (up to 94% less) to keep an up-to-date + consensus. Implements proposal 140; closes ticket 13339. +

1 0

[tor/master] Remove excess indentation from previous commit.
by nickm＠torproject.org 04 May '17

04 May '17

commit 94ae99067fc8806f947f9ab1470a25fd41f7beba Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 3 14:21:01 2017 -0400 Remove excess indentation from previous commit. Review this with 'diff -b' to confirm --- src/or/directory.c | 60 +++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index f430ddf..9c007e3 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -486,41 +486,41 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, const char *resource) { time_t if_modified_since = 0; - { - int flav = FLAV_NS; - networkstatus_t *v; - if (resource) - flav = networkstatus_parse_flavor_name(resource); - /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus - * period of 1 hour. - */ + int flav = FLAV_NS; + networkstatus_t *v; + if (resource) + flav = networkstatus_parse_flavor_name(resource); + + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ #define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) - if (flav != -1) { - /* IF we have a parsed consensus of this type, we can do an - * if-modified-time based on it. */ - v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) { - /* In networks with particularly short V3AuthVotingIntervals, - * ask for the consensus if it's been modified since half the - * V3AuthVotingInterval of the most recent consensus. */ - time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; - if (v->fresh_until > v->valid_after - && ims_delay > (v->fresh_until - v->valid_after)/2) { - ims_delay = (v->fresh_until - v->valid_after)/2; - } - if_modified_since = v->valid_after + ims_delay; + if (flav != -1) { + /* IF we have a parsed consensus of this type, we can do an + * if-modified-time based on it. */ + v = networkstatus_get_latest_consensus_by_flavor(flav); + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; } - } else { - /* Otherwise it might be a consensus we don't parse, but which we - * do cache. Look at the cached copy, perhaps. */ - cached_dir_t *cd = dirserv_get_consensus(resource); - /* We have no method of determining the voting interval from an - * unparsed consensus, so we use the default. */ - if (cd) - if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; + if_modified_since = v->valid_after + ims_delay; } + } else { + /* Otherwise it might be a consensus we don't parse, but which we + * do cache. Look at the cached copy, perhaps. */ + cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ + if (cd) + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; } + directory_request_set_if_modified_since(req, if_modified_since); }

1 0

[tor/master] New function to add additional headers to a directory request
by nickm＠torproject.org 04 May '17

04 May '17

commit 57710c15875cfe9bdb1f1b2b6f6fa1346ab46c11 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 3 14:04:04 2017 -0400 New function to add additional headers to a directory request --- src/or/directory.c | 24 ++++++++++++++++++++++++ src/or/directory.h | 4 +++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/or/directory.c b/src/or/directory.c index cf272f7..8f934bd 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -989,6 +989,9 @@ struct directory_request_t { time_t if_modified_since; /** Hidden-service-specific information */ const rend_data_t *rend_query; + /** Extra headers to append to the request */ + config_line_t *additional_headers; + /** */ /** Used internally to directory.c: gets informed when the attempt to * connect to the directory succeeds or fails, if that attempt bears on the * directory's usability as a directory guard. */ @@ -1087,6 +1090,7 @@ directory_request_free(directory_request_t *req) { if (req == NULL) return; + config_free_lines(req->additional_headers); tor_free(req); } /** @@ -1187,6 +1191,18 @@ directory_request_set_if_modified_since(directory_request_t *req, { req->if_modified_since = if_modified_since; } + +/** Include a header of name key with content val in the + * request. Neither may include newlines or other odd characters. Their + * ordering is not currently guaranteed. + */ +void +directory_request_add_header(directory_request_t *req, + const char *key, + const char *val) +{ + config_line_prepend(&req->additional_headers, key, val); +} /** * Set an object containing HS data to be associated with this request. Note * that only an alias to query is stored, so the query object @@ -1673,6 +1689,14 @@ directory_send_command(dir_connection_t *conn, proxystring[0] = 0; } + /* Add additional headers, if any */ + { + config_line_t *h; + for (h = req->additional_headers; h; h = h->next) { + smartlist_add_asprintf(headers, "%s: %s\r\n", h->key, h->value); + } + } + switch (purpose) { case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ diff --git a/src/or/directory.h b/src/or/directory.h index 8473d39..125333d 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -72,7 +72,9 @@ void directory_request_set_rend_query(directory_request_t *req, void directory_request_set_routerstatus(directory_request_t *req, const routerstatus_t *rs); - +void directory_request_add_header(directory_request_t *req, + const char *key, + const char *val); MOCK_DECL(void, directory_initiate_request, (directory_request_t *request)); int parse_http_response(const char *headers, int *code, time_t *date,

1 0