May 2017 - tor-commits - lists.torproject.org

[tor/master] consdiffmgr: Reload latest consensus entries on start.
by nickm＠torproject.org 15 May '17

15 May '17

commit 30dfb36148c05b8757f3ac263a8473b7aeba0ac5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 11:58:19 2017 -0400 consdiffmgr: Reload latest consensus entries on start. --- src/or/consdiffmgr.c | 67 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 0feb84a..ad63ce9 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -516,6 +516,20 @@ sort_and_find_most_recent(smartlist_t *lst) } } +/** Return i such that compress_consensus_with[i] == method. Return + * -1 if no such i exists. */ +static int +consensus_compression_method_pos(compress_method_t method) +{ + unsigned i; + for (i = 0; i < n_consensus_compression_methods(); ++i) { + if (compress_consensus_with[i] == method) { + return i; + } + } + return -1; +} + /** * If we know a consensus with the flavor flavor compressed with * method, set *entry_out to that value. Return values are as @@ -526,17 +540,9 @@ consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, compress_method_t method) { - int pos=-1; - unsigned i; tor_assert(flavor < N_CONSENSUS_FLAVORS); - // Find the index of method withing compress_consensus_with - for (i = 0; i < n_consensus_compression_methods(); ++i) { - if (compress_consensus_with[i] == method) { - pos = i; - break; - } - } + int pos = consensus_compression_method_pos(method); if (pos < 0) { // We don't compress consensuses with this method. return CONSDIFF_NOT_FOUND; @@ -933,6 +939,48 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) } /** + * Scan the cache for the latest consensuses and add their handles to + * latest_consensus + */ +static void +consdiffmgr_consensus_load(void) +{ + smartlist_t *matches = smartlist_new(); + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + smartlist_clear(matches); + consensus_cache_find_all(matches, cdm_cache_get(), + LABEL_FLAVOR, flavname); + consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); + if (! most_recent) + continue; // no consensuses. + const char *most_recent_sha3 = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(most_recent_sha3 == NULL)) + continue; // LCOV_EXCL_LINE + consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED, + most_recent_sha3); + + // Everything that remains matches the most recent consensus of this + // flavor. + SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) { + const char *lv_compression = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + compress_method_t method = + compression_method_get_by_name(lv_compression); + int pos = consensus_compression_method_pos(method); + if (pos < 0) + continue; + consensus_cache_entry_handle_free(latest_consensus[flav][pos]); + latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent); + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(matches); +} + +/** * Scan the cache for diffs, and add them to the hashtable. */ static void @@ -989,6 +1037,7 @@ consdiffmgr_rescan(void) if (cdm_cache_loaded == 0) { consdiffmgr_diffs_load(); + consdiffmgr_consensus_load(); cdm_cache_loaded = 1; }

1 0

[tor/master] Fix indentation when using the ternary operator in handle_get_status_vote().
by nickm＠torproject.org 15 May '17

15 May '17

commit 59d17ca2bb6ab243bd58b29d26ccf6dd10741abe Author: Alexander Færøy <ahf(a)torproject.org> Date: Fri May 12 13:17:43 2017 +0200 Fix indentation when using the ternary operator in handle_get_status_vote(). See: https://bugs.torproject.org/21667 --- src/or/directory.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index ed4f961..7965ffd 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3926,9 +3926,8 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compress_method != NO_METHOD - ? d->dir_compressed_len - : d->dir_len); + body_len += compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); @@ -3960,10 +3959,10 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compress_method != NO_METHOD - ? d->dir_compressed : d->dir, - compress_method != NO_METHOD - ? d->dir_compressed_len : d->dir_len, + connection_write_to_buf(compress_method != NO_METHOD ? + d->dir_compressed : d->dir, + compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done:

1 0

[tor/master] Cleanup logic: only retain zlib-compressed consensuses
by nickm＠torproject.org 15 May '17

15 May '17

commit 7b0dcf5c4a081ef892d79a9a3e36a6d4d1ec31f9 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 11:32:35 2017 -0400 Cleanup logic: only retain zlib-compressed consensuses Now that we're making a bunch of these with consdiffmgr, we should throw out all but one when we get a newer consensus. --- src/or/consdiffmgr.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 2f0a653..4fccb43 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -114,6 +114,12 @@ n_consensus_compression_methods(void) return ARRAY_LENGTH(compress_consensus_with); } +/** For which compression method do we retain old consensuses? There's no + * need to keep all of them, since we won't be serving them. We'll + * go with ZLIB_METHOD because it's pretty fast and everyone has it. + */ +#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -645,6 +651,43 @@ consdiffmgr_cleanup(void) smartlist_clear(diffs); } + // 3. Delete all consensuses except the most recent that are compressed with + // an un-preferred method. + for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavname = networkstatus_get_flavor_name(flav); + /* Determine the most recent consensus of this flavor */ + consensus_cache_find_all(consensuses, cdm_cache_get(), + LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname); + consensus_cache_entry_t *most_recent = + sort_and_find_most_recent(consensuses); + if (most_recent == NULL) + continue; + const char *most_recent_sha3_uncompressed = + consensus_cache_entry_get_value(most_recent, + LABEL_SHA3_DIGEST_UNCOMPRESSED); + const char *retain_methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + + + if (BUG(most_recent_sha3_uncompressed == NULL)) + continue; + SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) { + const char *lv_sha3_uncompressed = + consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED); + if (BUG(! lv_sha3_uncompressed)) + continue; + if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed)) + continue; // This _is_ the most recent. + const char *lv_methodname = + consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE); + if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) { + consensus_cache_entry_mark_for_removal(ent); + ++n_to_delete; + } + } SMARTLIST_FOREACH_END(ent); + } + smartlist_free(objects); smartlist_free(consensuses); smartlist_free(diffs); @@ -748,10 +791,14 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) // 1. find the most recent consensus, and the ones that we might want // to diff to it. + const char *methodname = compression_method_get_name( + RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD); + matches = smartlist_new(); consensus_cache_find_all(matches, cdm_cache_get(), LABEL_FLAVOR, flavname); consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname); consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches); if (!most_recent) { log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "

1 0

[tor/master] consdiffmgr: compress incoming consensuses in the background
by nickm＠torproject.org 15 May '17

15 May '17

commit 151cd121a2733506c69162330b210c3c45044dfa Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 11:17:59 2017 -0400 consdiffmgr: compress incoming consensuses in the background Also, compress them in several ways. This breaks the unit tests; subsequent commits will make them pass again. --- src/or/consdiffmgr.c | 233 +++++++++++++++++++++++++++++++++++++-------------- src/or/consdiffmgr.h | 1 + 2 files changed, 173 insertions(+), 61 deletions(-) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index b108c80..2f0a653 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -96,6 +96,24 @@ n_diff_compression_methods(void) return ARRAY_LENGTH(compress_diffs_with); } +/** Which methods do we use for precompressing consensuses? */ +static const compress_method_t compress_consensus_with[] = { + ZLIB_METHOD, +#ifdef HAVE_LZMA + LZMA_METHOD, +#endif +#ifdef HAVE_ZSTD + ZSTD_METHOD, +#endif +}; + +/** How many different methods will we try to use for diff compression? */ +static unsigned +n_consensus_compression_methods(void) +{ + return ARRAY_LENGTH(compress_consensus_with); +} + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -135,13 +153,13 @@ static consdiff_cfg_t consdiff_cfg = { }; static int consdiffmgr_ensure_space_for_files(int n); +static int consensus_queue_compression_work(const char *consensus, + consensus_flavor_t flavor, + time_t valid_after); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); static void consdiffmgr_set_cache_flags(void); -/* Just gzip consensuses for now. */ -#define COMPRESS_CONSENSUS_WITH GZIP_METHOD - /* ===== * Hashtable setup * ===== */ @@ -410,11 +428,6 @@ cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after) consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); consensus_cache_entry_t *result = NULL; - if (smartlist_len(matches) > 1) { - log_warn(LD_BUG, "How odd; there appear to be two matching consensuses " - "with flavor %s published at %s.", - flavname, formatted_time); - } if (smartlist_len(matches)) { result = smartlist_get(matches, 0); } @@ -458,59 +471,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - consdiffmgr_ensure_space_for_files(1); - - { - size_t bodylen = strlen(consensus); - config_line_t *labels = NULL; - char formatted_time[ISO_TIME_LEN+1]; - format_iso_time_nospace(formatted_time, valid_after); - const char *flavname = networkstatus_get_flavor_name(flavor); - - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, - (const uint8_t *)consensus, bodylen); - { - const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { - start = consensus; - end = consensus+bodylen; - } - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, - (const uint8_t *)start, - end - start); - } - - char *body_compressed = NULL; - size_t size_compressed = 0; - if (tor_compress(&body_compressed, &size_compressed, - consensus, bodylen, COMPRESS_CONSENSUS_WITH) < 0) { - config_free_lines(labels); - return -1; - } - cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST, - (const uint8_t *)body_compressed, size_compressed); - config_line_prepend(&labels, LABEL_COMPRESSION_TYPE, - compression_method_get_name(COMPRESS_CONSENSUS_WITH)); - config_line_prepend(&labels, LABEL_FLAVOR, flavname); - config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); - config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); - - entry = consensus_cache_add(cdm_cache_get(), - labels, - (const uint8_t *)body_compressed, - size_compressed); - tor_free(body_compressed); - config_free_lines(labels); - } - - if (entry) { - consensus_cache_entry_mark_for_aggressive_release(entry); - consensus_cache_entry_decref(entry); - } - - cdm_cache_dirty = 1; - return entry ? 0 : -1; + return consensus_queue_compression_work(consensus, flavor, valid_after); } /** @@ -835,6 +796,10 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor) if (strmap_get(have_diff_from, va) != NULL) continue; /* we already have this one. */ smartlist_add(compute_diffs_from, ent); + /* Since we are not going to serve this as the most recent consensus + * any more, we should stop keeping it mmap'd when it's not in use. + */ + consensus_cache_entry_mark_for_aggressive_release(ent); } SMARTLIST_FOREACH_END(ent); log_info(LD_DIRSERV, @@ -1147,6 +1112,8 @@ store_multiple(consensus_cache_entry_handle_t **handles_out, labels, body_out, bodylen_out); + if (BUG(ent == NULL)) + continue; status = CDM_DIFF_PRESENT; handles_out[i] = consensus_cache_entry_handle_new(ent); @@ -1464,3 +1431,147 @@ consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, return -1; } +/** + * Holds requests and replies for consensus_compress_workers. + */ +typedef struct consensus_compress_worker_job_t { + char *consensus; + size_t consensus_len; + consensus_flavor_t flavor; + time_t valid_after; + compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)]; +} consensus_compress_worker_job_t; + +/** + * Free all resources held in job + */ +static void +consensus_compress_worker_job_free(consensus_compress_worker_job_t *job) +{ + if (!job) + return; + tor_free(job->consensus); + unsigned u; + for (u = 0; u < n_consensus_compression_methods(); ++u) { + config_free_lines(job->out[u].labels); + tor_free(job->out[u].body); + } + tor_free(job); +} +/** + * Worker function. This function runs inside a worker thread and receives + * a consensus_compress_worker_job_t as its input. + */ +static workqueue_reply_t +consensus_compress_worker_threadfn(void *state_, void *work_) +{ + (void)state_; + consensus_compress_worker_job_t *job = work_; + consensus_flavor_t flavor = job->flavor; + const char *consensus = job->consensus; + size_t bodylen = job->consensus_len; + time_t valid_after = job->valid_after; + + config_line_t *labels = NULL; + char formatted_time[ISO_TIME_LEN+1]; + format_iso_time_nospace(formatted_time, valid_after); + + const char *flavname = networkstatus_get_flavor_name(flavor); + + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, + (const uint8_t *)consensus, bodylen); + { + const char *start, *end; + if (router_get_networkstatus_v3_signed_boundaries(consensus, + &start, &end) < 0) { + start = consensus; + end = consensus+bodylen; + } + cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED, + (const uint8_t *)start, + end - start); + } + config_line_prepend(&labels, LABEL_FLAVOR, flavname); + config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time); + config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS); + + compress_multiple(job->out, + n_consensus_compression_methods(), + compress_consensus_with, + (const uint8_t*)consensus, bodylen, labels); + config_free_lines(labels); + return WQ_RPL_REPLY; +} + +/** + * Worker function: This function runs in the main thread, and receives + * a consensus_diff_compress_job_t that the worker thread has already + * processed. + */ +static void +consensus_compress_worker_replyfn(void *work_) +{ + consensus_compress_worker_job_t *job = work_; + + consensus_cache_entry_handle_t *handles[ + ARRAY_LENGTH(compress_consensus_with)]; + memset(handles, 0, sizeof(handles)); + + store_multiple(handles, + n_consensus_compression_methods(), + compress_consensus_with, + job->out, + "consensus"); + cdm_cache_dirty = 1; + + consensus_compress_worker_job_free(job); +} + +/** + * If true, we compress in worker threads. + */ +static int background_compression = 0; + +/** + * Queue a job to compress consensus and store its compressed + * text in the cache. + */ +static int +consensus_queue_compression_work(const char *consensus, + consensus_flavor_t flavor, + time_t valid_after) +{ + consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); + job->consensus = tor_strdup(consensus); + job->consensus_len = strlen(consensus); + job->flavor = flavor; + job->valid_after = valid_after; + + if (background_compression) { + workqueue_entry_t *work; + work = cpuworker_queue_work(consensus_compress_worker_threadfn, + consensus_compress_worker_replyfn, + job); + if (!work) { + consensus_compress_worker_job_free(job); + return -1; + } + + return 0; + } else { + consensus_compress_worker_threadfn(NULL, job); + consensus_compress_worker_replyfn(job); + return 0; + } +} + +/** + * Tell the consdiffmgr backend to compress consensuses in worker threads. + */ +void +consdiffmgr_enable_background_compression(void) +{ + // This isn't the default behavior because it would break unit tests. + background_compression = 1; +} + diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index 048dae4..cbff599 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -32,6 +32,7 @@ consdiff_status_t consdiffmgr_find_diff_from( compress_method_t method); void consdiffmgr_rescan(void); int consdiffmgr_cleanup(void); +void consdiffmgr_enable_background_compression(void); void consdiffmgr_configure(const consdiff_cfg_t *cfg); struct sandbox_cfg_elem; int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);

1 0

[tor/master] consdiffmgr: expose cached consensuses
by nickm＠torproject.org 15 May '17

15 May '17

commit 8100305e71fb1072831de1666cbb39196d1ae162 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 11:47:45 2017 -0400 consdiffmgr: expose cached consensuses --- src/or/consdiffmgr.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/or/consdiffmgr.h | 5 +++++ 2 files changed, 64 insertions(+) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 4fccb43..0feb84a 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -120,6 +120,12 @@ n_consensus_compression_methods(void) */ #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD +/** Handles pointing to the latest consensus entries as compressed and + * stored. */ +static consensus_cache_entry_handle_t * + latest_consensus[N_CONSENSUS_FLAVORS] + [ARRAY_LENGTH(compress_consensus_with)]; + /** Hashtable node used to remember the current status of the diff * from a given sha3 digest to the current consensus. */ typedef struct cdm_diff_t { @@ -511,6 +517,41 @@ sort_and_find_most_recent(smartlist_t *lst) } /** + * If we know a consensus with the flavor flavor compressed with + * method, set *entry_out to that value. Return values are as + * for consdiffmgr_find_diff_from(). + */ +consdiff_status_t +consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method) +{ + int pos=-1; + unsigned i; + tor_assert(flavor < N_CONSENSUS_FLAVORS); + + // Find the index of method withing compress_consensus_with + for (i = 0; i < n_consensus_compression_methods(); ++i) { + if (compress_consensus_with[i] == method) { + pos = i; + break; + } + } + if (pos < 0) { + // We don't compress consensuses with this method. + return CONSDIFF_NOT_FOUND; + } + consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos]; + if (!handle) + return CONSDIFF_NOT_FOUND; + *entry_out = consensus_cache_entry_handle_get(handle); + if (entry_out) + return CONSDIFF_AVAILABLE; + else + return CONSDIFF_NOT_FOUND; +} + +/** * Look up consensus_cache_entry_t for the consensus of type flavor, * from the source consensus with the specified digest (which must be SHA3). * @@ -1063,6 +1104,14 @@ consdiffmgr_free_all(void) next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff); cdm_diff_free(this); } + int i; + unsigned j; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + for (j = 0; j < n_consensus_compression_methods(); ++j) { + consensus_cache_entry_handle_free(latest_consensus[i][j]); + } + } + memset(latest_consensus, 0, sizeof(latest_consensus)); consensus_cache_free(cons_diff_cache); cons_diff_cache = NULL; } @@ -1571,6 +1620,16 @@ consensus_compress_worker_replyfn(void *work_) "consensus"); cdm_cache_dirty = 1; + unsigned u; + consensus_flavor_t f = job->flavor; + tor_assert(f < N_CONSENSUS_FLAVORS); + for (u = 0; u < ARRAY_LENGTH(handles); ++u) { + if (handles[u] == NULL) + continue; + consensus_cache_entry_handle_free(latest_consensus[f][u]); + latest_consensus[f][u] = handles[u]; + } + consensus_compress_worker_job_free(job); } diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index cbff599..5f7e944 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -23,6 +23,11 @@ struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, const networkstatus_t *as_parsed); +consdiff_status_t consdiffmgr_find_consensus( + struct consensus_cache_entry_t **entry_out, + consensus_flavor_t flavor, + compress_method_t method); + consdiff_status_t consdiffmgr_find_diff_from( struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor,

1 0

[tor/master] Test fix: expect old consensuses to be deleted if not deflate-compressed
by nickm＠torproject.org 15 May '17

15 May '17

commit db370bb8a8fd65a902d3abd996ace019db4c296c Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 12:10:45 2017 -0400 Test fix: expect old consensuses to be deleted if not deflate-compressed --- src/or/consdiffmgr.c | 2 +- src/or/consdiffmgr.h | 1 + src/test/test_consdiffmgr.c | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index ad63ce9..3e1edf0 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -108,7 +108,7 @@ static const compress_method_t compress_consensus_with[] = { }; /** How many different methods will we try to use for diff compression? */ -static unsigned +STATIC unsigned n_consensus_compression_methods(void) { return ARRAY_LENGTH(compress_consensus_with); diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h index 5f7e944..abe1ea9 100644 --- a/src/or/consdiffmgr.h +++ b/src/or/consdiffmgr.h @@ -46,6 +46,7 @@ int consdiffmgr_validate(void); #ifdef CONSDIFFMGR_PRIVATE STATIC unsigned n_diff_compression_methods(void); +STATIC unsigned n_consensus_compression_methods(void); STATIC consensus_cache_t *cdm_cache_get(void); STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( consensus_flavor_t flavor, time_t valid_after); diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 9d0c5b5..2e5dd59 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -759,9 +759,11 @@ test_consdiffmgr_cleanup_old_diffs(void *arg) consensus_cache_entry_incref(hold_ent); // incref, so it is preserved. /* Now add an even-more-recent consensus; this should make all previous - * diffs deletable */ + * diffs deletable, and make delete */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); - tt_int_op(2 * n_diff_compression_methods(), OP_EQ, consdiffmgr_cleanup()); + tt_int_op(2 * n_diff_compression_methods() + + (n_consensus_compression_methods() - 1) , OP_EQ, + consdiffmgr_cleanup()); tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ, lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));

1 0

[tor/master] Handle x-zstd and x-tor-lzma in parse_http_response().
by nickm＠torproject.org 15 May '17

15 May '17

commit fbef257c43b90bd5d80831808e162c57ce876283 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon May 8 13:52:40 2017 +0200 Handle x-zstd and x-tor-lzma in parse_http_response(). See: https://bugs.torproject.org/21667 --- src/or/directory.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/or/directory.c b/src/or/directory.c index 83a2e6c..c7c3d38 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2056,6 +2056,10 @@ parse_http_response(const char *headers, int *code, time_t *date, *compression = ZLIB_METHOD; } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { *compression = GZIP_METHOD; + } else if (!strcmp(enc, "x-zstd")) { + *compression = ZSTD_METHOD; + } else if (!strcmp(enc, "x-tor-lzma")) { + *compression = LZMA_METHOD; } else { log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", escaped(enc));

1 0

[tor/master] actually enable background compresion for consensuses
by nickm＠torproject.org 15 May '17

15 May '17

commit 077d3085ec30411daa6b3985ba7b10fd8ef861fa Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 12:15:24 2017 -0400 actually enable background compresion for consensuses --- src/or/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/or/main.c b/src/or/main.c index f1a8cfb..9f0c29c 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2482,6 +2482,7 @@ do_main_loop(void) /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } + consdiffmgr_enable_background_compression(); /* Setup shared random protocol subsystem. */ if (authdir_mode_v3(get_options())) {

1 0

[tor/master] Copy valid/fresh-until and signatories values into diffs.
by nickm＠torproject.org 15 May '17

15 May '17

commit 7591518d163de1367cd21e1fc610960ab83993e0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun May 14 20:06:46 2017 -0400 Copy valid/fresh-until and signatories values into diffs. --- src/or/consdiffmgr.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index fef5f68..910f842 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -1348,6 +1348,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) const char *lv_to_valid_after = consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER); + const char *lv_to_fresh_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL); + const char *lv_to_valid_until = + consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL); + const char *lv_to_signatories = + consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES); const char *lv_from_valid_after = consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER); const char *lv_from_digest = @@ -1417,6 +1423,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) job->out[0].bodylen = difflen; config_line_t *common_labels = NULL; + if (lv_to_valid_until) + config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until); + if (lv_to_fresh_until) + config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until); + if (lv_to_signatories) + config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories); cdm_labels_prepend_sha3(&common_labels, LABEL_SHA3_DIGEST_UNCOMPRESSED, job->out[0].body,

1 0

[tor/master] Add stub functions for querying metadata about the consensus.
by nickm＠torproject.org 15 May '17

15 May '17

commit 363f4b8db5f9f642f848f2e50ae0c6bb0f1ff25e Author: Alexander Færøy <ahf(a)torproject.org> Date: Sat May 13 01:03:55 2017 +0200 Add stub functions for querying metadata about the consensus. --- src/or/conscache.c | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/or/conscache.h | 8 ++++++++ 2 files changed, 67 insertions(+) diff --git a/src/or/conscache.c b/src/or/conscache.c index 5ffa129..fd46cbe 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -374,6 +374,65 @@ consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, return 0; } +/** Read the lifetime of cached object ent into lifetime. */ +int consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, + long *lifetime) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + tor_assert(lifetime); + + // FIXME(ahf): Fill out. + *lifetime = 0; + + return 0; +} + +/** Return non-zero if the cache object found in ent is + * reasonably live, otherwise return 0. Use now to pass the + * timestamp used for comparison. */ +int consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, + time_t now) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + // FIXME(ahf): Fill out. + (void)now; + + return 1; +} + +/** Read the set of voters from the cached object ent into out. */ +int consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, + smartlist_t *out) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + // FIXME(ahf): Fill out. + (void)out; + + return 0; +} + +/** Read the valid until timestamp from the cached object ent + * into out. */ +int consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, + time_t *out) +{ + if (BUG(ent->magic != CCE_MAGIC)) + return -1; // LCOV_EXCL_LINE + + tor_assert(out); + + // FIXME(ahf): Fill out. + *out = time(NULL); + + return 0; +} + /** * Unmap every mmap'd element of cache that has been unused * since cutoff. diff --git a/src/or/conscache.h b/src/or/conscache.h index aef5420..fcc3f3e 100644 --- a/src/or/conscache.h +++ b/src/or/conscache.h @@ -52,6 +52,14 @@ void consensus_cache_entry_mark_for_aggressive_release( int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent, const uint8_t **body_out, size_t *sz_out); +int consensus_cache_entry_get_lifetime(const consensus_cache_entry_t *ent, + long *lifetime); +int consensus_cache_entry_is_reasonably_live(const consensus_cache_entry_t *ent, + time_t now); +int consensus_cache_entry_get_voters(const consensus_cache_entry_t *ent, + smartlist_t *out); +int consensus_cache_entry_valid_until(const consensus_cache_entry_t *ent, + time_t *out); #ifdef TOR_UNIT_TESTS int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);

1 0