commit 7fc64f02a3057405f9e75d70848afd2e9b95da05 Author: George Kadianakis <desnacked@riseup.net> Date: Mon Nov 6 14:48:22 2017 +0200 Introduce cache for outdated microdesc dirservers. We gonna use this cache to avoid dirservers without outdated md info. --- src/or/directory.c | 36 +++++++++++------- src/or/directory.h | 7 +++- src/or/microdesc.c | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ src/or/microdesc.h | 6 ++- src/or/networkstatus.c | 3 ++ 5 files changed, 138 insertions(+), 15 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 9494cf02b..129309ae4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -117,7 +117,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_extrainfo, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, - int status_code); + int status_code, + const char *dir_id); static int client_likes_consensus(const struct consensus_cache_entry_t *ent, const char *want_url); @@ -2178,8 +2179,6 @@ static int handle_response_fetch_detached_signatures(dir_connection_t *, const response_handler_args_t *); static int handle_response_fetch_desc(dir_connection_t *, const response_handler_args_t *); -static int handle_response_fetch_microdesc(dir_connection_t *, - const response_handler_args_t *); static int handle_response_upload_dir(dir_connection_t *, const response_handler_args_t *); static int handle_response_upload_vote(dir_connection_t *, @@ -2839,7 +2838,7 @@ handle_response_fetch_desc(dir_connection_t *conn, * Handler function: processes a response to a request for a group of * microdescriptors **/ -static int +STATIC int handle_response_fetch_microdesc(dir_connection_t *conn, const response_handler_args_t *args) { @@ -2856,6 +2855,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn, conn->base_.port); tor_assert(conn->requested_resource && !strcmpstart(conn->requested_resource, "d/")); + tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN)); which = smartlist_new(); dir_split_resource_into_fingerprints(conn->requested_resource+2, which, NULL, @@ -2866,7 +2866,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn, "soon.", status_code, escaped(reason), conn->base_.address, (int)conn->base_.port, conn->requested_resource); - dir_microdesc_download_failed(which, status_code); + dir_microdesc_download_failed(which, status_code, conn->identity_digest); SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); smartlist_free(which); return 0; @@ -2878,7 +2878,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn, now, which); if (smartlist_len(which)) { /* Mark remaining ones as failed. */ - dir_microdesc_download_failed(which, status_code); + dir_microdesc_download_failed(which, status_code, conn->identity_digest); } if (mds && smartlist_len(mds)) { control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, @@ -5546,13 +5546,14 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */ } -/** Called when a connection to download microdescriptors has failed in whole - * or in part. <b>failed</b> is a list of every microdesc digest we didn't - * get. <b>status_code</b> is the http status code we received. Reschedule the - * microdesc downloads as appropriate. */ +/** Called when a connection to download microdescriptors from relay with + * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list + * of every microdesc digest we didn't get. <b>status_code</b> is the http + * status code we received. Reschedule the microdesc downloads as + * appropriate. */ static void dir_microdesc_download_failed(smartlist_t *failed, - int status_code) + int status_code, const char *dir_id) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC); @@ -5563,17 +5564,26 @@ dir_microdesc_download_failed(smartlist_t *failed, if (! consensus) return; + + /* We failed to fetch a microdescriptor from 'dir_id', note it down + * so that we don't try the same relay next time... */ + microdesc_note_outdated_dirserver(dir_id); + SMARTLIST_FOREACH_BEGIN(failed, const char *, d) { rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d); if (!rs) continue; dls = &rs->dl_status; if (dls->n_download_failures >= - get_options()->TestingMicrodescMaxDownloadTries) + get_options()->TestingMicrodescMaxDownloadTries) { continue; - { + } + + { /* Increment the failure count for this md fetch */ char buf[BASE64_DIGEST256_LEN+1]; digest256_to_base64(buf, d); + log_info(LD_DIR, "Failed to download md %s from %s", + buf, hex_str(dir_id, DIGEST_LEN)); download_status_increment_failure(dls, status_code, buf, server, now); } diff --git a/src/or/directory.h b/src/or/directory.h index 14d5ae9ef..571c30a0f 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -166,7 +166,12 @@ STATIC char *accept_encoding_header(void); STATIC int allowed_anonymous_connection_compression_method(compress_method_t); STATIC void warn_disallowed_anonymous_compression_method(compress_method_t); -#endif +struct response_handler_args_t; + +STATIC int handle_response_fetch_microdesc(dir_connection_t *conn, + const struct response_handler_args_t *args); + +#endif /* defined(DIRECTORY_PRIVATE) */ #ifdef TOR_UNIT_TESTS /* Used only by test_dir.c */ diff --git a/src/or/microdesc.c b/src/or/microdesc.c index a4e6b409c..32242d005 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -74,6 +74,102 @@ HT_GENERATE2(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_, 0.6, tor_reallocarray_, tor_free_) +/************************* md fetch fail cache *****************************/ + +/* If we end up with too many outdated dirservers, something probably went + * wrong so clean up the list. */ +#define TOO_MANY_OUTDATED_DIRSERVERS 30 + +/** List of dirservers with outdated microdesc information. The smartlist is + * filled with the hex digests of outdated dirservers. */ +static smartlist_t *outdated_dirserver_list = NULL; + +/** Note that we failed to fetch a microdescriptor from the relay with + * <b>relay_digest</b> (of size DIGEST_LEN). */ +void +microdesc_note_outdated_dirserver(const char *relay_digest) +{ + char relay_hexdigest[HEX_DIGEST_LEN+1]; + + /* Don't register outdated dirservers if we don't have a live consensus, + * since we might be trying to fetch microdescriptors that are not even + * currently active. */ + if (!networkstatus_get_live_consensus(approx_time())) { + return; + } + + if (!outdated_dirserver_list) { + outdated_dirserver_list = smartlist_new(); + } + + tor_assert(outdated_dirserver_list); + + /* If the list grows too big, clean it up */ + if (BUG(smartlist_len(outdated_dirserver_list) > + TOO_MANY_OUTDATED_DIRSERVERS)) { + microdesc_reset_outdated_dirservers_list(); + } + + /* Turn the binary relay digest to a hex since smartlists have better support + * for strings than digests. */ + base16_encode(relay_hexdigest,sizeof(relay_hexdigest), + relay_digest, DIGEST_LEN); + + /* Make sure we don't add a dirauth as an outdated dirserver */ + if (router_get_trusteddirserver_by_digest(relay_digest)) { + log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest); + return; + } + + /* Don't double-add outdated dirservers */ + if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) { + return; + } + + /* Add it to the list of outdated dirservers */ + smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest); + + log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest); +} + +/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an + * outdated dirserver */ +int +microdesc_relay_is_outdated_dirserver(const char *relay_digest) +{ + char relay_hexdigest[HEX_DIGEST_LEN+1]; + + if (!outdated_dirserver_list) { + return 0; + } + + /* Convert identity digest to hex digest */ + base16_encode(relay_hexdigest, sizeof(relay_hexdigest), + relay_digest, DIGEST_LEN); + + /* Last time we tried to fetch microdescs, was this directory mirror missing + * any mds we asked for? */ + if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) { + return 1; + } + + return 0; +} + +/** Reset the list of outdated dirservers. */ +void +microdesc_reset_outdated_dirservers_list(void) +{ + if (!outdated_dirserver_list) { + return; + } + + SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp)); + smartlist_clear(outdated_dirserver_list); +} + +/****************************************************************************/ + /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. * On success, return the total number of bytes written, and set * *<b>annotation_len_out</b> to the number of bytes written as @@ -789,6 +885,11 @@ microdesc_free_all(void) tor_free(the_microdesc_cache->journal_fname); tor_free(the_microdesc_cache); } + + if (outdated_dirserver_list) { + SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp)); + smartlist_free(outdated_dirserver_list); + } } /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 943873066..1be12156a 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -50,5 +50,9 @@ int we_fetch_microdescriptors(const or_options_t *options); int we_fetch_router_descriptors(const or_options_t *options); int we_use_microdescriptors_for_circuits(const or_options_t *options); -#endif +void microdesc_note_outdated_dirserver(const char *relay_digest); +int microdesc_relay_is_outdated_dirserver(const char *relay_digest); +void microdesc_reset_outdated_dirservers_list(void); + +#endif /* !defined(TOR_MICRODESC_H) */ diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 997280de5..36e62020e 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -2041,6 +2041,9 @@ networkstatus_set_current_consensus(const char *consensus, "CLOCK_SKEW MIN_SKEW=%ld SOURCE=CONSENSUS", delta); } + /* We got a new consesus. Reset our md fetch fail cache */ + microdesc_reset_outdated_dirservers_list(); + router_dir_info_changed(); result = 0;