[tor-commits] [tor/master] Split routerlist.c into 4 separate modules

commit 08e3b88f0774fedb41a6b1c170a710dd12a7cb90 Author: Nick Mathewson nickm@torproject.org Date: Wed Sep 19 14:10:15 2018 -0400

Split routerlist.c into 4 separate modules

There are now separate modules for: * the list of router descriptors * the list of authorities and fallbacks * managing authority certificates * selecting random nodes --- src/app/config/config.c | 2 +- src/core/include.am | 6 + src/core/mainloop/main.c | 2 + src/core/or/channeltls.c | 2 +- src/core/or/circuitbuild.c | 2 +- src/core/or/connection_or.c | 1 + src/core/or/reasons.c | 2 +- src/feature/client/bridges.c | 2 + src/feature/client/entrynodes.c | 2 +- src/feature/control/control.c | 2 + src/feature/dirauth/dirvote.c | 2 + src/feature/dirauth/shared_random.c | 3 +- src/feature/dircache/directory.c | 4 + src/feature/dircache/dirserv.c | 2 + src/feature/hs/hs_service.c | 2 +- src/feature/nodelist/authcert.c | 1205 +++++++++++++++ src/feature/nodelist/authcert.h | 60 + src/feature/nodelist/dirlist.c | 421 ++++++ src/feature/nodelist/dirlist.h | 47 + src/feature/nodelist/microdesc.c | 1 + src/feature/nodelist/networkstatus.c | 3 + src/feature/nodelist/node_select.c | 1108 ++++++++++++++ src/feature/nodelist/node_select.h | 102 ++ src/feature/nodelist/nodelist.c | 2 + src/feature/nodelist/routerlist.c | 2777 +--------------------------------- src/feature/nodelist/routerlist.h | 154 +- src/feature/nodelist/routerparse.c | 1 + src/feature/relay/router.c | 2 + src/feature/rend/rendservice.c | 2 +- src/test/test_config.c | 1 + src/test/test_controller.c | 2 +- src/test/test_dir.c | 4 + src/test/test_dir_handle_get.c | 2 + src/test/test_routerlist.c | 3 + src/test/test_shared_random.c | 3 +- 35 files changed, 3074 insertions(+), 2862 deletions(-)

diff --git a/src/app/config/config.c b/src/app/config/config.c index 253d7ac9e..a3e775f48 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -105,7 +105,7 @@ #include "feature/stats/rephist.h" #include "feature/relay/router.h" #include "lib/sandbox/sandbox.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerset.h" #include "core/or/scheduler.h" #include "app/config/statefile.h" diff --git a/src/core/include.am b/src/core/include.am index 4ccabe438..7adcc0d9d 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -80,9 +80,12 @@ LIBTOR_APP_A_SOURCES = \ src/feature/hs/hs_stats.c \ src/feature/hs_common/replaycache.c \ src/feature/hs_common/shared_random_client.c \ + src/feature/nodelist/authcert.c \ + src/feature/nodelist/dirlist.c \ src/feature/nodelist/microdesc.c \ src/feature/nodelist/networkstatus.c \ src/feature/nodelist/nodelist.c \ + src/feature/nodelist/node_select.c \ src/feature/nodelist/parsecommon.c \ src/feature/nodelist/routerlist.c \ src/feature/nodelist/routerparse.c \ @@ -256,8 +259,10 @@ noinst_HEADERS += \ src/feature/hs/hsdir_index_st.h \ src/feature/hs_common/replaycache.h \ src/feature/hs_common/shared_random_client.h \ + src/feature/nodelist/authcert.h \ src/feature/nodelist/authority_cert_st.h \ src/feature/nodelist/desc_store_st.h \ + src/feature/nodelist/dirlist.h \ src/feature/nodelist/document_signature_st.h \ src/feature/nodelist/extrainfo_st.h \ src/feature/nodelist/microdesc.h \ @@ -268,6 +273,7 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/node_st.h \ src/feature/nodelist/nodelist.h \ + src/feature/nodelist/node_select.h \ src/feature/nodelist/parsecommon.h \ src/feature/nodelist/routerinfo_st.h \ src/feature/nodelist/routerlist.h \ diff --git a/src/core/mainloop/main.c b/src/core/mainloop/main.c index 0a31a3145..6884021bb 100644 --- a/src/core/mainloop/main.c +++ b/src/core/mainloop/main.c @@ -101,6 +101,8 @@ #include "feature/stats/rephist.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "core/or/scheduler.h" diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index b90a2ea87..c81a00b49 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -53,7 +53,7 @@ #include "core/or/relay.h" #include "feature/stats/rephist.h" #include "feature/relay/router.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/dirlist.h" #include "core/or/scheduler.h" #include "feature/nodelist/torcert.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index b1dcfb853..ea23c1dfd 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -60,6 +60,7 @@ #include "feature/rend/rendcommon.h" #include "feature/stats/rephist.h" #include "feature/relay/router.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" @@ -3007,4 +3008,3 @@ circuit_upgrade_circuits_from_guard_wait(void)

smartlist_free(to_upgrade); } - diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index e559e6b98..de6066020 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -56,6 +56,7 @@ #include "feature/stats/rephist.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/ext_orport.h" #include "core/or/scheduler.h" diff --git a/src/core/or/reasons.c b/src/core/or/reasons.c index 7a1e09e9f..610338ee1 100644 --- a/src/core/or/reasons.c +++ b/src/core/or/reasons.c @@ -17,7 +17,7 @@ #include "core/or/or.h" #include "app/config/config.h" #include "core/or/reasons.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/node_select.h" #include "lib/tls/tortls.h"

/***************************** Edge (stream) reasons **********************/ diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 7f4422f78..5ab0f6f4f 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -23,7 +23,9 @@ #include "feature/nodelist/nodelist.h" #include "core/or/policies.h" #include "feature/relay/router.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" + #include "feature/nodelist/routerset.h" #include "feature/client/transports.h"

diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 494ad3352..f16de722d 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -133,7 +133,7 @@ #include "feature/nodelist/nodelist.h" #include "core/or/policies.h" #include "feature/relay/router.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" #include "feature/client/transports.h" diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 5ac7f6d99..cff3c414c 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -77,6 +77,8 @@ #include "feature/rend/rendservice.h" #include "feature/stats/rephist.h" #include "feature/relay/router.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "feature/hs_common/shared_random_client.h" diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 14357c770..76877490b 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -19,6 +19,8 @@ #include "feature/stats/rephist.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "feature/client/entrynodes.h" /* needed for guardfraction methods */ diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c index 8ae267977..85ec82ac0 100644 --- a/src/feature/dirauth/shared_random.c +++ b/src/feature/dirauth/shared_random.c @@ -96,7 +96,7 @@ #include "feature/nodelist/networkstatus.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/dirlist.h" #include "feature/hs_common/shared_random_client.h" #include "feature/dirauth/shared_random_state.h" #include "feature/dircommon/voting_schedule.h" @@ -1288,4 +1288,3 @@ set_num_srv_agreements(int32_t value) }

#endif /* defined(TOR_UNIT_TESTS) */ - diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c index b94c5317a..8628a35c5 100644 --- a/src/feature/dircache/directory.c +++ b/src/feature/dircache/directory.c @@ -40,7 +40,11 @@ #include "feature/rend/rendservice.h" #include "feature/stats/rephist.h" #include "feature/relay/router.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" + #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" #include "lib/encoding/confline.h" diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index b85db8324..b3cd83e21 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -28,7 +28,9 @@ #include "core/or/protover.h" #include "feature/stats/rephist.h" #include "feature/relay/router.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" + #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" #include "feature/nodelist/torcert.h" diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 243ecb9b5..83607e624 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -27,7 +27,7 @@ #include "feature/rend/rendservice.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/node_select.h" #include "feature/hs_common/shared_random_client.h" #include "app/config/statefile.h"

diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c new file mode 100644 index 000000000..c4a037590 --- /dev/null +++ b/src/feature/nodelist/authcert.c @@ -0,0 +1,1205 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file authcert.c + * \brief Code to maintain directory authorities' certificates. + * + * Authority certificates are signed with authority identity keys; they + * are used to authenticate shorter-term authority signing keys. We + * fetch them when we find a consensus or a vote that has been signed + * with a signing key we don't recognize. We cache them on disk and + * load them on startup. Authority operators generate them with the + * "tor-gencert" utility. + */ + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/main.h" +#include "core/or/policies.h" +#include "feature/client/bridges.h" +#include "feature/dircache/directory.h" +#include "feature/dircommon/fp_pair.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerparse.h" +#include "feature/relay/router.h" + +#include "core/or/connection_st.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/dircommon/dir_connection_st.h" +#include "feature/nodelist/authority_cert_st.h" +#include "feature/nodelist/document_signature_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/networkstatus_voter_info_st.h" +#include "feature/nodelist/node_st.h" + +DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t) +#define DSMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \ + valvar) +#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn)) + +/* Forward declaration for cert_list_t */ +typedef struct cert_list_t cert_list_t; + +static void download_status_reset_by_sk_in_cl(cert_list_t *cl, + const char *digest); +static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl, + const char *digest, + time_t now); +static void list_pending_fpsk_downloads(fp_pair_map_t *result); + +/** List of certificates for a single authority, and download status for + * latest certificate. + */ +struct cert_list_t { + /* + * The keys of download status map are cert->signing_key_digest for pending + * downloads by (identity digest/signing key digest) pair; functions such + * as authority_cert_get_by_digest() already assume these are unique. + */ + struct digest_ds_map_t *dl_status_map; + /* There is also a dlstatus for the download by identity key only */ + download_status_t dl_status_by_id; + smartlist_t *certs; +}; +/** Map from v3 identity key digest to cert_list_t. */ +static digestmap_t *trusted_dir_certs = NULL; + +/** True iff any key certificate in at least one member of + * <b>trusted_dir_certs</b> has changed since we last flushed the + * certificates to disk. */ +static int trusted_dir_servers_certs_changed = 0; + +/** Initialise schedule, want_authority, and increment_on in the download + * status dlstatus, then call download_status_reset() on it. + * It is safe to call this function or download_status_reset() multiple times + * on a new dlstatus. But it should *not* be called after a dlstatus has been + * used to count download attempts or failures. */ +static void +download_status_cert_init(download_status_t *dlstatus) +{ + dlstatus->schedule = DL_SCHED_CONSENSUS; + dlstatus->want_authority = DL_WANT_ANY_DIRSERVER; + dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE; + dlstatus->last_backoff_position = 0; + dlstatus->last_delay_used = 0; + + /* Use the new schedule to set next_attempt_at */ + download_status_reset(dlstatus); +} + +/** Reset the download status of a specified element in a dsmap */ +static void +download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest) +{ + download_status_t *dlstatus = NULL; + + tor_assert(cl); + tor_assert(digest); + + /* Make sure we have a dsmap */ + if (!(cl->dl_status_map)) { + cl->dl_status_map = dsmap_new(); + } + /* Look for a download_status_t in the map with this digest */ + dlstatus = dsmap_get(cl->dl_status_map, digest); + /* Got one? */ + if (!dlstatus) { + /* Insert before we reset */ + dlstatus = tor_malloc_zero(sizeof(*dlstatus)); + dsmap_set(cl->dl_status_map, digest, dlstatus); + download_status_cert_init(dlstatus); + } + tor_assert(dlstatus); + /* Go ahead and reset it */ + download_status_reset(dlstatus); +} + +/** + * Return true if the download for this signing key digest in cl is ready + * to be re-attempted. + */ +static int +download_status_is_ready_by_sk_in_cl(cert_list_t *cl, + const char *digest, + time_t now) +{ + int rv = 0; + download_status_t *dlstatus = NULL; + + tor_assert(cl); + tor_assert(digest); + + /* Make sure we have a dsmap */ + if (!(cl->dl_status_map)) { + cl->dl_status_map = dsmap_new(); + } + /* Look for a download_status_t in the map with this digest */ + dlstatus = dsmap_get(cl->dl_status_map, digest); + /* Got one? */ + if (dlstatus) { + /* Use download_status_is_ready() */ + rv = download_status_is_ready(dlstatus, now); + } else { + /* + * If we don't know anything about it, return 1, since we haven't + * tried this one before. We need to create a new entry here, + * too. + */ + dlstatus = tor_malloc_zero(sizeof(*dlstatus)); + download_status_cert_init(dlstatus); + dsmap_set(cl->dl_status_map, digest, dlstatus); + rv = 1; + } + + return rv; +} + +/** Helper: Return the cert_list_t for an authority whose authority ID is + * <b>id_digest</b>, allocating a new list if necessary. */ +static cert_list_t * +get_cert_list(const char *id_digest) +{ + cert_list_t *cl; + if (!trusted_dir_certs) + trusted_dir_certs = digestmap_new(); + cl = digestmap_get(trusted_dir_certs, id_digest); + if (!cl) { + cl = tor_malloc_zero(sizeof(cert_list_t)); + download_status_cert_init(&cl->dl_status_by_id); + cl->certs = smartlist_new(); + cl->dl_status_map = dsmap_new(); + digestmap_set(trusted_dir_certs, id_digest, cl); + } + return cl; +} + +/** Return a list of authority ID digests with potentially enumerable lists + * of download_status_t objects; used by controller GETINFO queries. + */ + +MOCK_IMPL(smartlist_t *, +list_authority_ids_with_downloads, (void)) +{ + smartlist_t *ids = smartlist_new(); + digestmap_iter_t *i; + const char *digest; + char *tmp; + void *cl; + + if (trusted_dir_certs) { + for (i = digestmap_iter_init(trusted_dir_certs); + !(digestmap_iter_done(i)); + i = digestmap_iter_next(trusted_dir_certs, i)) { + /* + * We always have at least dl_status_by_id to query, so no need to + * probe deeper than the existence of a cert_list_t. + */ + digestmap_iter_get(i, &digest, &cl); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, digest, DIGEST_LEN); + smartlist_add(ids, tmp); + } + } + /* else definitely no downloads going since nothing even has a cert list */ + + return ids; +} + +/** Given an authority ID digest, return a pointer to the default download + * status, or NULL if there is no such entry in trusted_dir_certs */ + +MOCK_IMPL(download_status_t *, +id_only_download_status_for_authority_id, (const char *digest)) +{ + download_status_t *dl = NULL; + cert_list_t *cl; + + if (trusted_dir_certs) { + cl = digestmap_get(trusted_dir_certs, digest); + if (cl) { + dl = &(cl->dl_status_by_id); + } + } + + return dl; +} + +/** Given an authority ID digest, return a smartlist of signing key digests + * for which download_status_t is potentially queryable, or NULL if no such + * authority ID digest is known. */ + +MOCK_IMPL(smartlist_t *, +list_sk_digests_for_authority_id, (const char *digest)) +{ + smartlist_t *sks = NULL; + cert_list_t *cl; + dsmap_iter_t *i; + const char *sk_digest; + char *tmp; + download_status_t *dl; + + if (trusted_dir_certs) { + cl = digestmap_get(trusted_dir_certs, digest); + if (cl) { + sks = smartlist_new(); + if (cl->dl_status_map) { + for (i = dsmap_iter_init(cl->dl_status_map); + !(dsmap_iter_done(i)); + i = dsmap_iter_next(cl->dl_status_map, i)) { + /* Pull the digest out and add it to the list */ + dsmap_iter_get(i, &sk_digest, &dl); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, sk_digest, DIGEST_LEN); + smartlist_add(sks, tmp); + } + } + } + } + + return sks; +} + +/** Given an authority ID digest and a signing key digest, return the + * download_status_t or NULL if none exists. */ + +MOCK_IMPL(download_status_t *, +download_status_for_authority_id_and_sk,(const char *id_digest, + const char *sk_digest)) +{ + download_status_t *dl = NULL; + cert_list_t *cl = NULL; + + if (trusted_dir_certs) { + cl = digestmap_get(trusted_dir_certs, id_digest); + if (cl && cl->dl_status_map) { + dl = dsmap_get(cl->dl_status_map, sk_digest); + } + } + + return dl; +} + +#define cert_list_free(val) \ + FREE_AND_NULL(cert_list_t, cert_list_free_, (val)) + +/** Release all space held by a cert_list_t */ +static void +cert_list_free_(cert_list_t *cl) +{ + if (!cl) + return; + + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, + authority_cert_free(cert)); + smartlist_free(cl->certs); + dsmap_free(cl->dl_status_map, tor_free_); + tor_free(cl); +} + +/** Wrapper for cert_list_free so we can pass it to digestmap_free */ +static void +cert_list_free_void(void *cl) +{ + cert_list_free_(cl); +} + +/** Reload the cached v3 key certificates from the cached-certs file in + * the data directory. Return 0 on success, -1 on failure. */ +int +trusted_dirs_reload_certs(void) +{ + char *filename; + char *contents; + int r; + + filename = get_cachedir_fname("cached-certs"); + contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + tor_free(filename); + if (!contents) + return 0; + r = trusted_dirs_load_certs_from_string( + contents, + TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL); + tor_free(contents); + return r; +} + +/** Helper: return true iff we already have loaded the exact cert + * <b>cert</b>. */ +static inline int +already_have_cert(authority_cert_t *cert) +{ + cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest); + + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c, + { + if (tor_memeq(c->cache_info.signed_descriptor_digest, + cert->cache_info.signed_descriptor_digest, + DIGEST_LEN)) + return 1; + }); + return 0; +} + +/** Load a bunch of new key certificates from the string <b>contents</b>. If + * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are + * from the cache, and we don't need to flush them to disk. If we are a + * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF. + * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST + * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST. If <b>flush</b> is true, we + * need to flush any changed certificates to disk now. Return 0 on success, + * -1 if any certs fail to parse. + * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved certificates from, so try it first to + * fetch any missing certificates. + */ +int +trusted_dirs_load_certs_from_string(const char *contents, int source, + int flush, const char *source_dir) +{ + dir_server_t *ds; + const char *s, *eos; + int failure_code = 0; + int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE); + int added_trusted_cert = 0; + + for (s = contents; *s; s = eos) { + authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); + cert_list_t *cl; + if (!cert) { + failure_code = -1; + break; + } + ds = trusteddirserver_get_by_v3_auth_digest( + cert->cache_info.identity_digest); + log_debug(LD_DIR, "Parsed certificate for %s", + ds ? ds->nickname : "unknown authority"); + + if (already_have_cert(cert)) { + /* we already have this one. continue. */ + log_info(LD_DIR, "Skipping %s certificate for %s that we " + "already have.", + from_store ? "cached" : "downloaded", + ds ? ds->nickname : "an old or new authority"); + + /* + * A duplicate on download should be treated as a failure, so we call + * authority_cert_dl_failed() to reset the download status to make sure + * we can't try again. Since we've implemented the fp-sk mechanism + * to download certs by signing key, this should be much rarer than it + * was and is perhaps cause for concern. + */ + if (!from_store) { + if (authdir_mode(get_options())) { + log_warn(LD_DIR, + "Got a certificate for %s, but we already have it. " + "Maybe they haven't updated it. Waiting for a while.", + ds ? ds->nickname : "an old or new authority"); + } else { + log_info(LD_DIR, + "Got a certificate for %s, but we already have it. " + "Maybe they haven't updated it. Waiting for a while.", + ds ? ds->nickname : "an old or new authority"); + } + + /* + * This is where we care about the source; authority_cert_dl_failed() + * needs to know whether the download was by fp or (fp,sk) pair to + * twiddle the right bit in the download map. + */ + if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) { + authority_cert_dl_failed(cert->cache_info.identity_digest, + NULL, 404); + } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) { + authority_cert_dl_failed(cert->cache_info.identity_digest, + cert->signing_key_digest, 404); + } + } + + authority_cert_free(cert); + continue; + } + + if (ds) { + added_trusted_cert = 1; + log_info(LD_DIR, "Adding %s certificate for directory authority %s with " + "signing key %s", from_store ? "cached" : "downloaded", + ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN)); + } else { + int adding = we_want_to_fetch_unknown_auth_certs(get_options()); + log_info(LD_DIR, "%s %s certificate for unrecognized directory " + "authority with signing key %s", + adding ? "Adding" : "Not adding", + from_store ? "cached" : "downloaded", + hex_str(cert->signing_key_digest,DIGEST_LEN)); + if (!adding) { + authority_cert_free(cert); + continue; + } + } + + cl = get_cert_list(cert->cache_info.identity_digest); + smartlist_add(cl->certs, cert); + if (ds && cert->cache_info.published_on > ds->addr_current_at) { + /* Check to see whether we should update our view of the authority's + * address. */ + if (cert->addr && cert->dir_port && + (ds->addr != cert->addr || + ds->dir_port != cert->dir_port)) { + char *a = tor_dup_ip(cert->addr); + log_notice(LD_DIR, "Updating address for directory authority %s " + "from %s:%d to %s:%d based on certificate.", + ds->nickname, ds->address, (int)ds->dir_port, + a, cert->dir_port); + tor_free(a); + ds->addr = cert->addr; + ds->dir_port = cert->dir_port; + } + ds->addr_current_at = cert->cache_info.published_on; + } + + if (!from_store) + trusted_dir_servers_certs_changed = 1; + } + + if (flush) + trusted_dirs_flush_certs_to_disk(); + + /* call this even if failure_code is <0, since some certs might have + * succeeded, but only pass source_dir if there were no failures, + * and at least one more authority certificate was added to the store. + * This avoids retrying a directory that's serving bad or entirely duplicate + * certificates. */ + if (failure_code == 0 && added_trusted_cert) { + networkstatus_note_certs_arrived(source_dir); + } else { + networkstatus_note_certs_arrived(NULL); + } + + return failure_code; +} + +/** Save all v3 key certificates to the cached-certs file. */ +void +trusted_dirs_flush_certs_to_disk(void) +{ + char *filename; + smartlist_t *chunks; + + if (!trusted_dir_servers_certs_changed || !trusted_dir_certs) + return; + + chunks = smartlist_new(); + DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, + { + sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t)); + c->bytes = cert->cache_info.signed_descriptor_body; + c->len = cert->cache_info.signed_descriptor_len; + smartlist_add(chunks, c); + }); + } DIGESTMAP_FOREACH_END; + + filename = get_cachedir_fname("cached-certs"); + if (write_chunks_to_file(filename, chunks, 0, 0)) { + log_warn(LD_FS, "Error writing certificates to disk."); + } + tor_free(filename); + SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c)); + smartlist_free(chunks); + + trusted_dir_servers_certs_changed = 0; +} + +static int +compare_certs_by_pubdates(const void **_a, const void **_b) +{ + const authority_cert_t *cert1 = *_a, *cert2=*_b; + + if (cert1->cache_info.published_on < cert2->cache_info.published_on) + return -1; + else if (cert1->cache_info.published_on > cert2->cache_info.published_on) + return 1; + else + return 0; +} + +/** Remove all expired v3 authority certificates that have been superseded for + * more than 48 hours or, if not expired, that were published more than 7 days + * before being superseded. (If the most recent cert was published more than 48 + * hours ago, then we aren't going to get any consensuses signed with older + * keys.) */ +void +trusted_dirs_remove_old_certs(void) +{ + time_t now = time(NULL); +#define DEAD_CERT_LIFETIME (2*24*60*60) +#define SUPERSEDED_CERT_LIFETIME (2*24*60*60) + if (!trusted_dir_certs) + return; + + DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { + /* Sort the list from first-published to last-published */ + smartlist_sort(cl->certs, compare_certs_by_pubdates); + + SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { + if (cert_sl_idx == smartlist_len(cl->certs) - 1) { + /* This is the most recently published cert. Keep it. */ + continue; + } + authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1); + const time_t next_cert_published = next_cert->cache_info.published_on; + if (next_cert_published > now) { + /* All later certs are published in the future. Keep everything + * we didn't discard. */ + break; + } + int should_remove = 0; + if (cert->expires + DEAD_CERT_LIFETIME < now) { + /* Certificate has been expired for at least DEAD_CERT_LIFETIME. + * Remove it. */ + should_remove = 1; + } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) { + /* Certificate has been superseded for OLD_CERT_LIFETIME. + * Remove it. + */ + should_remove = 1; + } + if (should_remove) { + SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert); + authority_cert_free(cert); + trusted_dir_servers_certs_changed = 1; + } + } SMARTLIST_FOREACH_END(cert); + + } DIGESTMAP_FOREACH_END; +#undef DEAD_CERT_LIFETIME +#undef OLD_CERT_LIFETIME + + trusted_dirs_flush_certs_to_disk(); +} + +/** Return the newest v3 authority certificate whose v3 authority identity key + * has digest <b>id_digest</b>. Return NULL if no such authority is known, + * or it has no certificate. */ +authority_cert_t * +authority_cert_get_newest_by_id(const char *id_digest) +{ + cert_list_t *cl; + authority_cert_t *best = NULL; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return NULL; + + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, + { + if (!best || cert->cache_info.published_on > best->cache_info.published_on) + best = cert; + }); + return best; +} + +/** Return the newest v3 authority certificate whose directory signing key has + * digest <b>sk_digest</b>. Return NULL if no such certificate is known. + */ +authority_cert_t * +authority_cert_get_by_sk_digest(const char *sk_digest) +{ + authority_cert_t *c; + if (!trusted_dir_certs) + return NULL; + + if ((c = get_my_v3_authority_cert()) && + tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN)) + return c; + if ((c = get_my_v3_legacy_cert()) && + tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN)) + return c; + + DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, + { + if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN)) + return cert; + }); + } DIGESTMAP_FOREACH_END; + return NULL; +} + +/** Return the v3 authority certificate with signing key matching + * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>. + * Return NULL if no such authority is known. */ +authority_cert_t * +authority_cert_get_by_digests(const char *id_digest, + const char *sk_digest) +{ + cert_list_t *cl; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return NULL; + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, + if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN)) + return cert; ); + + return NULL; +} + +/** Add every known authority_cert_t to <b>certs_out</b>. */ +void +authority_cert_get_all(smartlist_t *certs_out) +{ + tor_assert(certs_out); + if (!trusted_dir_certs) + return; + + DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c, + smartlist_add(certs_out, c)); + } DIGESTMAP_FOREACH_END; +} + +/** Called when an attempt to download a certificate with the authority with + * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest + * fails with HTTP response code <b>status</b>: remember the failure, so we + * don't try again immediately. */ +void +authority_cert_dl_failed(const char *id_digest, + const char *signing_key_digest, int status) +{ + cert_list_t *cl; + download_status_t *dlstatus = NULL; + char id_digest_str[2*DIGEST_LEN+1]; + char sk_digest_str[2*DIGEST_LEN+1]; + + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return; + + /* + * Are we noting a failed download of the latest cert for the id digest, + * or of a download by (id, signing key) digest pair? + */ + if (!signing_key_digest) { + /* Just by id digest */ + download_status_failed(&cl->dl_status_by_id, status); + } else { + /* Reset by (id, signing key) digest pair + * + * Look for a download_status_t in the map with this digest + */ + dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest); + /* Got one? */ + if (dlstatus) { + download_status_failed(dlstatus, status); + } else { + /* + * Do this rather than hex_str(), since hex_str clobbers + * old results and we call twice in the param list. + */ + base16_encode(id_digest_str, sizeof(id_digest_str), + id_digest, DIGEST_LEN); + base16_encode(sk_digest_str, sizeof(sk_digest_str), + signing_key_digest, DIGEST_LEN); + log_warn(LD_BUG, + "Got failure for cert fetch with (fp,sk) = (%s,%s), with " + "status %d, but knew nothing about the download.", + id_digest_str, sk_digest_str, status); + } + } +} + +static const char *BAD_SIGNING_KEYS[] = { + "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31 + "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26 + "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09 + "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09 + "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56 + "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01 + "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30 + "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09 + "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10 + "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57 + NULL, +}; + +/** Return true iff <b>cert</b> authenticates some atuhority signing key + * which, because of the old openssl heartbleed vulnerability, should + * never be trusted. */ +int +authority_cert_is_blacklisted(const authority_cert_t *cert) +{ + char hex_digest[HEX_DIGEST_LEN+1]; + int i; + base16_encode(hex_digest, sizeof(hex_digest), + cert->signing_key_digest, sizeof(cert->signing_key_digest)); + + for (i = 0; BAD_SIGNING_KEYS[i]; ++i) { + if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) { + return 1; + } + } + return 0; +} + +/** Return true iff when we've been getting enough failures when trying to + * download the certificate with ID digest <b>id_digest</b> that we're willing + * to start bugging the user about it. */ +int +authority_cert_dl_looks_uncertain(const char *id_digest) +{ +#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2 + cert_list_t *cl; + int n_failures; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return 0; + + n_failures = download_status_get_n_failures(&cl->dl_status_by_id); + return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; +} + +/* Fetch the authority certificates specified in resource. + * If we are a bridge client, and node is a configured bridge, fetch from node + * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from + * rs. Otherwise, fetch from a random directory mirror. */ +static void +authority_certs_fetch_resource_impl(const char *resource, + const char *dir_hint, + const node_t *node, + const routerstatus_t *rs) +{ + const or_options_t *options = get_options(); + int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0, + resource); + + /* Make sure bridge clients never connect to anything but a bridge */ + if (options->UseBridges) { + if (node && !node_is_a_configured_bridge(node)) { + /* If we're using bridges, and node is not a bridge, use a 3-hop path. */ + get_via_tor = 1; + } else if (!node) { + /* If we're using bridges, and there's no node, use a 3-hop path. */ + get_via_tor = 1; + } + } + + const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS + : DIRIND_ONEHOP; + + directory_request_t *req = NULL; + /* If we've just downloaded a consensus from a bridge, re-use that + * bridge */ + if (options->UseBridges && node && node->ri && !get_via_tor) { + /* clients always make OR connections to bridges */ + tor_addr_port_t or_ap; + /* we are willing to use a non-preferred address if we need to */ + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, + &or_ap); + + req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); + directory_request_set_or_addr_port(req, &or_ap); + if (dir_hint) + directory_request_set_directory_id_digest(req, dir_hint); + } else if (rs) { + /* And if we've just downloaded a consensus from a directory, re-use that + * directory */ + req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); + directory_request_set_routerstatus(req, rs); + } + + if (req) { + /* We've set up a request object -- fill in the other request fields, and + * send the request. */ + directory_request_set_indirection(req, indirection); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); + return; + } + + /* Otherwise, we want certs from a random fallback or directory + * mirror, because they will almost always succeed. */ + directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, + resource, PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); +} + +/** Try to download any v3 authority certificates that we may be missing. If + * <b>status</b> is provided, try to get all the ones that were used to sign + * <b>status</b>. Additionally, try to have a non-expired certificate for + * every V3 authority in trusted_dir_servers. Don't fetch certificates we + * already have. + * + * If dir_hint is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved a consensus or certificates from, so try + * it first to fetch any missing certificates. + **/ +void +authority_certs_fetch_missing(networkstatus_t *status, time_t now, + const char *dir_hint) +{ + /* + * The pending_id digestmap tracks pending certificate downloads by + * identity digest; the pending_cert digestmap tracks pending downloads + * by (identity digest, signing key digest) pairs. + */ + digestmap_t *pending_id; + fp_pair_map_t *pending_cert; + /* + * The missing_id_digests smartlist will hold a list of id digests + * we want to fetch the newest cert for; the missing_cert_digests + * smartlist will hold a list of fp_pair_t with an identity and + * signing key digest. + */ + smartlist_t *missing_cert_digests, *missing_id_digests; + char *resource = NULL; + cert_list_t *cl; + const or_options_t *options = get_options(); + const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options); + fp_pair_t *fp_tmp = NULL; + char id_digest_str[2*DIGEST_LEN+1]; + char sk_digest_str[2*DIGEST_LEN+1]; + + if (should_delay_dir_fetches(options, NULL)) + return; + + pending_cert = fp_pair_map_new(); + pending_id = digestmap_new(); + missing_cert_digests = smartlist_new(); + missing_id_digests = smartlist_new(); + + /* + * First, we get the lists of already pending downloads so we don't + * duplicate effort. + */ + list_pending_downloads(pending_id, NULL, + DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); + list_pending_fpsk_downloads(pending_cert); + + /* + * Now, we download any trusted authority certs we don't have by + * identity digest only. This gets the latest cert for that authority. + */ + SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(), + dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_DIRINFO)) + continue; + if (smartlist_contains_digest(missing_id_digests, + ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { + if (now < cert->expires) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&(cl->dl_status_by_id)); + /* No sense trying to download it specifically by signing key hash */ + download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest); + found = 1; + break; + } + } SMARTLIST_FOREACH_END(cert); + if (!found && + download_status_is_ready(&(cl->dl_status_by_id), now) && + !digestmap_get(pending_id, ds->v3_identity_digest)) { + log_info(LD_DIR, + "No current certificate known for authority %s " + "(ID digest %s); launching request.", + ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN)); + smartlist_add(missing_id_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); + + /* + * Next, if we have a consensus, scan through it and look for anything + * signed with a key from a cert we don't have. Those get downloaded + * by (fp,sk) pair, but if we don't know any certs at all for the fp + * (identity digest), and it's one of the trusted dir server certs + * we started off above or a pending download in pending_id, don't + * try to get it yet. Most likely, the one we'll get for that will + * have the right signing key too, and we'd just be downloading + * redundantly. + */ + if (status) { + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!keep_unknown && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We don't want unknown certs, and we don't know this + * authority.*/ + + /* + * If we don't know *any* cert for this authority, and a download by ID + * is pending or we added it to missing_id_digests above, skip this + * one for now to avoid duplicate downloads. + */ + cl = get_cert_list(voter->identity_digest); + if (smartlist_len(cl->certs) == 0) { + /* We have no certs at all for this one */ + + /* Do we have a download of one pending? */ + if (digestmap_get(pending_id, voter->identity_digest)) + continue; + + /* + * Are we about to launch a download of one due to the trusted + * dir server check above? + */ + if (smartlist_contains_digest(missing_id_digests, + voter->identity_digest)) + continue; + } + + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + authority_cert_t *cert = + authority_cert_get_by_digests(voter->identity_digest, + sig->signing_key_digest); + if (cert) { + if (now < cert->expires) + download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest); + continue; + } + if (download_status_is_ready_by_sk_in_cl( + cl, sig->signing_key_digest, now) && + !fp_pair_map_get_by_digests(pending_cert, + voter->identity_digest, + sig->signing_key_digest)) { + /* + * Do this rather than hex_str(), since hex_str clobbers + * old results and we call twice in the param list. + */ + base16_encode(id_digest_str, sizeof(id_digest_str), + voter->identity_digest, DIGEST_LEN); + base16_encode(sk_digest_str, sizeof(sk_digest_str), + sig->signing_key_digest, DIGEST_LEN); + + if (voter->nickname) { + log_info(LD_DIR, + "We're missing a certificate from authority %s " + "(ID digest %s) with signing key %s: " + "launching request.", + voter->nickname, id_digest_str, sk_digest_str); + } else { + log_info(LD_DIR, + "We're missing a certificate from authority ID digest " + "%s with signing key %s: launching request.", + id_digest_str, sk_digest_str); + } + + /* Allocate a new fp_pair_t to append */ + fp_tmp = tor_malloc(sizeof(*fp_tmp)); + memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first)); + memcpy(fp_tmp->second, sig->signing_key_digest, + sizeof(fp_tmp->second)); + smartlist_add(missing_cert_digests, fp_tmp); + } + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); + } + + /* Bridge clients look up the node for the dir_hint */ + const node_t *node = NULL; + /* All clients, including bridge clients, look up the routerstatus for the + * dir_hint */ + const routerstatus_t *rs = NULL; + + /* If we still need certificates, try the directory that just successfully + * served us a consensus or certificates. + * As soon as the directory fails to provide additional certificates, we try + * another, randomly selected directory. This avoids continual retries. + * (We only ever have one outstanding request per certificate.) + */ + if (dir_hint) { + if (options->UseBridges) { + /* Bridge clients try the nodelist. If the dir_hint is from an authority, + * or something else fetched over tor, we won't find the node here, but + * we will find the rs. */ + node = node_get_by_id(dir_hint); + } + + /* All clients try the consensus routerstatus, then the fallback + * routerstatus */ + rs = router_get_consensus_status_by_id(dir_hint); + if (!rs) { + /* This will also find authorities */ + const dir_server_t *ds = router_get_fallback_dirserver_by_digest( + dir_hint); + if (ds) { + rs = &ds->fake_status; + } + } + + if (!node && !rs) { + log_warn(LD_BUG, "Directory %s delivered a consensus, but %s" + "no routerstatus could be found for it.", + options->UseBridges ? "no node and " : "", + hex_str(dir_hint, DIGEST_LEN)); + } + } + + /* Do downloads by identity digest */ + if (smartlist_len(missing_id_digests) > 0) { + int need_plus = 0; + smartlist_t *fps = smartlist_new(); + + smartlist_add_strdup(fps, "fp/"); + + SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) { + char *fp = NULL; + + if (digestmap_get(pending_id, d)) + continue; + + base16_encode(id_digest_str, sizeof(id_digest_str), + d, DIGEST_LEN); + + if (need_plus) { + tor_asprintf(&fp, "+%s", id_digest_str); + } else { + /* No need for tor_asprintf() in this case; first one gets no '+' */ + fp = tor_strdup(id_digest_str); + need_plus = 1; + } + + smartlist_add(fps, fp); + } SMARTLIST_FOREACH_END(d); + + if (smartlist_len(fps) > 1) { + resource = smartlist_join_strings(fps, "", 0, NULL); + /* node and rs are directories that just gave us a consensus or + * certificates */ + authority_certs_fetch_resource_impl(resource, dir_hint, node, rs); + tor_free(resource); + } + /* else we didn't add any: they were all pending */ + + SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp)); + smartlist_free(fps); + } + + /* Do downloads by identity digest/signing key pair */ + if (smartlist_len(missing_cert_digests) > 0) { + int need_plus = 0; + smartlist_t *fp_pairs = smartlist_new(); + + smartlist_add_strdup(fp_pairs, "fp-sk/"); + + SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) { + char *fp_pair = NULL; + + if (fp_pair_map_get(pending_cert, d)) + continue; + + /* Construct string encodings of the digests */ + base16_encode(id_digest_str, sizeof(id_digest_str), + d->first, DIGEST_LEN); + base16_encode(sk_digest_str, sizeof(sk_digest_str), + d->second, DIGEST_LEN); + + /* Now tor_asprintf() */ + if (need_plus) { + tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str); + } else { + /* First one in the list doesn't get a '+' */ + tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str); + need_plus = 1; + } + + /* Add it to the list of pairs to request */ + smartlist_add(fp_pairs, fp_pair); + } SMARTLIST_FOREACH_END(d); + + if (smartlist_len(fp_pairs) > 1) { + resource = smartlist_join_strings(fp_pairs, "", 0, NULL); + /* node and rs are directories that just gave us a consensus or + * certificates */ + authority_certs_fetch_resource_impl(resource, dir_hint, node, rs); + tor_free(resource); + } + /* else they were all pending */ + + SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p)); + smartlist_free(fp_pairs); + } + + smartlist_free(missing_id_digests); + SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p)); + smartlist_free(missing_cert_digests); + digestmap_free(pending_id, NULL); + fp_pair_map_free(pending_cert, NULL); +} + +void +authcert_free_all(void) +{ + if (trusted_dir_certs) { + digestmap_free(trusted_dir_certs, cert_list_free_void); + trusted_dir_certs = NULL; + } +} + +/** Free storage held in <b>cert</b>. */ +void +authority_cert_free_(authority_cert_t *cert) +{ + if (!cert) + return; + + tor_free(cert->cache_info.signed_descriptor_body); + crypto_pk_free(cert->signing_key); + crypto_pk_free(cert->identity_key); + + tor_free(cert); +} + +/** For every certificate we are currently downloading by (identity digest, + * signing key digest) pair, set result[fp_pair] to (void *1). + */ +static void +list_pending_fpsk_downloads(fp_pair_map_t *result) +{ + const char *pfx = "fp-sk/"; + smartlist_t *tmp; + smartlist_t *conns; + const char *resource; + + tor_assert(result); + + tmp = smartlist_new(); + conns = get_connection_array(); + + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + if (conn->type == CONN_TYPE_DIR && + conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE && + !conn->marked_for_close) { + resource = TO_DIR_CONN(conn)->requested_resource; + if (!strcmpstart(resource, pfx)) + dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx), + tmp); + } + } SMARTLIST_FOREACH_END(conn); + + SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) { + fp_pair_map_set(result, fp, (void*)1); + tor_free(fp); + } SMARTLIST_FOREACH_END(fp); + + smartlist_free(tmp); +} diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h new file mode 100644 index 000000000..48326d7bd --- /dev/null +++ b/src/feature/nodelist/authcert.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file authcert.h + * \brief Header file for authcert.c + **/ + +#ifndef TOR_AUTHCERT_H +#define TOR_AUTHCERT_H + +#include "lib/testsupport/testsupport.h" + +int trusted_dirs_reload_certs(void); + +/* + * Pass one of these as source to trusted_dirs_load_certs_from_string() + * to indicate whence string originates; this controls error handling + * behavior such as marking downloads as failed. + */ + +#define TRUSTED_DIRS_CERTS_SRC_SELF 0 +#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1 +#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2 +#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3 +#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4 + +int trusted_dirs_load_certs_from_string(const char *contents, int source, + int flush, const char *source_dir); +void trusted_dirs_remove_old_certs(void); +void trusted_dirs_flush_certs_to_disk(void); +authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); +authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); +authority_cert_t *authority_cert_get_by_digests(const char *id_digest, + const char *sk_digest); +void authority_cert_get_all(smartlist_t *certs_out); +void authority_cert_dl_failed(const char *id_digest, + const char *signing_key_digest, int status); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now, + const char *dir_hint); +int authority_cert_dl_looks_uncertain(const char *id_digest); +int authority_cert_is_blacklisted(const authority_cert_t *cert); + +void authority_cert_free_(authority_cert_t *cert); +#define authority_cert_free(cert) \ + FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert)) + +MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void)); +MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id, + (const char *digest)); +MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id, + (const char *digest)); +MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk, + (const char *id_digest, const char *sk_digest)); + +void authcert_free_all(void); + +#endif diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c new file mode 100644 index 000000000..8f78df521 --- /dev/null +++ b/src/feature/nodelist/dirlist.c @@ -0,0 +1,421 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirlist.c + * \brief Code to maintain our lists of directory authorities and + * fallback directories. + * + * For the directory authorities, we have a list containing the public + * identity key, and contact points, for each authority. The + * authorities receive descriptors from relays, and publish consensuses, + * descriptors, and microdescriptors. This list is pre-configured. + * + * Fallback directories are well-known, stable, but untrusted directory + * caches that clients which have not yet bootstrapped can use to get + * their first networkstatus consensus, in order to find out where the + * Tor network really is. This list is pre-configured in + * fallback_dirs.inc. Every authority also serves as a fallback. + * + * Both fallback directories and directory authorities are are + * represented by a dir_server_t. + */ + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/or/policies.h" +#include "feature/control/control.h" +#include "feature/dircache/directory.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerset.h" +#include "feature/relay/router.h" +#include "lib/net/resolve.h" + +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/node_st.h" + +/** Global list of a dir_server_t object for each directory + * authority. */ +static smartlist_t *trusted_dir_servers = NULL; +/** Global list of dir_server_t objects for all directory authorities + * and all fallback directory servers. */ +static smartlist_t *fallback_dir_servers = NULL; + +/** Return the number of directory authorities whose type matches some bit set + * in <b>type</b> */ +int +get_n_authorities(dirinfo_type_t type) +{ + int n = 0; + if (!trusted_dir_servers) + return 0; + SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, + if (ds->type & type) + ++n); + return n; +} + +/** Return a smartlist containing a list of dir_server_t * for all + * known trusted dirservers. Callers must not modify the list or its + * contents. + */ +smartlist_t * +router_get_trusted_dir_servers_mutable(void) +{ + if (!trusted_dir_servers) + trusted_dir_servers = smartlist_new(); + + return trusted_dir_servers; +} + +smartlist_t * +router_get_fallback_dir_servers_mutable(void) +{ + if (!fallback_dir_servers) + fallback_dir_servers = smartlist_new(); + + return fallback_dir_servers; +} + +const smartlist_t * +router_get_trusted_dir_servers(void) +{ + return router_get_trusted_dir_servers_mutable(); +} + +const smartlist_t * +router_get_fallback_dir_servers(void) +{ + return router_get_fallback_dir_servers_mutable(); +} + +/** Reset all internal variables used to count failed downloads of network + * status objects. */ +void +router_reset_status_download_failures(void) +{ + mark_all_dirservers_up(fallback_dir_servers); +} + +/** Return the dir_server_t for the directory authority whose identity + * key hashes to <b>digest</b>, or NULL if no such authority is known. + */ +dir_server_t * +router_get_trusteddirserver_by_digest(const char *digest) +{ + if (!trusted_dir_servers) + return NULL; + + SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, + { + if (tor_memeq(ds->digest, digest, DIGEST_LEN)) + return ds; + }); + + return NULL; +} + +/** Return the dir_server_t for the fallback dirserver whose identity + * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of + * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir + * and UseDefaultFallbackDirs torrc options.) + * The list of fallback directories includes the list of authorities. + */ +dir_server_t * +router_get_fallback_dirserver_by_digest(const char *digest) +{ + if (!fallback_dir_servers) + return NULL; + + if (!digest) + return NULL; + + SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds, + { + if (tor_memeq(ds->digest, digest, DIGEST_LEN)) + return ds; + }); + + return NULL; +} + +/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>, + * or 0 if no such fallback is in the list of fallback_dir_servers. + * (fallback_dir_servers is affected by the FallbackDir and + * UseDefaultFallbackDirs torrc options.) + * The list of fallback directories includes the list of authorities. + */ +int +router_digest_is_fallback_dir(const char *digest) +{ + return (router_get_fallback_dirserver_by_digest(digest) != NULL); +} + +/** Return the dir_server_t for the directory authority whose + * v3 identity key hashes to <b>digest</b>, or NULL if no such authority + * is known. + */ +MOCK_IMPL(dir_server_t *, +trusteddirserver_get_by_v3_auth_digest, (const char *digest)) +{ + if (!trusted_dir_servers) + return NULL; + + SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, + { + if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) && + (ds->type & V3_DIRINFO)) + return ds; + }); + + return NULL; +} + +/** Mark as running every dir_server_t in <b>server_list</b>. */ +void +mark_all_dirservers_up(smartlist_t *server_list) +{ + if (server_list) { + SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) { + routerstatus_t *rs; + node_t *node; + dir->is_running = 1; + node = node_get_mutable_by_id(dir->digest); + if (node) + node->is_running = 1; + rs = router_get_mutable_consensus_status_by_id(dir->digest); + if (rs) { + rs->last_dir_503_at = 0; + control_event_networkstatus_changed_single(rs); + } + } SMARTLIST_FOREACH_END(dir); + } + router_dir_info_changed(); +} + +/** Return true iff <b>digest</b> is the digest of the identity key of a + * trusted directory matching at least one bit of <b>type</b>. If <b>type</b> + * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */ +int +router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type) +{ + if (!trusted_dir_servers) + return 0; + if (authdir_mode(get_options()) && router_digest_is_me(digest)) + return 1; + SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent, + if (tor_memeq(digest, ent->digest, DIGEST_LEN)) { + return (!type) || ((type & ent->type) != 0); + }); + return 0; +} + +/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity + * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, + * add ourself. If <b>is_authority</b>, this is a directory authority. Return + * the new directory server entry on success or NULL on failure. */ +static dir_server_t * +dir_server_new(int is_authority, + const char *nickname, + const tor_addr_t *addr, + const char *hostname, + uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, + const char *digest, const char *v3_auth_digest, + dirinfo_type_t type, + double weight) +{ + dir_server_t *ent; + uint32_t a; + char *hostname_ = NULL; + + tor_assert(digest); + + if (weight < 0) + return NULL; + + if (tor_addr_family(addr) == AF_INET) + a = tor_addr_to_ipv4h(addr); + else + return NULL; + + if (!hostname) + hostname_ = tor_addr_to_str_dup(addr); + else + hostname_ = tor_strdup(hostname); + + ent = tor_malloc_zero(sizeof(dir_server_t)); + ent->nickname = nickname ? tor_strdup(nickname) : NULL; + ent->address = hostname_; + ent->addr = a; + ent->dir_port = dir_port; + ent->or_port = or_port; + ent->is_running = 1; + ent->is_authority = is_authority; + ent->type = type; + ent->weight = weight; + if (addrport_ipv6) { + if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) { + log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6."); + tor_addr_make_unspec(&ent->ipv6_addr); + } else { + tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr); + ent->ipv6_orport = addrport_ipv6->port; + } + } else { + tor_addr_make_unspec(&ent->ipv6_addr); + } + + memcpy(ent->digest, digest, DIGEST_LEN); + if (v3_auth_digest && (type & V3_DIRINFO)) + memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN); + + if (nickname) + tor_asprintf(&ent->description, "directory server "%s" at %s:%d", + nickname, hostname_, (int)dir_port); + else + tor_asprintf(&ent->description, "directory server at %s:%d", + hostname_, (int)dir_port); + + ent->fake_status.addr = ent->addr; + tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr); + memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN); + if (nickname) + strlcpy(ent->fake_status.nickname, nickname, + sizeof(ent->fake_status.nickname)); + else + ent->fake_status.nickname[0] = '\0'; + ent->fake_status.dir_port = ent->dir_port; + ent->fake_status.or_port = ent->or_port; + ent->fake_status.ipv6_orport = ent->ipv6_orport; + + return ent; +} + +/** Create an authoritative directory server at + * <b>address</b>:<b>port</b>, with identity key <b>digest</b>. If + * <b>address</b> is NULL, add ourself. Return the new trusted directory + * server entry on success or NULL if we couldn't add it. */ +dir_server_t * +trusted_dir_server_new(const char *nickname, const char *address, + uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *ipv6_addrport, + const char *digest, const char *v3_auth_digest, + dirinfo_type_t type, double weight) +{ + uint32_t a; + tor_addr_t addr; + char *hostname=NULL; + dir_server_t *result; + + if (!address) { /* The address is us; we should guess. */ + if (resolve_my_address(LOG_WARN, get_options(), + &a, NULL, &hostname) < 0) { + log_warn(LD_CONFIG, + "Couldn't find a suitable address when adding ourself as a " + "trusted directory server."); + return NULL; + } + if (!hostname) + hostname = tor_dup_ip(a); + } else { + if (tor_lookup_hostname(address, &a)) { + log_warn(LD_CONFIG, + "Unable to lookup address for directory server at '%s'", + address); + return NULL; + } + hostname = tor_strdup(address); + } + tor_addr_from_ipv4h(&addr, a); + + result = dir_server_new(1, nickname, &addr, hostname, + dir_port, or_port, + ipv6_addrport, + digest, + v3_auth_digest, type, weight); + tor_free(hostname); + return result; +} + +/** Return a new dir_server_t for a fallback directory server at + * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest + * <b>id_digest</b> */ +dir_server_t * +fallback_dir_server_new(const tor_addr_t *addr, + uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, + const char *id_digest, double weight) +{ + return dir_server_new(0, NULL, addr, NULL, dir_port, or_port, + addrport_ipv6, + id_digest, + NULL, ALL_DIRINFO, weight); +} + +/** Add a directory server to the global list(s). */ +void +dir_server_add(dir_server_t *ent) +{ + if (!trusted_dir_servers) + trusted_dir_servers = smartlist_new(); + if (!fallback_dir_servers) + fallback_dir_servers = smartlist_new(); + + if (ent->is_authority) + smartlist_add(trusted_dir_servers, ent); + + smartlist_add(fallback_dir_servers, ent); + router_dir_info_changed(); +} + +#define dir_server_free(val) \ + FREE_AND_NULL(dir_server_t, dir_server_free_, (val)) + +/** Free storage held in <b>ds</b>. */ +static void +dir_server_free_(dir_server_t *ds) +{ + if (!ds) + return; + + tor_free(ds->nickname); + tor_free(ds->description); + tor_free(ds->address); + tor_free(ds); +} + +/** Remove all members from the list of dir servers. */ +void +clear_dir_servers(void) +{ + if (fallback_dir_servers) { + SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent, + dir_server_free(ent)); + smartlist_clear(fallback_dir_servers); + } else { + fallback_dir_servers = smartlist_new(); + } + if (trusted_dir_servers) { + smartlist_clear(trusted_dir_servers); + } else { + trusted_dir_servers = smartlist_new(); + } + router_dir_info_changed(); +} + +void +dirlist_free_all(void) +{ + clear_dir_servers(); + smartlist_free(trusted_dir_servers); + smartlist_free(fallback_dir_servers); + trusted_dir_servers = fallback_dir_servers = NULL; +} diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h new file mode 100644 index 000000000..6baa5686c --- /dev/null +++ b/src/feature/nodelist/dirlist.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirlist.h + * \brief Header file for dirlist.c + **/ + +#ifndef TOR_DIRLIST_H +#define TOR_DIRLIST_H + +int get_n_authorities(dirinfo_type_t type); +const smartlist_t *router_get_trusted_dir_servers(void); +const smartlist_t *router_get_fallback_dir_servers(void); +smartlist_t *router_get_trusted_dir_servers_mutable(void); +smartlist_t *router_get_fallback_dir_servers_mutable(void); +void mark_all_dirservers_up(smartlist_t *server_list); + +dir_server_t *router_get_trusteddirserver_by_digest(const char *d); +dir_server_t *router_get_fallback_dirserver_by_digest( + const char *digest); +int router_digest_is_fallback_dir(const char *digest); +MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, + (const char *d)); + +int router_digest_is_trusted_dir_type(const char *digest, + dirinfo_type_t type); +#define router_digest_is_trusted_dir(d) \ + router_digest_is_trusted_dir_type((d), NO_DIRINFO) + +dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, + uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, + const char *digest, const char *v3_auth_digest, + dirinfo_type_t type, double weight); +dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, + uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, + const char *id_digest, double weight); +void dir_server_add(dir_server_t *ent); + +void clear_dir_servers(void); +void dirlist_free_all(void); + +#endif diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index aa5e6b1db..203d079b1 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -22,6 +22,7 @@ #include "feature/nodelist/nodelist.h" #include "core/or/policies.h" #include "feature/relay/router.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h"

diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 81da9756a..d4340b6ca 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -63,6 +63,9 @@ #include "core/or/protover.h" #include "core/or/relay.h" #include "feature/relay/router.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "core/or/scheduler.h" diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c new file mode 100644 index 000000000..a9f045d45 --- /dev/null +++ b/src/feature/nodelist/node_select.c @@ -0,0 +1,1108 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file node_select.c + * \brief Code to choose nodes randomly based on restrictions and + * weighted probabilities. + **/ + +#define NODE_SELECT_PRIVATE +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/policies.h" +#include "core/or/reasons.h" +#include "feature/client/entrynodes.h" +#include "feature/dircache/directory.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerset.h" +#include "feature/relay/router.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/math/fp.h" + +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/routerstatus_st.h" + +static int compute_weighted_bandwidths(const smartlist_t *sl, + bandwidth_weight_rule_t rule, + double **bandwidths_out, + double *total_bandwidth_out); +static const routerstatus_t *router_pick_trusteddirserver_impl( + const smartlist_t *sourcelist, dirinfo_type_t auth, + int flags, int *n_busy_out); +static const routerstatus_t *router_pick_dirserver_generic( + smartlist_t *sourcelist, + dirinfo_type_t type, int flags); + +/** Try to find a running dirserver that supports operations of <b>type</b>. + * + * If there are no running dirservers in our routerlist and the + * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones + * (including authorities) as running again, and pick one. + * + * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include + * dirservers that we can't reach. + * + * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself + * (if we're a dirserver). + * + * Don't pick a fallback directory mirror if any non-fallback is viable; + * (the fallback directory mirrors include the authorities) + * try to avoid using servers that have returned 503 recently. + */ +const routerstatus_t * +router_pick_directory_server(dirinfo_type_t type, int flags) +{ + int busy = 0; + const routerstatus_t *choice; + + choice = router_pick_directory_server_impl(type, flags, &busy); + if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) + return choice; + + if (busy) { + /* If the reason that we got no server is that servers are "busy", + * we must be excluding good servers because we already have serverdesc + * fetches with them. Do not mark down servers up because of this. */ + tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))); + return NULL; + } + + log_info(LD_DIR, + "No reachable router entries for dirservers. " + "Trying them all again."); + /* mark all fallback directory mirrors as up again */ + router_reset_status_download_failures(); + /* try again */ + choice = router_pick_directory_server_impl(type, flags, NULL); + return choice; +} + +/** Try to find a running fallback directory. Flags are as for + * router_pick_directory_server. + */ +const routerstatus_t * +router_pick_dirserver_generic(smartlist_t *sourcelist, + dirinfo_type_t type, int flags) +{ + const routerstatus_t *choice; + int busy = 0; + + if (smartlist_len(sourcelist) == 1) { + /* If there's only one choice, then we should disable the logic that + * would otherwise prevent us from choosing ourself. */ + flags |= PDS_ALLOW_SELF; + } + + choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy); + if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) + return choice; + if (busy) { + /* If the reason that we got no server is that servers are "busy", + * we must be excluding good servers because we already have serverdesc + * fetches with them. Do not mark down servers up because of this. */ + tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))); + return NULL; + } + + log_info(LD_DIR, + "No dirservers are reachable. Trying them all again."); + mark_all_dirservers_up(sourcelist); + return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL); +} + +/* Common retry code for router_pick_directory_server_impl and + * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version. + * Must be called before RETRY_WITHOUT_EXCLUDE(). + * + * If we got no result, and we are applying IP preferences, and we are a + * client that could use an alternate IP version, try again with the + * opposite preferences. */ +#define RETRY_ALTERNATE_IP_VERSION(retry_label) \ + STMT_BEGIN \ + if (result == NULL && try_ip_pref && options->ClientUseIPv4 \ + && fascist_firewall_use_ipv6(options) && !server_mode(options) \ + && !n_busy) { \ + n_excluded = 0; \ + n_busy = 0; \ + try_ip_pref = 0; \ + goto retry_label; \ + } \ + STMT_END \ + +/* Common retry code for router_pick_directory_server_impl and + * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with + * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION(). + * + * If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ +#define RETRY_WITHOUT_EXCLUDE(retry_label) \ + STMT_BEGIN \ + if (result == NULL && try_excluding && !options->StrictNodes \ + && n_excluded && !n_busy) { \ + try_excluding = 0; \ + n_excluded = 0; \ + n_busy = 0; \ + try_ip_pref = 1; \ + goto retry_label; \ + } \ + STMT_END + +/* Common code used in the loop within router_pick_directory_server_impl and + * router_pick_trusteddirserver_impl. + * + * Check if the given <b>identity</b> supports extrainfo. If not, skip further + * checks. + */ +#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity) \ + STMT_BEGIN \ + int is_trusted_extrainfo = router_digest_is_trusted_dir_type( \ + (identity), EXTRAINFO_DIRINFO); \ + if (((type) & EXTRAINFO_DIRINFO) && \ + !router_supports_extrainfo((identity), is_trusted_extrainfo)) \ + continue; \ + STMT_END + +#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP +#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0 +#endif + +/* Log a message if rs is not found or not a preferred address */ +static void +router_picked_poor_directory_log(const routerstatus_t *rs) +{ + const networkstatus_t *usable_consensus; + usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL), + usable_consensus_flavor()); + +#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP + /* Don't log early in the bootstrap process, it's normal to pick from a + * small pool of nodes. Of course, this won't help if we're trying to + * diagnose bootstrap issues. */ + if (!smartlist_len(nodelist_get_list()) || !usable_consensus + || !router_have_minimum_dir_info()) { + return; + } +#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */ + + /* We couldn't find a node, or the one we have doesn't fit our preferences. + * Sometimes this is normal, sometimes it can be a reachability issue. */ + if (!rs) { + /* This happens a lot, so it's at debug level */ + log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but " + "we couldn't find a directory that fit our criteria. " + "Perhaps we will succeed next time with less strict criteria."); + } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1) + && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1) + ) { + /* This is rare, and might be interesting to users trying to diagnose + * connection issues on dual-stack machines. */ + log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir " + "addresses for launching an outgoing connection: " + "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d", + routerstatus_describe(rs), + fmt_addr32(rs->addr), rs->or_port, + rs->dir_port, fmt_addr(&rs->ipv6_addr), + rs->ipv6_orport, rs->dir_port); + } +} + +#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP + +/* Check if we already have a directory fetch from ap, for serverdesc + * (including extrainfo) or microdesc documents. + * If so, return 1, if not, return 0. + * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0. + */ +STATIC int +router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc, + int microdesc) +{ + if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) { + return 0; + } + + /* XX/teor - we're not checking tunnel connections here, see #17848 + */ + if (serverdesc && ( + connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC) + || connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) { + return 1; + } + + if (microdesc && ( + connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) { + return 1; + } + + return 0; +} + +/* Check if we already have a directory fetch from the ipv4 or ipv6 + * router, for serverdesc (including extrainfo) or microdesc documents. + * If so, return 1, if not, return 0. + */ +static int +router_is_already_dir_fetching_(uint32_t ipv4_addr, + const tor_addr_t *ipv6_addr, + uint16_t dir_port, + int serverdesc, + int microdesc) +{ + tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap; + + /* Assume IPv6 DirPort is the same as IPv4 DirPort */ + tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr); + ipv4_dir_ap.port = dir_port; + tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr); + ipv6_dir_ap.port = dir_port; + + return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc) + || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc)); +} + +/** Pick a random running valid directory server/mirror from our + * routerlist. Arguments are as for router_pick_directory_server(), except: + * + * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of + * directories that we excluded for no other reason than + * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH. + */ +STATIC const routerstatus_t * +router_pick_directory_server_impl(dirinfo_type_t type, int flags, + int *n_busy_out) +{ + const or_options_t *options = get_options(); + const node_t *result; + smartlist_t *direct, *tunnel; + smartlist_t *trusted_direct, *trusted_tunnel; + smartlist_t *overloaded_direct, *overloaded_tunnel; + time_t now = time(NULL); + const networkstatus_t *consensus = networkstatus_get_latest_consensus(); + const int requireother = ! (flags & PDS_ALLOW_SELF); + const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); + const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); + const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH); + int try_excluding = 1, n_excluded = 0, n_busy = 0; + int try_ip_pref = 1; + + if (!consensus) + return NULL; + + retry_search: + + direct = smartlist_new(); + tunnel = smartlist_new(); + trusted_direct = smartlist_new(); + trusted_tunnel = smartlist_new(); + overloaded_direct = smartlist_new(); + overloaded_tunnel = smartlist_new(); + + const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref); + const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref); + const int must_have_or = directory_must_use_begindir(options); + + /* Find all the running dirservers we know about. */ + SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { + int is_trusted; + int is_overloaded; + const routerstatus_t *status = node->rs; + const country_t country = node->country; + if (!status) + continue; + + if (!node->is_running || !node_is_dir(node) || !node->is_valid) + continue; + if (requireother && router_digest_is_me(node->identity)) + continue; + + SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity); + + if (try_excluding && + routerset_contains_routerstatus(options->ExcludeNodes, status, + country)) { + ++n_excluded; + continue; + } + + if (router_is_already_dir_fetching_(status->addr, + &status->ipv6_addr, + status->dir_port, + no_serverdesc_fetching, + no_microdesc_fetching)) { + ++n_busy; + continue; + } + + is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now; + is_trusted = router_digest_is_trusted_dir(node->identity); + + /* Clients use IPv6 addresses if the server has one and the client + * prefers IPv6. + * Add the router if its preferred address and port are reachable. + * If we don't get any routers, we'll try again with the non-preferred + * address for each router (if any). (To ensure correct load-balancing + * we try routers that only have one address both times.) + */ + if (!fascistfirewall || skip_or_fw || + fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, + try_ip_pref)) + smartlist_add(is_trusted ? trusted_tunnel : + is_overloaded ? overloaded_tunnel : tunnel, (void*)node); + else if (!must_have_or && (skip_dir_fw || + fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION, + try_ip_pref))) + smartlist_add(is_trusted ? trusted_direct : + is_overloaded ? overloaded_direct : direct, (void*)node); + } SMARTLIST_FOREACH_END(node); + + if (smartlist_len(tunnel)) { + result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); + } else if (smartlist_len(overloaded_tunnel)) { + result = node_sl_choose_by_bandwidth(overloaded_tunnel, + WEIGHT_FOR_DIR); + } else if (smartlist_len(trusted_tunnel)) { + /* FFFF We don't distinguish between trusteds and overloaded trusteds + * yet. Maybe one day we should. */ + /* FFFF We also don't load balance over authorities yet. I think this + * is a feature, but it could easily be a bug. -RD */ + result = smartlist_choose(trusted_tunnel); + } else if (smartlist_len(direct)) { + result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); + } else if (smartlist_len(overloaded_direct)) { + result = node_sl_choose_by_bandwidth(overloaded_direct, + WEIGHT_FOR_DIR); + } else { + result = smartlist_choose(trusted_direct); + } + smartlist_free(direct); + smartlist_free(tunnel); + smartlist_free(trusted_direct); + smartlist_free(trusted_tunnel); + smartlist_free(overloaded_direct); + smartlist_free(overloaded_tunnel); + + RETRY_ALTERNATE_IP_VERSION(retry_search); + + RETRY_WITHOUT_EXCLUDE(retry_search); + + if (n_busy_out) + *n_busy_out = n_busy; + + router_picked_poor_directory_log(result ? result->rs : NULL); + + return result ? result->rs : NULL; +} + +/** Given an array of double/uint64_t unions that are currently being used as + * doubles, convert them to uint64_t, and try to scale them linearly so as to + * much of the range of uint64_t. If <b>total_out</b> is provided, set it to + * the sum of all elements in the array _before_ scaling. */ +STATIC void +scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in, + int n_entries, + uint64_t *total_out) +{ + double total = 0.0; + double scale_factor = 0.0; + int i; + + for (i = 0; i < n_entries; ++i) + total += entries_in[i]; + + if (total > 0.0) { + scale_factor = ((double)INT64_MAX) / total; + scale_factor /= 4.0; /* make sure we're very far away from overflowing */ + } + + for (i = 0; i < n_entries; ++i) + entries_out[i] = tor_llround(entries_in[i] * scale_factor); + + if (total_out) + *total_out = (uint64_t) total; +} + +/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>, + * choosing each element with a probability proportional to its (uint64_t) + * value, and return the index of that element. If all elements are 0, choose + * an index at random. Return -1 on error. + */ +STATIC int +choose_array_element_by_weight(const uint64_t *entries, int n_entries) +{ + int i; + uint64_t rand_val; + uint64_t total = 0; + + for (i = 0; i < n_entries; ++i) + total += entries[i]; + + if (n_entries < 1) + return -1; + + if (total == 0) + return crypto_rand_int(n_entries); + + tor_assert(total < INT64_MAX); + + rand_val = crypto_rand_uint64(total); + + return select_array_member_cumulative_timei( + entries, n_entries, total, rand_val); +} + +/** Return bw*1000, unless bw*1000 would overflow, in which case return + * INT32_MAX. */ +static inline int32_t +kb_to_bytes(uint32_t bw) +{ + return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000; +} + +/** Helper function: + * choose a random element of smartlist <b>sl</b> of nodes, weighted by + * the advertised bandwidth of each element using the consensus + * bandwidth weights. + * + * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all + * nodes' bandwidth equally regardless of their Exit status, since there may + * be some in the list because they exit to obscure ports. If + * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight + * exit-node's bandwidth less depending on the smallness of the fraction of + * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a + * guard node: consider all guard's bandwidth equally. Otherwise, weight + * guards proportionally less. + */ +static const node_t * +smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, + bandwidth_weight_rule_t rule) +{ + double *bandwidths_dbl=NULL; + uint64_t *bandwidths_u64=NULL; + + if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0) + return NULL; + + bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t)); + scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl, + smartlist_len(sl), NULL); + + { + int idx = choose_array_element_by_weight(bandwidths_u64, + smartlist_len(sl)); + tor_free(bandwidths_dbl); + tor_free(bandwidths_u64); + return idx < 0 ? NULL : smartlist_get(sl, idx); + } +} + +/** When weighting bridges, enforce these values as lower and upper + * bound for believable bandwidth, because there is no way for us + * to verify a bridge's bandwidth currently. */ +#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */ +#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */ + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity, making sure to stay within the + * interval between bridge-min-believe-bw and + * bridge-max-believe-bw. */ +static uint32_t +bridge_get_advertised_bandwidth_bounded(routerinfo_t *router) +{ + uint32_t result = router->bandwidthcapacity; + if (result > router->bandwidthrate) + result = router->bandwidthrate; + if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH; + else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH; + return result; +} + +/** Given a list of routers and a weighting rule as in + * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth + * values for each node and store them in a freshly allocated + * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results + * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total + * of all the bandwidths. + * Return 0 on success, -1 on failure. */ +static int +compute_weighted_bandwidths(const smartlist_t *sl, + bandwidth_weight_rule_t rule, + double **bandwidths_out, + double *total_bandwidth_out) +{ + int64_t weight_scale; + double Wg = -1, Wm = -1, We = -1, Wd = -1; + double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; + guardfraction_bandwidth_t guardfraction_bw; + double *bandwidths = NULL; + double total_bandwidth = 0.0; + + tor_assert(sl); + tor_assert(bandwidths_out); + + /* Can't choose exit and guard at same time */ + tor_assert(rule == NO_WEIGHTING || + rule == WEIGHT_FOR_EXIT || + rule == WEIGHT_FOR_GUARD || + rule == WEIGHT_FOR_MID || + rule == WEIGHT_FOR_DIR); + + *bandwidths_out = NULL; + + if (total_bandwidth_out) { + *total_bandwidth_out = 0.0; + } + + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to consensus weight node " + "selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return -1; + } + + weight_scale = networkstatus_get_weight_scale_param(NULL); + + if (rule == WEIGHT_FOR_GUARD) { + Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ + We = 0; + Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_MID) { + Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); + We = networkstatus_get_bw_weight(NULL, "Wme", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_EXIT) { + // Guards CAN be exits if they have weird exit policies + // They are d then I guess... + We = networkstatus_get_bw_weight(NULL, "Wee", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ + Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); + Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_DIR) { + We = networkstatus_get_bw_weight(NULL, "Wbe", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); + Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); + + Wgb = Wmb = Web = Wdb = weight_scale; + } else if (rule == NO_WEIGHTING) { + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 + || Web < 0) { + log_debug(LD_CIRC, + "Got negative bandwidth weights. Defaulting to naive selection" + " algorithm."); + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + Wg /= weight_scale; + Wm /= weight_scale; + We /= weight_scale; + Wd /= weight_scale; + + Wgb /= weight_scale; + Wmb /= weight_scale; + Web /= weight_scale; + Wdb /= weight_scale; + + bandwidths = tor_calloc(smartlist_len(sl), sizeof(double)); + + // Cycle through smartlist and total the bandwidth. + static int warned_missing_bw = 0; + SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; + double weight = 1; + double weight_without_guard_flag = 0; /* Used for guardfraction */ + double final_weight = 0; + is_exit = node->is_exit && ! node->is_bad_exit; + is_guard = node->is_possible_guard; + is_dir = node_is_dir(node); + if (node->rs) { + if (!node->rs->has_bandwidth) { + /* This should never happen, unless all the authorities downgrade + * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ + if (! warned_missing_bw) { + log_warn(LD_BUG, + "Consensus is missing some bandwidths. Using a naive " + "router selection algorithm"); + warned_missing_bw = 1; + } + this_bw = 30000; /* Chosen arbitrarily */ + } else { + this_bw = kb_to_bytes(node->rs->bandwidth_kb); + } + } else if (node->ri) { + /* bridge or other descriptor not in our consensus */ + this_bw = bridge_get_advertised_bandwidth_bounded(node->ri); + } else { + /* We can't use this one. */ + continue; + } + + if (is_guard && is_exit) { + weight = (is_dir ? Wdb*Wd : Wd); + weight_without_guard_flag = (is_dir ? Web*We : We); + } else if (is_guard) { + weight = (is_dir ? Wgb*Wg : Wg); + weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm); + } else if (is_exit) { + weight = (is_dir ? Web*We : We); + } else { // middle + weight = (is_dir ? Wmb*Wm : Wm); + } + /* These should be impossible; but overflows here would be bad, so let's + * make sure. */ + if (this_bw < 0) + this_bw = 0; + if (weight < 0.0) + weight = 0.0; + if (weight_without_guard_flag < 0.0) + weight_without_guard_flag = 0.0; + + /* If guardfraction information is available in the consensus, we + * want to calculate this router's bandwidth according to its + * guardfraction. Quoting from proposal236: + * + * Let Wpf denote the weight from the 'bandwidth-weights' line a + * client would apply to N for position p if it had the guard + * flag, Wpn the weight if it did not have the guard flag, and B the + * measured bandwidth of N in the consensus. Then instead of choosing + * N for position p proportionally to Wpf*B or Wpn*B, clients should + * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. + */ + if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) { + /* XXX The assert should actually check for is_guard. However, + * that crashes dirauths because of #13297. This should be + * equivalent: */ + tor_assert(node->rs->is_possible_guard); + + guard_get_guardfraction_bandwidth(&guardfraction_bw, + this_bw, + node->rs->guardfraction_percentage); + + /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */ + final_weight = + guardfraction_bw.guard_bw * weight + + guardfraction_bw.non_guard_bw * weight_without_guard_flag; + + log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)", + node->rs->nickname, final_weight, weight*this_bw, + bandwidth_weight_rule_to_string(rule)); + } else { /* no guardfraction information. calculate the weight normally. */ + final_weight = weight*this_bw; + } + + bandwidths[node_sl_idx] = final_weight; + total_bandwidth += final_weight; + } SMARTLIST_FOREACH_END(node); + + log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " + "on weights " + "Wg=%f Wm=%f We=%f Wd=%f with total bw %f", + bandwidth_weight_rule_to_string(rule), + Wg, Wm, We, Wd, total_bandwidth); + + *bandwidths_out = bandwidths; + + if (total_bandwidth_out) { + *total_bandwidth_out = total_bandwidth; + } + + return 0; +} + +/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted + * by their weighted bandwidths with rule <b>rule</b>, for which we have + * descriptors. + * + * If <b>for_direct_connect</b> is true, we intend to connect to the node + * directly, as the first hop of a circuit; otherwise, we intend to connect + * to it indirectly, or use it as if we were connecting to it indirectly. */ +double +frac_nodes_with_descriptors(const smartlist_t *sl, + bandwidth_weight_rule_t rule, + int for_direct_conn) +{ + double *bandwidths = NULL; + double total, present; + + if (smartlist_len(sl) == 0) + return 0.0; + + if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 || + total <= 0.0) { + int n_with_descs = 0; + SMARTLIST_FOREACH(sl, const node_t *, node, { + if (node_has_preferred_descriptor(node, for_direct_conn)) + n_with_descs++; + }); + tor_free(bandwidths); + return ((double)n_with_descs) / smartlist_len(sl); + } + + present = 0.0; + SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { + if (node_has_preferred_descriptor(node, for_direct_conn)) + present += bandwidths[node_sl_idx]; + } SMARTLIST_FOREACH_END(node); + + tor_free(bandwidths); + + return present / total; +} + +/** Choose a random element of status list <b>sl</b>, weighted by + * the advertised bandwidth of each node */ +const node_t * +node_sl_choose_by_bandwidth(const smartlist_t *sl, + bandwidth_weight_rule_t rule) +{ /*XXXX MOVE */ + return smartlist_choose_node_by_bandwidth_weights(sl, rule); +} + +/** Given a <b>router</b>, add every node_t in its family (including the + * node itself!) to <b>sl</b>. + * + * Note the type mismatch: This function takes a routerinfo, but adds nodes + * to the smartlist! + */ +static void +routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) +{ + /* XXXX MOVE ? */ + node_t fake_node; + const node_t *node = node_get_by_id(router->cache_info.identity_digest); + if (node == NULL) { + memset(&fake_node, 0, sizeof(fake_node)); + fake_node.ri = (routerinfo_t *)router; + memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN); + node = &fake_node; + } + nodelist_add_node_and_family(sl, node); +} + +/** Return a random running node from the nodelist. Never + * pick a node that is in + * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>, + * even if they are the only nodes available. + * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than + * a minimum uptime, return one of those. + * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the + * advertised capacity of each router. + * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers. + * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if + * picking an exit node, otherwise we weight bandwidths for picking a relay + * node (that is, possibly discounting exit nodes). + * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that + * have a routerinfo or microdescriptor -- that is, enough info to be + * used to build a circuit. + * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that + * have an address that is preferred by the ClientPreferIPv6ORPort setting + * (regardless of this flag, we exclude nodes that aren't allowed by the + * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0). + */ +const node_t * +router_choose_random_node(smartlist_t *excludedsmartlist, + routerset_t *excludedset, + router_crn_flags_t flags) +{ /* XXXX MOVE */ + const int need_uptime = (flags & CRN_NEED_UPTIME) != 0; + const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; + const int need_guard = (flags & CRN_NEED_GUARD) != 0; + const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; + const int need_desc = (flags & CRN_NEED_DESC) != 0; + const int pref_addr = (flags & CRN_PREF_ADDR) != 0; + const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; + const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0; + + smartlist_t *sl=smartlist_new(), + *excludednodes=smartlist_new(); + const node_t *choice = NULL; + const routerinfo_t *r; + bandwidth_weight_rule_t rule; + + tor_assert(!(weight_for_exit && need_guard)); + rule = weight_for_exit ? WEIGHT_FOR_EXIT : + (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); + + SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + if (node_allows_single_hop_exits(node)) { + /* Exclude relays that allow single hop exit circuits. This is an + * obsolete option since 0.2.9.2-alpha and done by default in + * 0.3.1.0-alpha. */ + smartlist_add(excludednodes, node); + } else if (rendezvous_v3 && + !node_supports_v3_rendezvous_point(node)) { + /* Exclude relays that do not support to rendezvous for a hidden service + * version 3. */ + smartlist_add(excludednodes, node); + } + } SMARTLIST_FOREACH_END(node); + + /* If the node_t is not found we won't be to exclude ourself but we + * won't be able to pick ourself in router_choose_random_node() so + * this is fine to at least try with our routerinfo_t object. */ + if ((r = router_get_my_routerinfo())) + routerlist_add_node_and_family(excludednodes, r); + + router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity, + need_guard, need_desc, pref_addr, + direct_conn); + log_debug(LD_CIRC, + "We found %d running nodes.", + smartlist_len(sl)); + + smartlist_subtract(sl,excludednodes); + log_debug(LD_CIRC, + "We removed %d excludednodes, leaving %d nodes.", + smartlist_len(excludednodes), + smartlist_len(sl)); + + if (excludedsmartlist) { + smartlist_subtract(sl,excludedsmartlist); + log_debug(LD_CIRC, + "We removed %d excludedsmartlist, leaving %d nodes.", + smartlist_len(excludedsmartlist), + smartlist_len(sl)); + } + if (excludedset) { + routerset_subtract_nodes(sl,excludedset); + log_debug(LD_CIRC, + "We removed excludedset, leaving %d nodes.", + smartlist_len(sl)); + } + + // Always weight by bandwidth + choice = node_sl_choose_by_bandwidth(sl, rule); + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD| + CRN_PREF_ADDR); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); + } + smartlist_free(excludednodes); + if (!choice) { + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); + } + return choice; +} + +/** Try to find a running directory authority. Flags are as for + * router_pick_directory_server. + */ +const routerstatus_t * +router_pick_trusteddirserver(dirinfo_type_t type, int flags) +{ + return router_pick_dirserver_generic( + router_get_trusted_dir_servers_mutable(), + type, flags); +} + +/** Try to find a running fallback directory. Flags are as for + * router_pick_directory_server. + */ +const routerstatus_t * +router_pick_fallback_dirserver(dirinfo_type_t type, int flags) +{ + return router_pick_dirserver_generic( + router_get_fallback_dir_servers_mutable(), + type, flags); +} + +/** Pick a random element from a list of dir_server_t, weighting by their + * <b>weight</b> field. */ +static const dir_server_t * +dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight) +{ + int n = smartlist_len(servers); + int i; + double *weights_dbl; + uint64_t *weights_u64; + const dir_server_t *ds; + + weights_dbl = tor_calloc(n, sizeof(double)); + weights_u64 = tor_calloc(n, sizeof(uint64_t)); + for (i = 0; i < n; ++i) { + ds = smartlist_get(servers, i); + weights_dbl[i] = ds->weight; + if (ds->is_authority) + weights_dbl[i] *= authority_weight; + } + + scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL); + i = choose_array_element_by_weight(weights_u64, n); + tor_free(weights_dbl); + tor_free(weights_u64); + return (i < 0) ? NULL : smartlist_get(servers, i); +} + +/** Choose randomly from among the dir_server_ts in sourcelist that + * are up. Flags are as for router_pick_directory_server_impl(). + */ +static const routerstatus_t * +router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, + dirinfo_type_t type, int flags, + int *n_busy_out) +{ + const or_options_t *options = get_options(); + smartlist_t *direct, *tunnel; + smartlist_t *overloaded_direct, *overloaded_tunnel; + const routerinfo_t *me = router_get_my_routerinfo(); + const routerstatus_t *result = NULL; + time_t now = time(NULL); + const int requireother = ! (flags & PDS_ALLOW_SELF); + const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); + const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); + const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH); + const double auth_weight = + (sourcelist == router_get_fallback_dir_servers()) ? + options->DirAuthorityFallbackRate : 1.0; + smartlist_t *pick_from; + int n_busy = 0; + int try_excluding = 1, n_excluded = 0; + int try_ip_pref = 1; + + if (!sourcelist) + return NULL; + + retry_search: + + direct = smartlist_new(); + tunnel = smartlist_new(); + overloaded_direct = smartlist_new(); + overloaded_tunnel = smartlist_new(); + + const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref); + const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref); + const int must_have_or = directory_must_use_begindir(options); + + SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d) + { + int is_overloaded = + d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now; + if (!d->is_running) continue; + if ((type & d->type) == 0) + continue; + + SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest); + + if (requireother && me && router_digest_is_me(d->digest)) + continue; + if (try_excluding && + routerset_contains_routerstatus(options->ExcludeNodes, + &d->fake_status, -1)) { + ++n_excluded; + continue; + } + + if (router_is_already_dir_fetching_(d->addr, + &d->ipv6_addr, + d->dir_port, + no_serverdesc_fetching, + no_microdesc_fetching)) { + ++n_busy; + continue; + } + + /* Clients use IPv6 addresses if the server has one and the client + * prefers IPv6. + * Add the router if its preferred address and port are reachable. + * If we don't get any routers, we'll try again with the non-preferred + * address for each router (if any). (To ensure correct load-balancing + * we try routers that only have one address both times.) + */ + if (!fascistfirewall || skip_or_fw || + fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION, + try_ip_pref)) + smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d); + else if (!must_have_or && (skip_dir_fw || + fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION, + try_ip_pref))) + smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d); + } + SMARTLIST_FOREACH_END(d); + + if (smartlist_len(tunnel)) { + pick_from = tunnel; + } else if (smartlist_len(overloaded_tunnel)) { + pick_from = overloaded_tunnel; + } else if (smartlist_len(direct)) { + pick_from = direct; + } else { + pick_from = overloaded_direct; + } + + { + const dir_server_t *selection = + dirserver_choose_by_weight(pick_from, auth_weight); + + if (selection) + result = &selection->fake_status; + } + + smartlist_free(direct); + smartlist_free(tunnel); + smartlist_free(overloaded_direct); + smartlist_free(overloaded_tunnel); + + RETRY_ALTERNATE_IP_VERSION(retry_search); + + RETRY_WITHOUT_EXCLUDE(retry_search); + + router_picked_poor_directory_log(result); + + if (n_busy_out) + *n_busy_out = n_busy; + return result; +} diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h new file mode 100644 index 000000000..05dabd123 --- /dev/null +++ b/src/feature/nodelist/node_select.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file node_select.h + * \brief Header file for node_select.c + **/ + +#ifndef TOR_NODE_SELECT_H +#define TOR_NODE_SELECT_H + +/** Flags to be passed to control router_choose_random_node() to indicate what + * kind of nodes to pick according to what algorithm. */ +typedef enum router_crn_flags_t { + CRN_NEED_UPTIME = 1<<0, + CRN_NEED_CAPACITY = 1<<1, + CRN_NEED_GUARD = 1<<2, + /* XXXX not used, apparently. */ + CRN_WEIGHT_AS_EXIT = 1<<5, + CRN_NEED_DESC = 1<<6, + /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */ + CRN_PREF_ADDR = 1<<7, + /* On clients, only provide nodes that we can connect to directly, based on + * our firewall rules */ + CRN_DIRECT_CONN = 1<<8, + /* On clients, only provide nodes with HSRend >= 2 protocol version which + * is required for hidden service version >= 3. */ + CRN_RENDEZVOUS_V3 = 1<<9, +} router_crn_flags_t; + +/** Possible ways to weight routers when choosing one randomly. See + * routerlist_sl_choose_by_bandwidth() for more information.*/ +typedef enum bandwidth_weight_rule_t { + NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, + WEIGHT_FOR_DIR +} bandwidth_weight_rule_t; + +/* Flags for pick_directory_server() and pick_trusteddirserver(). */ +/** Flag to indicate that we should not automatically be willing to use + * ourself to answer a directory request. + * Passed to router_pick_directory_server (et al).*/ +#define PDS_ALLOW_SELF (1<<0) +/** Flag to indicate that if no servers seem to be up, we should mark all + * directory servers as up and try again. + * Passed to router_pick_directory_server (et al).*/ +#define PDS_RETRY_IF_NO_SERVERS (1<<1) +/** Flag to indicate that we should not exclude directory servers that + * our ReachableAddress settings would exclude. This usually means that + * we're going to connect to the server over Tor, and so we don't need to + * worry about our firewall telling us we can't. + * Passed to router_pick_directory_server (et al).*/ +#define PDS_IGNORE_FASCISTFIREWALL (1<<2) +/** Flag to indicate that we should not use any directory authority to which + * we have an existing directory connection for downloading server descriptors + * or extrainfo documents. + * + * Passed to router_pick_directory_server (et al) + */ +#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) +/** Flag to indicate that we should not use any directory authority to which + * we have an existing directory connection for downloading microdescs. + * + * Passed to router_pick_directory_server (et al) + */ +#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4) + +const routerstatus_t *router_pick_directory_server(dirinfo_type_t type, + int flags); + +int router_get_my_share_of_directory_requests(double *v3_share_out); + +const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl, + bandwidth_weight_rule_t rule); +double frac_nodes_with_descriptors(const smartlist_t *sl, + bandwidth_weight_rule_t rule, + int for_direct_conn); +const node_t *router_choose_random_node(smartlist_t *excludedsmartlist, + struct routerset_t *excludedset, + router_crn_flags_t flags); + +const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type, + int flags); +const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, + int flags); + +#ifdef NODE_SELECT_PRIVATE +STATIC int choose_array_element_by_weight(const uint64_t *entries, + int n_entries); +STATIC void scale_array_elements_to_u64(uint64_t *entries_out, + const double *entries_in, + int n_entries, + uint64_t *total_out); +STATIC const routerstatus_t *router_pick_directory_server_impl( + dirinfo_type_t auth, int flags, + int *n_busy_out); +STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap, + int serverdesc, int microdesc); +#endif + +#endif diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 50dc8f7d3..c990e6940 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -59,6 +59,8 @@ #include "core/or/protover.h" #include "feature/rend/rendservice.h" #include "feature/relay/router.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index bcc5c1f07..547b54bb9 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -57,78 +57,35 @@ * nodes according to different rules and weights. Historically, they * were all in this module. Now, they are spread across this module, * nodelist.c, and networkstatus.c. (TODO: Fix that.) - * - * <br> - * - * (For historical reasons) this module also contains code for handling - * the list of fallback directories, the list of directory authorities, - * and the list of authority certificates. - * - * For the directory authorities, we have a list containing the public - * identity key, and contact points, for each authority. The - * authorities receive descriptors from relays, and publish consensuses, - * descriptors, and microdescriptors. This list is pre-configured. - * - * Fallback directories are well-known, stable, but untrusted directory - * caches that clients which have not yet bootstrapped can use to get - * their first networkstatus consensus, in order to find out where the - * Tor network really is. This list is pre-configured in - * fallback_dirs.inc. Every authority also serves as a fallback. - * - * Both fallback directories and directory authorities are are - * represented by a dir_server_t. - * - * Authority certificates are signed with authority identity keys; they - * are used to authenticate shorter-term authority signing keys. We - * fetch them when we find a consensus or a vote that has been signed - * with a signing key we don't recognize. We cache them on disk and - * load them on startup. Authority operators generate them with the - * "tor-gencert" utility. - * - * TODO: Authority certificates should be a separate module. - * - * TODO: dir_server_t stuff should be in a separate module. **/

#define ROUTERLIST_PRIVATE #include "core/or/or.h" -#include "lib/err/backtrace.h" -#include "feature/client/bridges.h" -#include "lib/crypt_ops/crypto_ed25519.h" -#include "lib/crypt_ops/crypto_format.h" -#include "core/or/circuitstats.h" + #include "app/config/config.h" #include "core/mainloop/connection.h" +#include "core/mainloop/main.h" +#include "core/or/policies.h" +#include "feature/client/bridges.h" #include "feature/control/control.h" -#include "lib/crypt_ops/crypto_rand.h" +#include "feature/dirauth/mode.h" #include "feature/dircache/directory.h" #include "feature/dircache/dirserv.h" -#include "feature/client/entrynodes.h" -#include "feature/dircommon/fp_pair.h" -#include "feature/stats/geoip.h" -#include "feature/hibernate/hibernate.h" -#include "core/mainloop/main.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" -#include "core/or/policies.h" -#include "core/or/reasons.h" -#include "feature/rend/rendcommon.h" -#include "feature/rend/rendservice.h" -#include "feature/stats/rephist.h" -#include "feature/relay/router.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" -#include "lib/sandbox/sandbox.h" #include "feature/nodelist/torcert.h" -#include "lib/math/fp.h" -#include "lib/net/resolve.h" - -#include "feature/dirauth/dirvote.h" -#include "feature/dirauth/mode.h" +#include "feature/relay/router.h" +#include "feature/stats/rephist.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_rand.h"

-#include "feature/nodelist/authority_cert_st.h" #include "feature/dircommon/dir_connection_st.h" #include "feature/dirclient/dir_server_st.h" #include "feature/nodelist/document_signature_st.h" @@ -156,7 +113,6 @@ DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t) DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t) DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t) -DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t) #define SDMAP_FOREACH(map, keyvar, valvar) \ DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \ valvar) @@ -164,1164 +120,31 @@ DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t) DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar) #define EIMAP_FOREACH(map, keyvar, valvar) \ DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar) -#define DSMAP_FOREACH(map, keyvar, valvar) \ - DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \ - valvar) #define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn)) #define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn)) -#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn)) #define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn))

-/* Forward declaration for cert_list_t */ -typedef struct cert_list_t cert_list_t; - /* static function prototypes */ -static int compute_weighted_bandwidths(const smartlist_t *sl, - bandwidth_weight_rule_t rule, - double **bandwidths_out, - double *total_bandwidth_out); -static const routerstatus_t *router_pick_trusteddirserver_impl( - const smartlist_t *sourcelist, dirinfo_type_t auth, - int flags, int *n_busy_out); -static const routerstatus_t *router_pick_dirserver_generic( - smartlist_t *sourcelist, - dirinfo_type_t type, int flags); -static void mark_all_dirservers_up(smartlist_t *server_list); static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); -static const char *signed_descriptor_get_body_impl( - const signed_descriptor_t *desc, - int with_annotations); -static void list_pending_downloads(digestmap_t *result, - digest256map_t *result256, - int purpose, const char *prefix); -static void list_pending_fpsk_downloads(fp_pair_map_t *result); -static void launch_dummy_descriptor_download_as_needed(time_t now, - const or_options_t *options); -static void download_status_reset_by_sk_in_cl(cert_list_t *cl, - const char *digest); -static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl, - const char *digest, - time_t now); - -/****************************************************************************/ - -/** Global list of a dir_server_t object for each directory - * authority. */ -static smartlist_t *trusted_dir_servers = NULL; -/** Global list of dir_server_t objects for all directory authorities - * and all fallback directory servers. */ -static smartlist_t *fallback_dir_servers = NULL; - -/** List of certificates for a single authority, and download status for - * latest certificate. - */ -struct cert_list_t { - /* - * The keys of download status map are cert->signing_key_digest for pending - * downloads by (identity digest/signing key digest) pair; functions such - * as authority_cert_get_by_digest() already assume these are unique. - */ - struct digest_ds_map_t *dl_status_map; - /* There is also a dlstatus for the download by identity key only */ - download_status_t dl_status_by_id; - smartlist_t *certs; -}; -/** Map from v3 identity key digest to cert_list_t. */ -static digestmap_t *trusted_dir_certs = NULL; -/** True iff any key certificate in at least one member of - * <b>trusted_dir_certs</b> has changed since we last flushed the - * certificates to disk. */ -static int trusted_dir_servers_certs_changed = 0; - -/** Global list of all of the routers that we know about. */ -static routerlist_t *routerlist = NULL; - -/** List of strings for nicknames we've already warned about and that are - * still unknown / unavailable. */ -static smartlist_t *warned_nicknames = NULL; - -/** The last time we tried to download any routerdesc, or 0 for "never". We - * use this to rate-limit download attempts when the number of routerdescs to - * download is low. */ -static time_t last_descriptor_download_attempted = 0; - -/** Return the number of directory authorities whose type matches some bit set - * in <b>type</b> */ -int -get_n_authorities(dirinfo_type_t type) -{ - int n = 0; - if (!trusted_dir_servers) - return 0; - SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, - if (ds->type & type) - ++n); - return n; -} - -/** Initialise schedule, want_authority, and increment_on in the download - * status dlstatus, then call download_status_reset() on it. - * It is safe to call this function or download_status_reset() multiple times - * on a new dlstatus. But it should *not* be called after a dlstatus has been - * used to count download attempts or failures. */ -static void -download_status_cert_init(download_status_t *dlstatus) -{ - dlstatus->schedule = DL_SCHED_CONSENSUS; - dlstatus->want_authority = DL_WANT_ANY_DIRSERVER; - dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE; - dlstatus->last_backoff_position = 0; - dlstatus->last_delay_used = 0; - - /* Use the new schedule to set next_attempt_at */ - download_status_reset(dlstatus); -} - -/** Reset the download status of a specified element in a dsmap */ -static void -download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest) -{ - download_status_t *dlstatus = NULL; - - tor_assert(cl); - tor_assert(digest); - - /* Make sure we have a dsmap */ - if (!(cl->dl_status_map)) { - cl->dl_status_map = dsmap_new(); - } - /* Look for a download_status_t in the map with this digest */ - dlstatus = dsmap_get(cl->dl_status_map, digest); - /* Got one? */ - if (!dlstatus) { - /* Insert before we reset */ - dlstatus = tor_malloc_zero(sizeof(*dlstatus)); - dsmap_set(cl->dl_status_map, digest, dlstatus); - download_status_cert_init(dlstatus); - } - tor_assert(dlstatus); - /* Go ahead and reset it */ - download_status_reset(dlstatus); -} - -/** - * Return true if the download for this signing key digest in cl is ready - * to be re-attempted. - */ -static int -download_status_is_ready_by_sk_in_cl(cert_list_t *cl, - const char *digest, - time_t now) -{ - int rv = 0; - download_status_t *dlstatus = NULL; - - tor_assert(cl); - tor_assert(digest); - - /* Make sure we have a dsmap */ - if (!(cl->dl_status_map)) { - cl->dl_status_map = dsmap_new(); - } - /* Look for a download_status_t in the map with this digest */ - dlstatus = dsmap_get(cl->dl_status_map, digest); - /* Got one? */ - if (dlstatus) { - /* Use download_status_is_ready() */ - rv = download_status_is_ready(dlstatus, now); - } else { - /* - * If we don't know anything about it, return 1, since we haven't - * tried this one before. We need to create a new entry here, - * too. - */ - dlstatus = tor_malloc_zero(sizeof(*dlstatus)); - download_status_cert_init(dlstatus); - dsmap_set(cl->dl_status_map, digest, dlstatus); - rv = 1; - } - - return rv; -} - -/** Helper: Return the cert_list_t for an authority whose authority ID is - * <b>id_digest</b>, allocating a new list if necessary. */ -static cert_list_t * -get_cert_list(const char *id_digest) -{ - cert_list_t *cl; - if (!trusted_dir_certs) - trusted_dir_certs = digestmap_new(); - cl = digestmap_get(trusted_dir_certs, id_digest); - if (!cl) { - cl = tor_malloc_zero(sizeof(cert_list_t)); - download_status_cert_init(&cl->dl_status_by_id); - cl->certs = smartlist_new(); - cl->dl_status_map = dsmap_new(); - digestmap_set(trusted_dir_certs, id_digest, cl); - } - return cl; -} - -/** Return a list of authority ID digests with potentially enumerable lists - * of download_status_t objects; used by controller GETINFO queries. - */ - -MOCK_IMPL(smartlist_t *, -list_authority_ids_with_downloads, (void)) -{ - smartlist_t *ids = smartlist_new(); - digestmap_iter_t *i; - const char *digest; - char *tmp; - void *cl; - - if (trusted_dir_certs) { - for (i = digestmap_iter_init(trusted_dir_certs); - !(digestmap_iter_done(i)); - i = digestmap_iter_next(trusted_dir_certs, i)) { - /* - * We always have at least dl_status_by_id to query, so no need to - * probe deeper than the existence of a cert_list_t. - */ - digestmap_iter_get(i, &digest, &cl); - tmp = tor_malloc(DIGEST_LEN); - memcpy(tmp, digest, DIGEST_LEN); - smartlist_add(ids, tmp); - } - } - /* else definitely no downloads going since nothing even has a cert list */ - - return ids; -} - -/** Given an authority ID digest, return a pointer to the default download - * status, or NULL if there is no such entry in trusted_dir_certs */ - -MOCK_IMPL(download_status_t *, -id_only_download_status_for_authority_id, (const char *digest)) -{ - download_status_t *dl = NULL; - cert_list_t *cl; - - if (trusted_dir_certs) { - cl = digestmap_get(trusted_dir_certs, digest); - if (cl) { - dl = &(cl->dl_status_by_id); - } - } - - return dl; -} - -/** Given an authority ID digest, return a smartlist of signing key digests - * for which download_status_t is potentially queryable, or NULL if no such - * authority ID digest is known. */ - -MOCK_IMPL(smartlist_t *, -list_sk_digests_for_authority_id, (const char *digest)) -{ - smartlist_t *sks = NULL; - cert_list_t *cl; - dsmap_iter_t *i; - const char *sk_digest; - char *tmp; - download_status_t *dl; - - if (trusted_dir_certs) { - cl = digestmap_get(trusted_dir_certs, digest); - if (cl) { - sks = smartlist_new(); - if (cl->dl_status_map) { - for (i = dsmap_iter_init(cl->dl_status_map); - !(dsmap_iter_done(i)); - i = dsmap_iter_next(cl->dl_status_map, i)) { - /* Pull the digest out and add it to the list */ - dsmap_iter_get(i, &sk_digest, &dl); - tmp = tor_malloc(DIGEST_LEN); - memcpy(tmp, sk_digest, DIGEST_LEN); - smartlist_add(sks, tmp); - } - } - } - } - - return sks; -} - -/** Given an authority ID digest and a signing key digest, return the - * download_status_t or NULL if none exists. */ - -MOCK_IMPL(download_status_t *, -download_status_for_authority_id_and_sk,(const char *id_digest, - const char *sk_digest)) -{ - download_status_t *dl = NULL; - cert_list_t *cl = NULL; - - if (trusted_dir_certs) { - cl = digestmap_get(trusted_dir_certs, id_digest); - if (cl && cl->dl_status_map) { - dl = dsmap_get(cl->dl_status_map, sk_digest); - } - } - - return dl; -} - -#define cert_list_free(val) \ - FREE_AND_NULL(cert_list_t, cert_list_free_, (val)) - -/** Release all space held by a cert_list_t */ -static void -cert_list_free_(cert_list_t *cl) -{ - if (!cl) - return; - - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - authority_cert_free(cert)); - smartlist_free(cl->certs); - dsmap_free(cl->dl_status_map, tor_free_); - tor_free(cl); -} - -/** Wrapper for cert_list_free so we can pass it to digestmap_free */ -static void -cert_list_free_void(void *cl) -{ - cert_list_free_(cl); -} - -/** Reload the cached v3 key certificates from the cached-certs file in - * the data directory. Return 0 on success, -1 on failure. */ -int -trusted_dirs_reload_certs(void) -{ - char *filename; - char *contents; - int r; - - filename = get_cachedir_fname("cached-certs"); - contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - tor_free(filename); - if (!contents) - return 0; - r = trusted_dirs_load_certs_from_string( - contents, - TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL); - tor_free(contents); - return r; -} - -/** Helper: return true iff we already have loaded the exact cert - * <b>cert</b>. */ -static inline int -already_have_cert(authority_cert_t *cert) -{ - cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest); - - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c, - { - if (tor_memeq(c->cache_info.signed_descriptor_digest, - cert->cache_info.signed_descriptor_digest, - DIGEST_LEN)) - return 1; - }); - return 0; -} - -/** Load a bunch of new key certificates from the string <b>contents</b>. If - * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are - * from the cache, and we don't need to flush them to disk. If we are a - * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF. - * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST - * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST. If <b>flush</b> is true, we - * need to flush any changed certificates to disk now. Return 0 on success, - * -1 if any certs fail to parse. - * - * If source_dir is non-NULL, it's the identity digest for a directory that - * we've just successfully retrieved certificates from, so try it first to - * fetch any missing certificates. - */ -int -trusted_dirs_load_certs_from_string(const char *contents, int source, - int flush, const char *source_dir) -{ - dir_server_t *ds; - const char *s, *eos; - int failure_code = 0; - int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE); - int added_trusted_cert = 0; - - for (s = contents; *s; s = eos) { - authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); - cert_list_t *cl; - if (!cert) { - failure_code = -1; - break; - } - ds = trusteddirserver_get_by_v3_auth_digest( - cert->cache_info.identity_digest); - log_debug(LD_DIR, "Parsed certificate for %s", - ds ? ds->nickname : "unknown authority"); - - if (already_have_cert(cert)) { - /* we already have this one. continue. */ - log_info(LD_DIR, "Skipping %s certificate for %s that we " - "already have.", - from_store ? "cached" : "downloaded", - ds ? ds->nickname : "an old or new authority"); - - /* - * A duplicate on download should be treated as a failure, so we call - * authority_cert_dl_failed() to reset the download status to make sure - * we can't try again. Since we've implemented the fp-sk mechanism - * to download certs by signing key, this should be much rarer than it - * was and is perhaps cause for concern. - */ - if (!from_store) { - if (authdir_mode(get_options())) { - log_warn(LD_DIR, - "Got a certificate for %s, but we already have it. " - "Maybe they haven't updated it. Waiting for a while.", - ds ? ds->nickname : "an old or new authority"); - } else { - log_info(LD_DIR, - "Got a certificate for %s, but we already have it. " - "Maybe they haven't updated it. Waiting for a while.", - ds ? ds->nickname : "an old or new authority"); - } - - /* - * This is where we care about the source; authority_cert_dl_failed() - * needs to know whether the download was by fp or (fp,sk) pair to - * twiddle the right bit in the download map. - */ - if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) { - authority_cert_dl_failed(cert->cache_info.identity_digest, - NULL, 404); - } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) { - authority_cert_dl_failed(cert->cache_info.identity_digest, - cert->signing_key_digest, 404); - } - } - - authority_cert_free(cert); - continue; - } - - if (ds) { - added_trusted_cert = 1; - log_info(LD_DIR, "Adding %s certificate for directory authority %s with " - "signing key %s", from_store ? "cached" : "downloaded", - ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN)); - } else { - int adding = we_want_to_fetch_unknown_auth_certs(get_options()); - log_info(LD_DIR, "%s %s certificate for unrecognized directory " - "authority with signing key %s", - adding ? "Adding" : "Not adding", - from_store ? "cached" : "downloaded", - hex_str(cert->signing_key_digest,DIGEST_LEN)); - if (!adding) { - authority_cert_free(cert); - continue; - } - } - - cl = get_cert_list(cert->cache_info.identity_digest); - smartlist_add(cl->certs, cert); - if (ds && cert->cache_info.published_on > ds->addr_current_at) { - /* Check to see whether we should update our view of the authority's - * address. */ - if (cert->addr && cert->dir_port && - (ds->addr != cert->addr || - ds->dir_port != cert->dir_port)) { - char *a = tor_dup_ip(cert->addr); - log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on certificate.", - ds->nickname, ds->address, (int)ds->dir_port, - a, cert->dir_port); - tor_free(a); - ds->addr = cert->addr; - ds->dir_port = cert->dir_port; - } - ds->addr_current_at = cert->cache_info.published_on; - } - - if (!from_store) - trusted_dir_servers_certs_changed = 1; - } - - if (flush) - trusted_dirs_flush_certs_to_disk(); - - /* call this even if failure_code is <0, since some certs might have - * succeeded, but only pass source_dir if there were no failures, - * and at least one more authority certificate was added to the store. - * This avoids retrying a directory that's serving bad or entirely duplicate - * certificates. */ - if (failure_code == 0 && added_trusted_cert) { - networkstatus_note_certs_arrived(source_dir); - } else { - networkstatus_note_certs_arrived(NULL); - } - - return failure_code; -} - -/** Save all v3 key certificates to the cached-certs file. */ -void -trusted_dirs_flush_certs_to_disk(void) -{ - char *filename; - smartlist_t *chunks; - - if (!trusted_dir_servers_certs_changed || !trusted_dir_certs) - return; - - chunks = smartlist_new(); - DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t)); - c->bytes = cert->cache_info.signed_descriptor_body; - c->len = cert->cache_info.signed_descriptor_len; - smartlist_add(chunks, c); - }); - } DIGESTMAP_FOREACH_END; - - filename = get_cachedir_fname("cached-certs"); - if (write_chunks_to_file(filename, chunks, 0, 0)) { - log_warn(LD_FS, "Error writing certificates to disk."); - } - tor_free(filename); - SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c)); - smartlist_free(chunks); - - trusted_dir_servers_certs_changed = 0; -} - -static int -compare_certs_by_pubdates(const void **_a, const void **_b) -{ - const authority_cert_t *cert1 = *_a, *cert2=*_b; - - if (cert1->cache_info.published_on < cert2->cache_info.published_on) - return -1; - else if (cert1->cache_info.published_on > cert2->cache_info.published_on) - return 1; - else - return 0; -} - -/** Remove all expired v3 authority certificates that have been superseded for - * more than 48 hours or, if not expired, that were published more than 7 days - * before being superseded. (If the most recent cert was published more than 48 - * hours ago, then we aren't going to get any consensuses signed with older - * keys.) */ -static void -trusted_dirs_remove_old_certs(void) -{ - time_t now = time(NULL); -#define DEAD_CERT_LIFETIME (2*24*60*60) -#define SUPERSEDED_CERT_LIFETIME (2*24*60*60) - if (!trusted_dir_certs) - return; - - DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { - /* Sort the list from first-published to last-published */ - smartlist_sort(cl->certs, compare_certs_by_pubdates); - - SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { - if (cert_sl_idx == smartlist_len(cl->certs) - 1) { - /* This is the most recently published cert. Keep it. */ - continue; - } - authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1); - const time_t next_cert_published = next_cert->cache_info.published_on; - if (next_cert_published > now) { - /* All later certs are published in the future. Keep everything - * we didn't discard. */ - break; - } - int should_remove = 0; - if (cert->expires + DEAD_CERT_LIFETIME < now) { - /* Certificate has been expired for at least DEAD_CERT_LIFETIME. - * Remove it. */ - should_remove = 1; - } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) { - /* Certificate has been superseded for OLD_CERT_LIFETIME. - * Remove it. - */ - should_remove = 1; - } - if (should_remove) { - SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert); - authority_cert_free(cert); - trusted_dir_servers_certs_changed = 1; - } - } SMARTLIST_FOREACH_END(cert); - - } DIGESTMAP_FOREACH_END; -#undef DEAD_CERT_LIFETIME -#undef OLD_CERT_LIFETIME - - trusted_dirs_flush_certs_to_disk(); -} - -/** Return the newest v3 authority certificate whose v3 authority identity key - * has digest <b>id_digest</b>. Return NULL if no such authority is known, - * or it has no certificate. */ -authority_cert_t * -authority_cert_get_newest_by_id(const char *id_digest) -{ - cert_list_t *cl; - authority_cert_t *best = NULL; - if (!trusted_dir_certs || - !(cl = digestmap_get(trusted_dir_certs, id_digest))) - return NULL; - - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!best || cert->cache_info.published_on > best->cache_info.published_on) - best = cert; - }); - return best; -} - -/** Return the newest v3 authority certificate whose directory signing key has - * digest <b>sk_digest</b>. Return NULL if no such certificate is known. - */ -authority_cert_t * -authority_cert_get_by_sk_digest(const char *sk_digest) -{ - authority_cert_t *c; - if (!trusted_dir_certs) - return NULL; - - if ((c = get_my_v3_authority_cert()) && - tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN)) - return c; - if ((c = get_my_v3_legacy_cert()) && - tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN)) - return c; - - DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN)) - return cert; - }); - } DIGESTMAP_FOREACH_END; - return NULL; -} - -/** Return the v3 authority certificate with signing key matching - * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>. - * Return NULL if no such authority is known. */ -authority_cert_t * -authority_cert_get_by_digests(const char *id_digest, - const char *sk_digest) -{ - cert_list_t *cl; - if (!trusted_dir_certs || - !(cl = digestmap_get(trusted_dir_certs, id_digest))) - return NULL; - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN)) - return cert; ); - - return NULL; -} - -/** Add every known authority_cert_t to <b>certs_out</b>. */ -void -authority_cert_get_all(smartlist_t *certs_out) -{ - tor_assert(certs_out); - if (!trusted_dir_certs) - return; - - DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c, - smartlist_add(certs_out, c)); - } DIGESTMAP_FOREACH_END; -} - -/** Called when an attempt to download a certificate with the authority with - * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest - * fails with HTTP response code <b>status</b>: remember the failure, so we - * don't try again immediately. */ -void -authority_cert_dl_failed(const char *id_digest, - const char *signing_key_digest, int status) -{ - cert_list_t *cl; - download_status_t *dlstatus = NULL; - char id_digest_str[2*DIGEST_LEN+1]; - char sk_digest_str[2*DIGEST_LEN+1]; - - if (!trusted_dir_certs || - !(cl = digestmap_get(trusted_dir_certs, id_digest))) - return; - - /* - * Are we noting a failed download of the latest cert for the id digest, - * or of a download by (id, signing key) digest pair? - */ - if (!signing_key_digest) { - /* Just by id digest */ - download_status_failed(&cl->dl_status_by_id, status); - } else { - /* Reset by (id, signing key) digest pair - * - * Look for a download_status_t in the map with this digest - */ - dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest); - /* Got one? */ - if (dlstatus) { - download_status_failed(dlstatus, status); - } else { - /* - * Do this rather than hex_str(), since hex_str clobbers - * old results and we call twice in the param list. - */ - base16_encode(id_digest_str, sizeof(id_digest_str), - id_digest, DIGEST_LEN); - base16_encode(sk_digest_str, sizeof(sk_digest_str), - signing_key_digest, DIGEST_LEN); - log_warn(LD_BUG, - "Got failure for cert fetch with (fp,sk) = (%s,%s), with " - "status %d, but knew nothing about the download.", - id_digest_str, sk_digest_str, status); - } - } -} - -static const char *BAD_SIGNING_KEYS[] = { - "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31 - "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26 - "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09 - "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09 - "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56 - "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01 - "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30 - "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09 - "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10 - "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57 - NULL, -}; - -/** Return true iff <b>cert</b> authenticates some atuhority signing key - * which, because of the old openssl heartbleed vulnerability, should - * never be trusted. */ -int -authority_cert_is_blacklisted(const authority_cert_t *cert) -{ - char hex_digest[HEX_DIGEST_LEN+1]; - int i; - base16_encode(hex_digest, sizeof(hex_digest), - cert->signing_key_digest, sizeof(cert->signing_key_digest)); - - for (i = 0; BAD_SIGNING_KEYS[i]; ++i) { - if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) { - return 1; - } - } - return 0; -} - -/** Return true iff when we've been getting enough failures when trying to - * download the certificate with ID digest <b>id_digest</b> that we're willing - * to start bugging the user about it. */ -int -authority_cert_dl_looks_uncertain(const char *id_digest) -{ -#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2 - cert_list_t *cl; - int n_failures; - if (!trusted_dir_certs || - !(cl = digestmap_get(trusted_dir_certs, id_digest))) - return 0; - - n_failures = download_status_get_n_failures(&cl->dl_status_by_id); - return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; -} - -/* Fetch the authority certificates specified in resource. - * If we are a bridge client, and node is a configured bridge, fetch from node - * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from - * rs. Otherwise, fetch from a random directory mirror. */ -static void -authority_certs_fetch_resource_impl(const char *resource, - const char *dir_hint, - const node_t *node, - const routerstatus_t *rs) -{ - const or_options_t *options = get_options(); - int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource); - - /* Make sure bridge clients never connect to anything but a bridge */ - if (options->UseBridges) { - if (node && !node_is_a_configured_bridge(node)) { - /* If we're using bridges, and node is not a bridge, use a 3-hop path. */ - get_via_tor = 1; - } else if (!node) { - /* If we're using bridges, and there's no node, use a 3-hop path. */ - get_via_tor = 1; - } - } - - const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS - : DIRIND_ONEHOP; - - directory_request_t *req = NULL; - /* If we've just downloaded a consensus from a bridge, re-use that - * bridge */ - if (options->UseBridges && node && node->ri && !get_via_tor) { - /* clients always make OR connections to bridges */ - tor_addr_port_t or_ap; - /* we are willing to use a non-preferred address if we need to */ - fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, - &or_ap); - - req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); - directory_request_set_or_addr_port(req, &or_ap); - if (dir_hint) - directory_request_set_directory_id_digest(req, dir_hint); - } else if (rs) { - /* And if we've just downloaded a consensus from a directory, re-use that - * directory */ - req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE); - directory_request_set_routerstatus(req, rs); - } - - if (req) { - /* We've set up a request object -- fill in the other request fields, and - * send the request. */ - directory_request_set_indirection(req, indirection); - directory_request_set_resource(req, resource); - directory_initiate_request(req); - directory_request_free(req); - return; - } - - /* Otherwise, we want certs from a random fallback or directory - * mirror, because they will almost always succeed. */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS, - DL_WANT_ANY_DIRSERVER); -} - -/** Try to download any v3 authority certificates that we may be missing. If - * <b>status</b> is provided, try to get all the ones that were used to sign - * <b>status</b>. Additionally, try to have a non-expired certificate for - * every V3 authority in trusted_dir_servers. Don't fetch certificates we - * already have. - * - * If dir_hint is non-NULL, it's the identity digest for a directory that - * we've just successfully retrieved a consensus or certificates from, so try - * it first to fetch any missing certificates. - **/ -void -authority_certs_fetch_missing(networkstatus_t *status, time_t now, - const char *dir_hint) -{ - /* - * The pending_id digestmap tracks pending certificate downloads by - * identity digest; the pending_cert digestmap tracks pending downloads - * by (identity digest, signing key digest) pairs. - */ - digestmap_t *pending_id; - fp_pair_map_t *pending_cert; - /* - * The missing_id_digests smartlist will hold a list of id digests - * we want to fetch the newest cert for; the missing_cert_digests - * smartlist will hold a list of fp_pair_t with an identity and - * signing key digest. - */ - smartlist_t *missing_cert_digests, *missing_id_digests; - char *resource = NULL; - cert_list_t *cl; - const or_options_t *options = get_options(); - const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options); - fp_pair_t *fp_tmp = NULL; - char id_digest_str[2*DIGEST_LEN+1]; - char sk_digest_str[2*DIGEST_LEN+1]; - - if (should_delay_dir_fetches(options, NULL)) - return; - - pending_cert = fp_pair_map_new(); - pending_id = digestmap_new(); - missing_cert_digests = smartlist_new(); - missing_id_digests = smartlist_new(); - - /* - * First, we get the lists of already pending downloads so we don't - * duplicate effort. - */ - list_pending_downloads(pending_id, NULL, - DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); - list_pending_fpsk_downloads(pending_cert); - - /* - * Now, we download any trusted authority certs we don't have by - * identity digest only. This gets the latest cert for that authority. - */ - SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) { - int found = 0; - if (!(ds->type & V3_DIRINFO)) - continue; - if (smartlist_contains_digest(missing_id_digests, - ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { - if (now < cert->expires) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&(cl->dl_status_by_id)); - /* No sense trying to download it specifically by signing key hash */ - download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest); - found = 1; - break; - } - } SMARTLIST_FOREACH_END(cert); - if (!found && - download_status_is_ready(&(cl->dl_status_by_id), now) && - !digestmap_get(pending_id, ds->v3_identity_digest)) { - log_info(LD_DIR, - "No current certificate known for authority %s " - "(ID digest %s); launching request.", - ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN)); - smartlist_add(missing_id_digests, ds->v3_identity_digest); - } - } SMARTLIST_FOREACH_END(ds); - - /* - * Next, if we have a consensus, scan through it and look for anything - * signed with a key from a cert we don't have. Those get downloaded - * by (fp,sk) pair, but if we don't know any certs at all for the fp - * (identity digest), and it's one of the trusted dir server certs - * we started off above or a pending download in pending_id, don't - * try to get it yet. Most likely, the one we'll get for that will - * have the right signing key too, and we'd just be downloading - * redundantly. - */ - if (status) { - SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, - voter) { - if (!smartlist_len(voter->sigs)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!keep_unknown && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We don't want unknown certs, and we don't know this - * authority.*/ - - /* - * If we don't know *any* cert for this authority, and a download by ID - * is pending or we added it to missing_id_digests above, skip this - * one for now to avoid duplicate downloads. - */ - cl = get_cert_list(voter->identity_digest); - if (smartlist_len(cl->certs) == 0) { - /* We have no certs at all for this one */ - - /* Do we have a download of one pending? */ - if (digestmap_get(pending_id, voter->identity_digest)) - continue; - - /* - * Are we about to launch a download of one due to the trusted - * dir server check above? - */ - if (smartlist_contains_digest(missing_id_digests, - voter->identity_digest)) - continue; - } - - SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - sig->signing_key_digest); - if (cert) { - if (now < cert->expires) - download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest); - continue; - } - if (download_status_is_ready_by_sk_in_cl( - cl, sig->signing_key_digest, now) && - !fp_pair_map_get_by_digests(pending_cert, - voter->identity_digest, - sig->signing_key_digest)) { - /* - * Do this rather than hex_str(), since hex_str clobbers - * old results and we call twice in the param list. - */ - base16_encode(id_digest_str, sizeof(id_digest_str), - voter->identity_digest, DIGEST_LEN); - base16_encode(sk_digest_str, sizeof(sk_digest_str), - sig->signing_key_digest, DIGEST_LEN); - - if (voter->nickname) { - log_info(LD_DIR, - "We're missing a certificate from authority %s " - "(ID digest %s) with signing key %s: " - "launching request.", - voter->nickname, id_digest_str, sk_digest_str); - } else { - log_info(LD_DIR, - "We're missing a certificate from authority ID digest " - "%s with signing key %s: launching request.", - id_digest_str, sk_digest_str); - } - - /* Allocate a new fp_pair_t to append */ - fp_tmp = tor_malloc(sizeof(*fp_tmp)); - memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first)); - memcpy(fp_tmp->second, sig->signing_key_digest, - sizeof(fp_tmp->second)); - smartlist_add(missing_cert_digests, fp_tmp); - } - } SMARTLIST_FOREACH_END(sig); - } SMARTLIST_FOREACH_END(voter); - } - - /* Bridge clients look up the node for the dir_hint */ - const node_t *node = NULL; - /* All clients, including bridge clients, look up the routerstatus for the - * dir_hint */ - const routerstatus_t *rs = NULL; - - /* If we still need certificates, try the directory that just successfully - * served us a consensus or certificates. - * As soon as the directory fails to provide additional certificates, we try - * another, randomly selected directory. This avoids continual retries. - * (We only ever have one outstanding request per certificate.) - */ - if (dir_hint) { - if (options->UseBridges) { - /* Bridge clients try the nodelist. If the dir_hint is from an authority, - * or something else fetched over tor, we won't find the node here, but - * we will find the rs. */ - node = node_get_by_id(dir_hint); - } - - /* All clients try the consensus routerstatus, then the fallback - * routerstatus */ - rs = router_get_consensus_status_by_id(dir_hint); - if (!rs) { - /* This will also find authorities */ - const dir_server_t *ds = router_get_fallback_dirserver_by_digest( - dir_hint); - if (ds) { - rs = &ds->fake_status; - } - } - - if (!node && !rs) { - log_warn(LD_BUG, "Directory %s delivered a consensus, but %s" - "no routerstatus could be found for it.", - options->UseBridges ? "no node and " : "", - hex_str(dir_hint, DIGEST_LEN)); - } - } - - /* Do downloads by identity digest */ - if (smartlist_len(missing_id_digests) > 0) { - int need_plus = 0; - smartlist_t *fps = smartlist_new(); - - smartlist_add_strdup(fps, "fp/"); - - SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) { - char *fp = NULL; - - if (digestmap_get(pending_id, d)) - continue; - - base16_encode(id_digest_str, sizeof(id_digest_str), - d, DIGEST_LEN); - - if (need_plus) { - tor_asprintf(&fp, "+%s", id_digest_str); - } else { - /* No need for tor_asprintf() in this case; first one gets no '+' */ - fp = tor_strdup(id_digest_str); - need_plus = 1; - } - - smartlist_add(fps, fp); - } SMARTLIST_FOREACH_END(d); - - if (smartlist_len(fps) > 1) { - resource = smartlist_join_strings(fps, "", 0, NULL); - /* node and rs are directories that just gave us a consensus or - * certificates */ - authority_certs_fetch_resource_impl(resource, dir_hint, node, rs); - tor_free(resource); - } - /* else we didn't add any: they were all pending */ - - SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp)); - smartlist_free(fps); - } - - /* Do downloads by identity digest/signing key pair */ - if (smartlist_len(missing_cert_digests) > 0) { - int need_plus = 0; - smartlist_t *fp_pairs = smartlist_new(); - - smartlist_add_strdup(fp_pairs, "fp-sk/"); - - SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) { - char *fp_pair = NULL; - - if (fp_pair_map_get(pending_cert, d)) - continue; - - /* Construct string encodings of the digests */ - base16_encode(id_digest_str, sizeof(id_digest_str), - d->first, DIGEST_LEN); - base16_encode(sk_digest_str, sizeof(sk_digest_str), - d->second, DIGEST_LEN); - - /* Now tor_asprintf() */ - if (need_plus) { - tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str); - } else { - /* First one in the list doesn't get a '+' */ - tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str); - need_plus = 1; - } +static const char *signed_descriptor_get_body_impl( + const signed_descriptor_t *desc, + int with_annotations); +static void launch_dummy_descriptor_download_as_needed(time_t now, + const or_options_t *options);

- /* Add it to the list of pairs to request */ - smartlist_add(fp_pairs, fp_pair); - } SMARTLIST_FOREACH_END(d); +/****************************************************************************/

- if (smartlist_len(fp_pairs) > 1) { - resource = smartlist_join_strings(fp_pairs, "", 0, NULL); - /* node and rs are directories that just gave us a consensus or - * certificates */ - authority_certs_fetch_resource_impl(resource, dir_hint, node, rs); - tor_free(resource); - } - /* else they were all pending */ +/** Global list of all of the routers that we know about. */ +static routerlist_t *routerlist = NULL;

- SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p)); - smartlist_free(fp_pairs); - } +/** List of strings for nicknames we've already warned about and that are + * still unknown / unavailable. */ +static smartlist_t *warned_nicknames = NULL;

- smartlist_free(missing_id_digests); - SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p)); - smartlist_free(missing_cert_digests); - digestmap_free(pending_id, NULL); - fp_pair_map_free(pending_cert, NULL); -} +/** The last time we tried to download any routerdesc, or 0 for "never". We + * use this to rate-limit download attempts when the number of routerdescs to + * download is low. */ +static time_t last_descriptor_download_attempted = 0;

/* Router descriptor storage. * @@ -1603,713 +426,60 @@ router_reload_router_list_impl(desc_store_t *store) else router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL, NULL, 0, NULL); - store->journal_len = (size_t) st.st_size; - tor_free(contents); - } - - tor_free(fname); - - if (store->journal_len) { - /* Always clear the journal on startup.*/ - router_rebuild_store(RRS_FORCE, store); - } else if (!extrainfo) { - /* Don't cache expired routers. (This is in an else because - * router_rebuild_store() also calls remove_old_routers().) */ - routerlist_remove_old_routers(); - } - - return 0; -} - -/** Load all cached router descriptors and extra-info documents from the - * store. Return 0 on success and -1 on failure. - */ -int -router_reload_router_list(void) -{ - routerlist_t *rl = router_get_routerlist(); - if (router_reload_router_list_impl(&rl->desc_store)) - return -1; - if (router_reload_router_list_impl(&rl->extrainfo_store)) - return -1; - return 0; -} - -/** Return a smartlist containing a list of dir_server_t * for all - * known trusted dirservers. Callers must not modify the list or its - * contents. - */ -const smartlist_t * -router_get_trusted_dir_servers(void) -{ - if (!trusted_dir_servers) - trusted_dir_servers = smartlist_new(); - - return trusted_dir_servers; -} - -const smartlist_t * -router_get_fallback_dir_servers(void) -{ - if (!fallback_dir_servers) - fallback_dir_servers = smartlist_new(); - - return fallback_dir_servers; -} - -/** Try to find a running dirserver that supports operations of <b>type</b>. - * - * If there are no running dirservers in our routerlist and the - * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones - * (including authorities) as running again, and pick one. - * - * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include - * dirservers that we can't reach. - * - * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself - * (if we're a dirserver). - * - * Don't pick a fallback directory mirror if any non-fallback is viable; - * (the fallback directory mirrors include the authorities) - * try to avoid using servers that have returned 503 recently. - */ -const routerstatus_t * -router_pick_directory_server(dirinfo_type_t type, int flags) -{ - int busy = 0; - const routerstatus_t *choice; - - if (!routerlist) - return NULL; - - choice = router_pick_directory_server_impl(type, flags, &busy); - if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) - return choice; - - if (busy) { - /* If the reason that we got no server is that servers are "busy", - * we must be excluding good servers because we already have serverdesc - * fetches with them. Do not mark down servers up because of this. */ - tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| - PDS_NO_EXISTING_MICRODESC_FETCH))); - return NULL; - } - - log_info(LD_DIR, - "No reachable router entries for dirservers. " - "Trying them all again."); - /* mark all fallback directory mirrors as up again */ - mark_all_dirservers_up(fallback_dir_servers); - /* try again */ - choice = router_pick_directory_server_impl(type, flags, NULL); - return choice; -} - -/** Return the dir_server_t for the directory authority whose identity - * key hashes to <b>digest</b>, or NULL if no such authority is known. - */ -dir_server_t * -router_get_trusteddirserver_by_digest(const char *digest) -{ - if (!trusted_dir_servers) - return NULL; - - SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, - { - if (tor_memeq(ds->digest, digest, DIGEST_LEN)) - return ds; - }); - - return NULL; -} - -/** Return the dir_server_t for the fallback dirserver whose identity - * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of - * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir - * and UseDefaultFallbackDirs torrc options.) - * The list of fallback directories includes the list of authorities. - */ -dir_server_t * -router_get_fallback_dirserver_by_digest(const char *digest) -{ - if (!fallback_dir_servers) - return NULL; - - if (!digest) - return NULL; - - SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds, - { - if (tor_memeq(ds->digest, digest, DIGEST_LEN)) - return ds; - }); - - return NULL; -} - -/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>, - * or 0 if no such fallback is in the list of fallback_dir_servers. - * (fallback_dir_servers is affected by the FallbackDir and - * UseDefaultFallbackDirs torrc options.) - * The list of fallback directories includes the list of authorities. - */ -int -router_digest_is_fallback_dir(const char *digest) -{ - return (router_get_fallback_dirserver_by_digest(digest) != NULL); -} - -/** Return the dir_server_t for the directory authority whose - * v3 identity key hashes to <b>digest</b>, or NULL if no such authority - * is known. - */ -MOCK_IMPL(dir_server_t *, -trusteddirserver_get_by_v3_auth_digest, (const char *digest)) -{ - if (!trusted_dir_servers) - return NULL; - - SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, - { - if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) && - (ds->type & V3_DIRINFO)) - return ds; - }); - - return NULL; -} - -/** Try to find a running directory authority. Flags are as for - * router_pick_directory_server. - */ -const routerstatus_t * -router_pick_trusteddirserver(dirinfo_type_t type, int flags) -{ - return router_pick_dirserver_generic(trusted_dir_servers, type, flags); -} - -/** Try to find a running fallback directory. Flags are as for - * router_pick_directory_server. - */ -const routerstatus_t * -router_pick_fallback_dirserver(dirinfo_type_t type, int flags) -{ - return router_pick_dirserver_generic(fallback_dir_servers, type, flags); -} - -/** Try to find a running fallback directory. Flags are as for - * router_pick_directory_server. - */ -static const routerstatus_t * -router_pick_dirserver_generic(smartlist_t *sourcelist, - dirinfo_type_t type, int flags) -{ - const routerstatus_t *choice; - int busy = 0; - - if (smartlist_len(sourcelist) == 1) { - /* If there's only one choice, then we should disable the logic that - * would otherwise prevent us from choosing ourself. */ - flags |= PDS_ALLOW_SELF; - } - - choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy); - if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) - return choice; - if (busy) { - /* If the reason that we got no server is that servers are "busy", - * we must be excluding good servers because we already have serverdesc - * fetches with them. Do not mark down servers up because of this. */ - tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| - PDS_NO_EXISTING_MICRODESC_FETCH))); - return NULL; - } - - log_info(LD_DIR, - "No dirservers are reachable. Trying them all again."); - mark_all_dirservers_up(sourcelist); - return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL); -} - -/* Check if we already have a directory fetch from ap, for serverdesc - * (including extrainfo) or microdesc documents. - * If so, return 1, if not, return 0. - * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0. - */ -STATIC int -router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc, - int microdesc) -{ - if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) { - return 0; - } - - /* XX/teor - we're not checking tunnel connections here, see #17848 - */ - if (serverdesc && ( - connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC) - || connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) { - return 1; - } - - if (microdesc && ( - connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) { - return 1; - } - - return 0; -} - -/* Check if we already have a directory fetch from the ipv4 or ipv6 - * router, for serverdesc (including extrainfo) or microdesc documents. - * If so, return 1, if not, return 0. - */ -static int -router_is_already_dir_fetching_(uint32_t ipv4_addr, - const tor_addr_t *ipv6_addr, - uint16_t dir_port, - int serverdesc, - int microdesc) -{ - tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap; - - /* Assume IPv6 DirPort is the same as IPv4 DirPort */ - tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr); - ipv4_dir_ap.port = dir_port; - tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr); - ipv6_dir_ap.port = dir_port; - - return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc) - || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc)); -} - -#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP -#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0 -#endif - -/* Log a message if rs is not found or not a preferred address */ -static void -router_picked_poor_directory_log(const routerstatus_t *rs) -{ - const networkstatus_t *usable_consensus; - usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL), - usable_consensus_flavor()); - -#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP - /* Don't log early in the bootstrap process, it's normal to pick from a - * small pool of nodes. Of course, this won't help if we're trying to - * diagnose bootstrap issues. */ - if (!smartlist_len(nodelist_get_list()) || !usable_consensus - || !router_have_minimum_dir_info()) { - return; - } -#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */ - - /* We couldn't find a node, or the one we have doesn't fit our preferences. - * Sometimes this is normal, sometimes it can be a reachability issue. */ - if (!rs) { - /* This happens a lot, so it's at debug level */ - log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but " - "we couldn't find a directory that fit our criteria. " - "Perhaps we will succeed next time with less strict criteria."); - } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1) - && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1) - ) { - /* This is rare, and might be interesting to users trying to diagnose - * connection issues on dual-stack machines. */ - log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir " - "addresses for launching an outgoing connection: " - "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d", - routerstatus_describe(rs), - fmt_addr32(rs->addr), rs->or_port, - rs->dir_port, fmt_addr(&rs->ipv6_addr), - rs->ipv6_orport, rs->dir_port); - } -} - -#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP - -/** How long do we avoid using a directory server after it's given us a 503? */ -#define DIR_503_TIMEOUT (60*60) - -/* Common retry code for router_pick_directory_server_impl and - * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version. - * Must be called before RETRY_WITHOUT_EXCLUDE(). - * - * If we got no result, and we are applying IP preferences, and we are a - * client that could use an alternate IP version, try again with the - * opposite preferences. */ -#define RETRY_ALTERNATE_IP_VERSION(retry_label) \ - STMT_BEGIN \ - if (result == NULL && try_ip_pref && options->ClientUseIPv4 \ - && fascist_firewall_use_ipv6(options) && !server_mode(options) \ - && !n_busy) { \ - n_excluded = 0; \ - n_busy = 0; \ - try_ip_pref = 0; \ - goto retry_label; \ - } \ - STMT_END \ - -/* Common retry code for router_pick_directory_server_impl and - * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with - * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION(). - * - * If we got no result, and we are excluding nodes, and StrictNodes is - * not set, try again without excluding nodes. */ -#define RETRY_WITHOUT_EXCLUDE(retry_label) \ - STMT_BEGIN \ - if (result == NULL && try_excluding && !options->StrictNodes \ - && n_excluded && !n_busy) { \ - try_excluding = 0; \ - n_excluded = 0; \ - n_busy = 0; \ - try_ip_pref = 1; \ - goto retry_label; \ - } \ - STMT_END - -/* Common code used in the loop within router_pick_directory_server_impl and - * router_pick_trusteddirserver_impl. - * - * Check if the given <b>identity</b> supports extrainfo. If not, skip further - * checks. - */ -#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity) \ - STMT_BEGIN \ - int is_trusted_extrainfo = router_digest_is_trusted_dir_type( \ - (identity), EXTRAINFO_DIRINFO); \ - if (((type) & EXTRAINFO_DIRINFO) && \ - !router_supports_extrainfo((identity), is_trusted_extrainfo)) \ - continue; \ - STMT_END - -/* When iterating through the routerlist, can OR address/port preference - * and reachability checks be skipped? - */ -int -router_skip_or_reachability(const or_options_t *options, int try_ip_pref) -{ - /* Servers always have and prefer IPv4. - * And if clients are checking against the firewall for reachability only, - * but there's no firewall, don't bother checking */ - return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or()); -} - -/* When iterating through the routerlist, can Dir address/port preference - * and reachability checks be skipped? - */ -static int -router_skip_dir_reachability(const or_options_t *options, int try_ip_pref) -{ - /* Servers always have and prefer IPv4. - * And if clients are checking against the firewall for reachability only, - * but there's no firewall, don't bother checking */ - return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir()); -} - -/** Pick a random running valid directory server/mirror from our - * routerlist. Arguments are as for router_pick_directory_server(), except: - * - * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of - * directories that we excluded for no other reason than - * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH. - */ -STATIC const routerstatus_t * -router_pick_directory_server_impl(dirinfo_type_t type, int flags, - int *n_busy_out) -{ - const or_options_t *options = get_options(); - const node_t *result; - smartlist_t *direct, *tunnel; - smartlist_t *trusted_direct, *trusted_tunnel; - smartlist_t *overloaded_direct, *overloaded_tunnel; - time_t now = time(NULL); - const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - const int requireother = ! (flags & PDS_ALLOW_SELF); - const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); - const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH); - int try_excluding = 1, n_excluded = 0, n_busy = 0; - int try_ip_pref = 1; - - if (!consensus) - return NULL; - - retry_search: - - direct = smartlist_new(); - tunnel = smartlist_new(); - trusted_direct = smartlist_new(); - trusted_tunnel = smartlist_new(); - overloaded_direct = smartlist_new(); - overloaded_tunnel = smartlist_new(); - - const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref); - const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref); - const int must_have_or = directory_must_use_begindir(options); - - /* Find all the running dirservers we know about. */ - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { - int is_trusted; - int is_overloaded; - const routerstatus_t *status = node->rs; - const country_t country = node->country; - if (!status) - continue; - - if (!node->is_running || !node_is_dir(node) || !node->is_valid) - continue; - if (requireother && router_digest_is_me(node->identity)) - continue; - - SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity); - - if (try_excluding && - routerset_contains_routerstatus(options->ExcludeNodes, status, - country)) { - ++n_excluded; - continue; - } - - if (router_is_already_dir_fetching_(status->addr, - &status->ipv6_addr, - status->dir_port, - no_serverdesc_fetching, - no_microdesc_fetching)) { - ++n_busy; - continue; - } - - is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now; - is_trusted = router_digest_is_trusted_dir(node->identity); - - /* Clients use IPv6 addresses if the server has one and the client - * prefers IPv6. - * Add the router if its preferred address and port are reachable. - * If we don't get any routers, we'll try again with the non-preferred - * address for each router (if any). (To ensure correct load-balancing - * we try routers that only have one address both times.) - */ - if (!fascistfirewall || skip_or_fw || - fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, - try_ip_pref)) - smartlist_add(is_trusted ? trusted_tunnel : - is_overloaded ? overloaded_tunnel : tunnel, (void*)node); - else if (!must_have_or && (skip_dir_fw || - fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION, - try_ip_pref))) - smartlist_add(is_trusted ? trusted_direct : - is_overloaded ? overloaded_direct : direct, (void*)node); - } SMARTLIST_FOREACH_END(node); - - if (smartlist_len(tunnel)) { - result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); - } else if (smartlist_len(overloaded_tunnel)) { - result = node_sl_choose_by_bandwidth(overloaded_tunnel, - WEIGHT_FOR_DIR); - } else if (smartlist_len(trusted_tunnel)) { - /* FFFF We don't distinguish between trusteds and overloaded trusteds - * yet. Maybe one day we should. */ - /* FFFF We also don't load balance over authorities yet. I think this - * is a feature, but it could easily be a bug. -RD */ - result = smartlist_choose(trusted_tunnel); - } else if (smartlist_len(direct)) { - result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); - } else if (smartlist_len(overloaded_direct)) { - result = node_sl_choose_by_bandwidth(overloaded_direct, - WEIGHT_FOR_DIR); - } else { - result = smartlist_choose(trusted_direct); - } - smartlist_free(direct); - smartlist_free(tunnel); - smartlist_free(trusted_direct); - smartlist_free(trusted_tunnel); - smartlist_free(overloaded_direct); - smartlist_free(overloaded_tunnel); - - RETRY_ALTERNATE_IP_VERSION(retry_search); - - RETRY_WITHOUT_EXCLUDE(retry_search); - - if (n_busy_out) - *n_busy_out = n_busy; - - router_picked_poor_directory_log(result ? result->rs : NULL); - - return result ? result->rs : NULL; -} - -/** Pick a random element from a list of dir_server_t, weighting by their - * <b>weight</b> field. */ -static const dir_server_t * -dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight) -{ - int n = smartlist_len(servers); - int i; - double *weights_dbl; - uint64_t *weights_u64; - const dir_server_t *ds; - - weights_dbl = tor_calloc(n, sizeof(double)); - weights_u64 = tor_calloc(n, sizeof(uint64_t)); - for (i = 0; i < n; ++i) { - ds = smartlist_get(servers, i); - weights_dbl[i] = ds->weight; - if (ds->is_authority) - weights_dbl[i] *= authority_weight; - } - - scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL); - i = choose_array_element_by_weight(weights_u64, n); - tor_free(weights_dbl); - tor_free(weights_u64); - return (i < 0) ? NULL : smartlist_get(servers, i); -} - -/** Choose randomly from among the dir_server_ts in sourcelist that - * are up. Flags are as for router_pick_directory_server_impl(). - */ -static const routerstatus_t * -router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, - dirinfo_type_t type, int flags, - int *n_busy_out) -{ - const or_options_t *options = get_options(); - smartlist_t *direct, *tunnel; - smartlist_t *overloaded_direct, *overloaded_tunnel; - const routerinfo_t *me = router_get_my_routerinfo(); - const routerstatus_t *result = NULL; - time_t now = time(NULL); - const int requireother = ! (flags & PDS_ALLOW_SELF); - const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); - const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH); - const double auth_weight = (sourcelist == fallback_dir_servers) ? - options->DirAuthorityFallbackRate : 1.0; - smartlist_t *pick_from; - int n_busy = 0; - int try_excluding = 1, n_excluded = 0; - int try_ip_pref = 1; - - if (!sourcelist) - return NULL; - - retry_search: - - direct = smartlist_new(); - tunnel = smartlist_new(); - overloaded_direct = smartlist_new(); - overloaded_tunnel = smartlist_new(); - - const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref); - const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref); - const int must_have_or = directory_must_use_begindir(options); - - SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d) - { - int is_overloaded = - d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now; - if (!d->is_running) continue; - if ((type & d->type) == 0) - continue; - - SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest); - - if (requireother && me && router_digest_is_me(d->digest)) - continue; - if (try_excluding && - routerset_contains_routerstatus(options->ExcludeNodes, - &d->fake_status, -1)) { - ++n_excluded; - continue; - } - - if (router_is_already_dir_fetching_(d->addr, - &d->ipv6_addr, - d->dir_port, - no_serverdesc_fetching, - no_microdesc_fetching)) { - ++n_busy; - continue; - } - - /* Clients use IPv6 addresses if the server has one and the client - * prefers IPv6. - * Add the router if its preferred address and port are reachable. - * If we don't get any routers, we'll try again with the non-preferred - * address for each router (if any). (To ensure correct load-balancing - * we try routers that only have one address both times.) - */ - if (!fascistfirewall || skip_or_fw || - fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION, - try_ip_pref)) - smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d); - else if (!must_have_or && (skip_dir_fw || - fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION, - try_ip_pref))) - smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d); - } - SMARTLIST_FOREACH_END(d); - - if (smartlist_len(tunnel)) { - pick_from = tunnel; - } else if (smartlist_len(overloaded_tunnel)) { - pick_from = overloaded_tunnel; - } else if (smartlist_len(direct)) { - pick_from = direct; - } else { - pick_from = overloaded_direct; + store->journal_len = (size_t) st.st_size; + tor_free(contents); }

- { - const dir_server_t *selection = - dirserver_choose_by_weight(pick_from, auth_weight); + tor_free(fname);

- if (selection) - result = &selection->fake_status; + if (store->journal_len) { + /* Always clear the journal on startup.*/ + router_rebuild_store(RRS_FORCE, store); + } else if (!extrainfo) { + /* Don't cache expired routers. (This is in an else because + * router_rebuild_store() also calls remove_old_routers().) */ + routerlist_remove_old_routers(); }

- smartlist_free(direct); - smartlist_free(tunnel); - smartlist_free(overloaded_direct); - smartlist_free(overloaded_tunnel); - - RETRY_ALTERNATE_IP_VERSION(retry_search); - - RETRY_WITHOUT_EXCLUDE(retry_search); + return 0; +}

- router_picked_poor_directory_log(result); +/** Load all cached router descriptors and extra-info documents from the + * store. Return 0 on success and -1 on failure. + */ +int +router_reload_router_list(void) +{ + routerlist_t *rl = router_get_routerlist(); + if (router_reload_router_list_impl(&rl->desc_store)) + return -1; + if (router_reload_router_list_impl(&rl->extrainfo_store)) + return -1; + return 0; +}

- if (n_busy_out) - *n_busy_out = n_busy; - return result; +/* When iterating through the routerlist, can OR address/port preference + * and reachability checks be skipped? + */ +int +router_skip_or_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or()); }

-/** Mark as running every dir_server_t in <b>server_list</b>. */ -static void -mark_all_dirservers_up(smartlist_t *server_list) +/* When iterating through the routerlist, can Dir address/port preference + * and reachability checks be skipped? + */ +int +router_skip_dir_reachability(const or_options_t *options, int try_ip_pref) { - if (server_list) { - SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) { - routerstatus_t *rs; - node_t *node; - dir->is_running = 1; - node = node_get_mutable_by_id(dir->digest); - if (node) - node->is_running = 1; - rs = router_get_mutable_consensus_status_by_id(dir->digest); - if (rs) { - rs->last_dir_503_at = 0; - control_event_networkstatus_changed_single(rs); - } - } SMARTLIST_FOREACH_END(dir); - } - router_dir_info_changed(); + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir()); }

/** Return true iff r1 and r2 have the same address and OR port. */ @@ -2321,35 +491,6 @@ routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2) r1->ipv6_orport == r2->ipv6_orport; }

-/** Reset all internal variables used to count failed downloads of network - * status objects. */ -void -router_reset_status_download_failures(void) -{ - mark_all_dirservers_up(fallback_dir_servers); -} - -/** Given a <b>router</b>, add every node_t in its family (including the - * node itself!) to <b>sl</b>. - * - * Note the type mismatch: This function takes a routerinfo, but adds nodes - * to the smartlist! - */ -static void -routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) -{ - /* XXXX MOVE ? */ - node_t fake_node; - const node_t *node = node_get_by_id(router->cache_info.identity_digest); - if (node == NULL) { - memset(&fake_node, 0, sizeof(fake_node)); - fake_node.ri = (routerinfo_t *)router; - memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN); - node = &fake_node; - } - nodelist_add_node_and_family(sl, node); -} - /** Add every suitable node from our nodelist to <b>sl</b>, so that * we can pick a node for a circuit. */ @@ -2431,511 +572,6 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router) return result; }

-/** Given an array of double/uint64_t unions that are currently being used as - * doubles, convert them to uint64_t, and try to scale them linearly so as to - * much of the range of uint64_t. If <b>total_out</b> is provided, set it to - * the sum of all elements in the array _before_ scaling. */ -STATIC void -scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in, - int n_entries, - uint64_t *total_out) -{ - double total = 0.0; - double scale_factor = 0.0; - int i; - - for (i = 0; i < n_entries; ++i) - total += entries_in[i]; - - if (total > 0.0) { - scale_factor = ((double)INT64_MAX) / total; - scale_factor /= 4.0; /* make sure we're very far away from overflowing */ - } - - for (i = 0; i < n_entries; ++i) - entries_out[i] = tor_llround(entries_in[i] * scale_factor); - - if (total_out) - *total_out = (uint64_t) total; -} - -/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>, - * choosing each element with a probability proportional to its (uint64_t) - * value, and return the index of that element. If all elements are 0, choose - * an index at random. Return -1 on error. - */ -STATIC int -choose_array_element_by_weight(const uint64_t *entries, int n_entries) -{ - int i; - uint64_t rand_val; - uint64_t total = 0; - - for (i = 0; i < n_entries; ++i) - total += entries[i]; - - if (n_entries < 1) - return -1; - - if (total == 0) - return crypto_rand_int(n_entries); - - tor_assert(total < INT64_MAX); - - rand_val = crypto_rand_uint64(total); - - return select_array_member_cumulative_timei( - entries, n_entries, total, rand_val); -} - -/** When weighting bridges, enforce these values as lower and upper - * bound for believable bandwidth, because there is no way for us - * to verify a bridge's bandwidth currently. */ -#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */ -#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */ - -/** Return the smaller of the router's configured BandwidthRate - * and its advertised capacity, making sure to stay within the - * interval between bridge-min-believe-bw and - * bridge-max-believe-bw. */ -static uint32_t -bridge_get_advertised_bandwidth_bounded(routerinfo_t *router) -{ - uint32_t result = router->bandwidthcapacity; - if (result > router->bandwidthrate) - result = router->bandwidthrate; - if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH) - result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH; - else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH) - result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH; - return result; -} - -/** Return bw*1000, unless bw*1000 would overflow, in which case return - * INT32_MAX. */ -static inline int32_t -kb_to_bytes(uint32_t bw) -{ - return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000; -} - -/** Helper function: - * choose a random element of smartlist <b>sl</b> of nodes, weighted by - * the advertised bandwidth of each element using the consensus - * bandwidth weights. - * - * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all - * nodes' bandwidth equally regardless of their Exit status, since there may - * be some in the list because they exit to obscure ports. If - * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight - * exit-node's bandwidth less depending on the smallness of the fraction of - * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a - * guard node: consider all guard's bandwidth equally. Otherwise, weight - * guards proportionally less. - */ -static const node_t * -smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, - bandwidth_weight_rule_t rule) -{ - double *bandwidths_dbl=NULL; - uint64_t *bandwidths_u64=NULL; - - if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0) - return NULL; - - bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t)); - scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl, - smartlist_len(sl), NULL); - - { - int idx = choose_array_element_by_weight(bandwidths_u64, - smartlist_len(sl)); - tor_free(bandwidths_dbl); - tor_free(bandwidths_u64); - return idx < 0 ? NULL : smartlist_get(sl, idx); - } -} - -/** Given a list of routers and a weighting rule as in - * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth - * values for each node and store them in a freshly allocated - * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results - * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total - * of all the bandwidths. - * Return 0 on success, -1 on failure. */ -static int -compute_weighted_bandwidths(const smartlist_t *sl, - bandwidth_weight_rule_t rule, - double **bandwidths_out, - double *total_bandwidth_out) -{ - int64_t weight_scale; - double Wg = -1, Wm = -1, We = -1, Wd = -1; - double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; - guardfraction_bandwidth_t guardfraction_bw; - double *bandwidths = NULL; - double total_bandwidth = 0.0; - - tor_assert(sl); - tor_assert(bandwidths_out); - - /* Can't choose exit and guard at same time */ - tor_assert(rule == NO_WEIGHTING || - rule == WEIGHT_FOR_EXIT || - rule == WEIGHT_FOR_GUARD || - rule == WEIGHT_FOR_MID || - rule == WEIGHT_FOR_DIR); - - *bandwidths_out = NULL; - - if (total_bandwidth_out) { - *total_bandwidth_out = 0.0; - } - - if (smartlist_len(sl) == 0) { - log_info(LD_CIRC, - "Empty routerlist passed in to consensus weight node " - "selection for rule %s", - bandwidth_weight_rule_to_string(rule)); - return -1; - } - - weight_scale = networkstatus_get_weight_scale_param(NULL); - - if (rule == WEIGHT_FOR_GUARD) { - Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); - Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ - We = 0; - Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); - - Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); - Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); - Web = networkstatus_get_bw_weight(NULL, "Web", -1); - Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); - } else if (rule == WEIGHT_FOR_MID) { - Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); - Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); - We = networkstatus_get_bw_weight(NULL, "Wme", -1); - Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); - - Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); - Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); - Web = networkstatus_get_bw_weight(NULL, "Web", -1); - Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); - } else if (rule == WEIGHT_FOR_EXIT) { - // Guards CAN be exits if they have weird exit policies - // They are d then I guess... - We = networkstatus_get_bw_weight(NULL, "Wee", -1); - Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ - Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); - Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ - - Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); - Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); - Web = networkstatus_get_bw_weight(NULL, "Web", -1); - Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); - } else if (rule == WEIGHT_FOR_DIR) { - We = networkstatus_get_bw_weight(NULL, "Wbe", -1); - Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); - Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); - Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); - - Wgb = Wmb = Web = Wdb = weight_scale; - } else if (rule == NO_WEIGHTING) { - Wg = Wm = We = Wd = weight_scale; - Wgb = Wmb = Web = Wdb = weight_scale; - } - - if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 - || Web < 0) { - log_debug(LD_CIRC, - "Got negative bandwidth weights. Defaulting to naive selection" - " algorithm."); - Wg = Wm = We = Wd = weight_scale; - Wgb = Wmb = Web = Wdb = weight_scale; - } - - Wg /= weight_scale; - Wm /= weight_scale; - We /= weight_scale; - Wd /= weight_scale; - - Wgb /= weight_scale; - Wmb /= weight_scale; - Web /= weight_scale; - Wdb /= weight_scale; - - bandwidths = tor_calloc(smartlist_len(sl), sizeof(double)); - - // Cycle through smartlist and total the bandwidth. - static int warned_missing_bw = 0; - SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; - double weight = 1; - double weight_without_guard_flag = 0; /* Used for guardfraction */ - double final_weight = 0; - is_exit = node->is_exit && ! node->is_bad_exit; - is_guard = node->is_possible_guard; - is_dir = node_is_dir(node); - if (node->rs) { - if (!node->rs->has_bandwidth) { - /* This should never happen, unless all the authorities downgrade - * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ - if (! warned_missing_bw) { - log_warn(LD_BUG, - "Consensus is missing some bandwidths. Using a naive " - "router selection algorithm"); - warned_missing_bw = 1; - } - this_bw = 30000; /* Chosen arbitrarily */ - } else { - this_bw = kb_to_bytes(node->rs->bandwidth_kb); - } - } else if (node->ri) { - /* bridge or other descriptor not in our consensus */ - this_bw = bridge_get_advertised_bandwidth_bounded(node->ri); - } else { - /* We can't use this one. */ - continue; - } - - if (is_guard && is_exit) { - weight = (is_dir ? Wdb*Wd : Wd); - weight_without_guard_flag = (is_dir ? Web*We : We); - } else if (is_guard) { - weight = (is_dir ? Wgb*Wg : Wg); - weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm); - } else if (is_exit) { - weight = (is_dir ? Web*We : We); - } else { // middle - weight = (is_dir ? Wmb*Wm : Wm); - } - /* These should be impossible; but overflows here would be bad, so let's - * make sure. */ - if (this_bw < 0) - this_bw = 0; - if (weight < 0.0) - weight = 0.0; - if (weight_without_guard_flag < 0.0) - weight_without_guard_flag = 0.0; - - /* If guardfraction information is available in the consensus, we - * want to calculate this router's bandwidth according to its - * guardfraction. Quoting from proposal236: - * - * Let Wpf denote the weight from the 'bandwidth-weights' line a - * client would apply to N for position p if it had the guard - * flag, Wpn the weight if it did not have the guard flag, and B the - * measured bandwidth of N in the consensus. Then instead of choosing - * N for position p proportionally to Wpf*B or Wpn*B, clients should - * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. - */ - if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) { - /* XXX The assert should actually check for is_guard. However, - * that crashes dirauths because of #13297. This should be - * equivalent: */ - tor_assert(node->rs->is_possible_guard); - - guard_get_guardfraction_bandwidth(&guardfraction_bw, - this_bw, - node->rs->guardfraction_percentage); - - /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */ - final_weight = - guardfraction_bw.guard_bw * weight + - guardfraction_bw.non_guard_bw * weight_without_guard_flag; - - log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)", - node->rs->nickname, final_weight, weight*this_bw, - bandwidth_weight_rule_to_string(rule)); - } else { /* no guardfraction information. calculate the weight normally. */ - final_weight = weight*this_bw; - } - - bandwidths[node_sl_idx] = final_weight; - total_bandwidth += final_weight; - } SMARTLIST_FOREACH_END(node); - - log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " - "on weights " - "Wg=%f Wm=%f We=%f Wd=%f with total bw %f", - bandwidth_weight_rule_to_string(rule), - Wg, Wm, We, Wd, total_bandwidth); - - *bandwidths_out = bandwidths; - - if (total_bandwidth_out) { - *total_bandwidth_out = total_bandwidth; - } - - return 0; -} - -/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted - * by their weighted bandwidths with rule <b>rule</b>, for which we have - * descriptors. - * - * If <b>for_direct_connect</b> is true, we intend to connect to the node - * directly, as the first hop of a circuit; otherwise, we intend to connect - * to it indirectly, or use it as if we were connecting to it indirectly. */ -double -frac_nodes_with_descriptors(const smartlist_t *sl, - bandwidth_weight_rule_t rule, - int for_direct_conn) -{ - double *bandwidths = NULL; - double total, present; - - if (smartlist_len(sl) == 0) - return 0.0; - - if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 || - total <= 0.0) { - int n_with_descs = 0; - SMARTLIST_FOREACH(sl, const node_t *, node, { - if (node_has_preferred_descriptor(node, for_direct_conn)) - n_with_descs++; - }); - tor_free(bandwidths); - return ((double)n_with_descs) / smartlist_len(sl); - } - - present = 0.0; - SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - if (node_has_preferred_descriptor(node, for_direct_conn)) - present += bandwidths[node_sl_idx]; - } SMARTLIST_FOREACH_END(node); - - tor_free(bandwidths); - - return present / total; -} - -/** Choose a random element of status list <b>sl</b>, weighted by - * the advertised bandwidth of each node */ -const node_t * -node_sl_choose_by_bandwidth(const smartlist_t *sl, - bandwidth_weight_rule_t rule) -{ /*XXXX MOVE */ - return smartlist_choose_node_by_bandwidth_weights(sl, rule); -} - -/** Return a random running node from the nodelist. Never - * pick a node that is in - * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>, - * even if they are the only nodes available. - * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than - * a minimum uptime, return one of those. - * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the - * advertised capacity of each router. - * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers. - * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if - * picking an exit node, otherwise we weight bandwidths for picking a relay - * node (that is, possibly discounting exit nodes). - * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that - * have a routerinfo or microdescriptor -- that is, enough info to be - * used to build a circuit. - * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that - * have an address that is preferred by the ClientPreferIPv6ORPort setting - * (regardless of this flag, we exclude nodes that aren't allowed by the - * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0). - */ -const node_t * -router_choose_random_node(smartlist_t *excludedsmartlist, - routerset_t *excludedset, - router_crn_flags_t flags) -{ /* XXXX MOVE */ - const int need_uptime = (flags & CRN_NEED_UPTIME) != 0; - const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; - const int need_guard = (flags & CRN_NEED_GUARD) != 0; - const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - const int need_desc = (flags & CRN_NEED_DESC) != 0; - const int pref_addr = (flags & CRN_PREF_ADDR) != 0; - const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; - const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0; - - smartlist_t *sl=smartlist_new(), - *excludednodes=smartlist_new(); - const node_t *choice = NULL; - const routerinfo_t *r; - bandwidth_weight_rule_t rule; - - tor_assert(!(weight_for_exit && need_guard)); - rule = weight_for_exit ? WEIGHT_FOR_EXIT : - (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); - - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { - if (node_allows_single_hop_exits(node)) { - /* Exclude relays that allow single hop exit circuits. This is an - * obsolete option since 0.2.9.2-alpha and done by default in - * 0.3.1.0-alpha. */ - smartlist_add(excludednodes, node); - } else if (rendezvous_v3 && - !node_supports_v3_rendezvous_point(node)) { - /* Exclude relays that do not support to rendezvous for a hidden service - * version 3. */ - smartlist_add(excludednodes, node); - } - } SMARTLIST_FOREACH_END(node); - - /* If the node_t is not found we won't be to exclude ourself but we - * won't be able to pick ourself in router_choose_random_node() so - * this is fine to at least try with our routerinfo_t object. */ - if ((r = router_get_my_routerinfo())) - routerlist_add_node_and_family(excludednodes, r); - - router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity, - need_guard, need_desc, pref_addr, - direct_conn); - log_debug(LD_CIRC, - "We found %d running nodes.", - smartlist_len(sl)); - - smartlist_subtract(sl,excludednodes); - log_debug(LD_CIRC, - "We removed %d excludednodes, leaving %d nodes.", - smartlist_len(excludednodes), - smartlist_len(sl)); - - if (excludedsmartlist) { - smartlist_subtract(sl,excludedsmartlist); - log_debug(LD_CIRC, - "We removed %d excludedsmartlist, leaving %d nodes.", - smartlist_len(excludedsmartlist), - smartlist_len(sl)); - } - if (excludedset) { - routerset_subtract_nodes(sl,excludedset); - log_debug(LD_CIRC, - "We removed excludedset, leaving %d nodes.", - smartlist_len(sl)); - } - - // Always weight by bandwidth - choice = node_sl_choose_by_bandwidth(sl, rule); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD| - CRN_PREF_ADDR); - choice = router_choose_random_node( - excludedsmartlist, excludedset, flags); - } - smartlist_free(excludednodes); - if (!choice) { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } - return choice; -} - /** Helper: given an extended nickname in <b>hexdigest</b> try to decode it. * Return 0 on success, -1 on failure. Store the result into the * DIGEST_LEN-byte buffer at <b>digest_out</b>, the single character at @@ -3013,23 +649,6 @@ hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest, return tor_memeq(digest, identity_digest, DIGEST_LEN); }

-/** Return true iff <b>digest</b> is the digest of the identity key of a - * trusted directory matching at least one bit of <b>type</b>. If <b>type</b> - * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */ -int -router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type) -{ - if (!trusted_dir_servers) - return 0; - if (authdir_mode(get_options()) && router_digest_is_me(digest)) - return 1; - SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent, - if (tor_memeq(digest, ent->digest, DIGEST_LEN)) { - return (!type) || ((type & ent->type) != 0); - }); - return 0; -} - /** If hexdigest is correctly formed, base16_decode it into * digest, which must have DIGEST_LEN space in it. * Return 0 on success, -1 on failure. @@ -3808,14 +1427,7 @@ routerlist_free_all(void) smartlist_free(warned_nicknames); warned_nicknames = NULL; } - clear_dir_servers(); - smartlist_free(trusted_dir_servers); - smartlist_free(fallback_dir_servers); - trusted_dir_servers = fallback_dir_servers = NULL; - if (trusted_dir_certs) { - digestmap_free(trusted_dir_certs, cert_list_free_void); - trusted_dir_certs = NULL; - } + authcert_free_all(); }

/** Forget that we have issued any router-related warnings, so that we'll @@ -4621,221 +2233,13 @@ router_exit_policy_rejects_all(const routerinfo_t *router) return router->policy_is_reject_star; }

-/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity - * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, - * add ourself. If <b>is_authority</b>, this is a directory authority. Return - * the new directory server entry on success or NULL on failure. */ -static dir_server_t * -dir_server_new(int is_authority, - const char *nickname, - const tor_addr_t *addr, - const char *hostname, - uint16_t dir_port, uint16_t or_port, - const tor_addr_port_t *addrport_ipv6, - const char *digest, const char *v3_auth_digest, - dirinfo_type_t type, - double weight) -{ - dir_server_t *ent; - uint32_t a; - char *hostname_ = NULL; - - tor_assert(digest); - - if (weight < 0) - return NULL; - - if (tor_addr_family(addr) == AF_INET) - a = tor_addr_to_ipv4h(addr); - else - return NULL; - - if (!hostname) - hostname_ = tor_addr_to_str_dup(addr); - else - hostname_ = tor_strdup(hostname); - - ent = tor_malloc_zero(sizeof(dir_server_t)); - ent->nickname = nickname ? tor_strdup(nickname) : NULL; - ent->address = hostname_; - ent->addr = a; - ent->dir_port = dir_port; - ent->or_port = or_port; - ent->is_running = 1; - ent->is_authority = is_authority; - ent->type = type; - ent->weight = weight; - if (addrport_ipv6) { - if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) { - log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6."); - tor_addr_make_unspec(&ent->ipv6_addr); - } else { - tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr); - ent->ipv6_orport = addrport_ipv6->port; - } - } else { - tor_addr_make_unspec(&ent->ipv6_addr); - } - - memcpy(ent->digest, digest, DIGEST_LEN); - if (v3_auth_digest && (type & V3_DIRINFO)) - memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN); - - if (nickname) - tor_asprintf(&ent->description, "directory server "%s" at %s:%d", - nickname, hostname_, (int)dir_port); - else - tor_asprintf(&ent->description, "directory server at %s:%d", - hostname_, (int)dir_port); - - ent->fake_status.addr = ent->addr; - tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr); - memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN); - if (nickname) - strlcpy(ent->fake_status.nickname, nickname, - sizeof(ent->fake_status.nickname)); - else - ent->fake_status.nickname[0] = '\0'; - ent->fake_status.dir_port = ent->dir_port; - ent->fake_status.or_port = ent->or_port; - ent->fake_status.ipv6_orport = ent->ipv6_orport; - - return ent; -} - -/** Create an authoritative directory server at - * <b>address</b>:<b>port</b>, with identity key <b>digest</b>. If - * <b>address</b> is NULL, add ourself. Return the new trusted directory - * server entry on success or NULL if we couldn't add it. */ -dir_server_t * -trusted_dir_server_new(const char *nickname, const char *address, - uint16_t dir_port, uint16_t or_port, - const tor_addr_port_t *ipv6_addrport, - const char *digest, const char *v3_auth_digest, - dirinfo_type_t type, double weight) -{ - uint32_t a; - tor_addr_t addr; - char *hostname=NULL; - dir_server_t *result; - - if (!address) { /* The address is us; we should guess. */ - if (resolve_my_address(LOG_WARN, get_options(), - &a, NULL, &hostname) < 0) { - log_warn(LD_CONFIG, - "Couldn't find a suitable address when adding ourself as a " - "trusted directory server."); - return NULL; - } - if (!hostname) - hostname = tor_dup_ip(a); - } else { - if (tor_lookup_hostname(address, &a)) { - log_warn(LD_CONFIG, - "Unable to lookup address for directory server at '%s'", - address); - return NULL; - } - hostname = tor_strdup(address); - } - tor_addr_from_ipv4h(&addr, a); - - result = dir_server_new(1, nickname, &addr, hostname, - dir_port, or_port, - ipv6_addrport, - digest, - v3_auth_digest, type, weight); - tor_free(hostname); - return result; -} - -/** Return a new dir_server_t for a fallback directory server at - * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest - * <b>id_digest</b> */ -dir_server_t * -fallback_dir_server_new(const tor_addr_t *addr, - uint16_t dir_port, uint16_t or_port, - const tor_addr_port_t *addrport_ipv6, - const char *id_digest, double weight) -{ - return dir_server_new(0, NULL, addr, NULL, dir_port, or_port, - addrport_ipv6, - id_digest, - NULL, ALL_DIRINFO, weight); -} - -/** Add a directory server to the global list(s). */ -void -dir_server_add(dir_server_t *ent) -{ - if (!trusted_dir_servers) - trusted_dir_servers = smartlist_new(); - if (!fallback_dir_servers) - fallback_dir_servers = smartlist_new(); - - if (ent->is_authority) - smartlist_add(trusted_dir_servers, ent); - - smartlist_add(fallback_dir_servers, ent); - router_dir_info_changed(); -} - -/** Free storage held in <b>cert</b>. */ -void -authority_cert_free_(authority_cert_t *cert) -{ - if (!cert) - return; - - tor_free(cert->cache_info.signed_descriptor_body); - crypto_pk_free(cert->signing_key); - crypto_pk_free(cert->identity_key); - - tor_free(cert); -} - -#define dir_server_free(val) \ - FREE_AND_NULL(dir_server_t, dir_server_free_, (val)) - -/** Free storage held in <b>ds</b>. */ -static void -dir_server_free_(dir_server_t *ds) -{ - if (!ds) - return; - - tor_free(ds->nickname); - tor_free(ds->description); - tor_free(ds->address); - tor_free(ds); -} - -/** Remove all members from the list of dir servers. */ -void -clear_dir_servers(void) -{ - if (fallback_dir_servers) { - SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent, - dir_server_free(ent)); - smartlist_clear(fallback_dir_servers); - } else { - fallback_dir_servers = smartlist_new(); - } - if (trusted_dir_servers) { - smartlist_clear(trusted_dir_servers); - } else { - trusted_dir_servers = smartlist_new(); - } - router_dir_info_changed(); -} - /** For every current directory connection whose purpose is <b>purpose</b>, * and where the resource being downloaded begins with <b>prefix</b>, split * rest of the resource into base16 fingerprints (or base64 fingerprints if * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the * corresponding elements of <b>result</b> to a nonzero value. */ -static void +void list_pending_downloads(digestmap_t *result, digest256map_t *result256, int purpose, const char *prefix) { @@ -4895,41 +2299,6 @@ list_pending_microdesc_downloads(digest256map_t *result) list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); }

-/** For every certificate we are currently downloading by (identity digest, - * signing key digest) pair, set result[fp_pair] to (void *1). - */ -static void -list_pending_fpsk_downloads(fp_pair_map_t *result) -{ - const char *pfx = "fp-sk/"; - smartlist_t *tmp; - smartlist_t *conns; - const char *resource; - - tor_assert(result); - - tmp = smartlist_new(); - conns = get_connection_array(); - - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (conn->type == CONN_TYPE_DIR && - conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE && - !conn->marked_for_close) { - resource = TO_DIR_CONN(conn)->requested_resource; - if (!strcmpstart(resource, pfx)) - dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx), - tmp); - } - } SMARTLIST_FOREACH_END(conn); - - SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) { - fp_pair_map_set(result, fp, (void*)1); - tor_free(fp); - } SMARTLIST_FOREACH_END(fp); - - smartlist_free(tmp); -} - /** Launch downloads for all the descriptors whose digests or digests256 * are listed as digests[i] for lo <= i < hi. (Lo and hi may be out of * range.) If <b>source</b> is given, download from <b>source</b>; diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index 4b7406364..c3e97d9dd 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -42,108 +42,13 @@ typedef enum was_router_added_t { ROUTER_CERTS_EXPIRED = -8 } was_router_added_t;

-/** Flags to be passed to control router_choose_random_node() to indicate what - * kind of nodes to pick according to what algorithm. */ -typedef enum router_crn_flags_t { - CRN_NEED_UPTIME = 1<<0, - CRN_NEED_CAPACITY = 1<<1, - CRN_NEED_GUARD = 1<<2, - /* XXXX not used, apparently. */ - CRN_WEIGHT_AS_EXIT = 1<<5, - CRN_NEED_DESC = 1<<6, - /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */ - CRN_PREF_ADDR = 1<<7, - /* On clients, only provide nodes that we can connect to directly, based on - * our firewall rules */ - CRN_DIRECT_CONN = 1<<8, - /* On clients, only provide nodes with HSRend >= 2 protocol version which - * is required for hidden service version >= 3. */ - CRN_RENDEZVOUS_V3 = 1<<9, -} router_crn_flags_t; +/** How long do we avoid using a directory server after it's given us a 503? */ +#define DIR_503_TIMEOUT (60*60)

-/** Possible ways to weight routers when choosing one randomly. See - * routerlist_sl_choose_by_bandwidth() for more information.*/ -typedef enum bandwidth_weight_rule_t { - NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, - WEIGHT_FOR_DIR -} bandwidth_weight_rule_t; - -/* Flags for pick_directory_server() and pick_trusteddirserver(). */ -/** Flag to indicate that we should not automatically be willing to use - * ourself to answer a directory request. - * Passed to router_pick_directory_server (et al).*/ -#define PDS_ALLOW_SELF (1<<0) -/** Flag to indicate that if no servers seem to be up, we should mark all - * directory servers as up and try again. - * Passed to router_pick_directory_server (et al).*/ -#define PDS_RETRY_IF_NO_SERVERS (1<<1) -/** Flag to indicate that we should not exclude directory servers that - * our ReachableAddress settings would exclude. This usually means that - * we're going to connect to the server over Tor, and so we don't need to - * worry about our firewall telling us we can't. - * Passed to router_pick_directory_server (et al).*/ -#define PDS_IGNORE_FASCISTFIREWALL (1<<2) -/** Flag to indicate that we should not use any directory authority to which - * we have an existing directory connection for downloading server descriptors - * or extrainfo documents. - * - * Passed to router_pick_directory_server (et al) - */ -#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) -/** Flag to indicate that we should not use any directory authority to which - * we have an existing directory connection for downloading microdescs. - * - * Passed to router_pick_directory_server (et al) - */ -#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4) - -int get_n_authorities(dirinfo_type_t type); -int trusted_dirs_reload_certs(void); - -/* - * Pass one of these as source to trusted_dirs_load_certs_from_string() - * to indicate whence string originates; this controls error handling - * behavior such as marking downloads as failed. - */ - -#define TRUSTED_DIRS_CERTS_SRC_SELF 0 -#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1 -#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2 -#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3 -#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4 - -int trusted_dirs_load_certs_from_string(const char *contents, int source, - int flush, const char *source_dir); -void trusted_dirs_flush_certs_to_disk(void); -authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); -authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); -authority_cert_t *authority_cert_get_by_digests(const char *id_digest, - const char *sk_digest); -void authority_cert_get_all(smartlist_t *certs_out); -void authority_cert_dl_failed(const char *id_digest, - const char *signing_key_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now, - const char *dir_hint); int router_reload_router_list(void); -int authority_cert_dl_looks_uncertain(const char *id_digest); -const smartlist_t *router_get_trusted_dir_servers(void); -const smartlist_t *router_get_fallback_dir_servers(void); -int authority_cert_is_blacklisted(const authority_cert_t *cert);

-const routerstatus_t *router_pick_directory_server(dirinfo_type_t type, - int flags); -dir_server_t *router_get_trusteddirserver_by_digest(const char *d); -dir_server_t *router_get_fallback_dirserver_by_digest( - const char *digest); -int router_digest_is_fallback_dir(const char *digest); -MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, - (const char *d)); -const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type, - int flags); -const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, - int flags); int router_skip_or_reachability(const or_options_t *options, int try_ip_pref); -int router_get_my_share_of_directory_requests(double *v3_share_out); +int router_skip_dir_reachability(const or_options_t *options, int try_ip_pref); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, @@ -155,21 +60,6 @@ const routerinfo_t *routerlist_find_my_routerinfo(void); uint32_t router_get_advertised_bandwidth(const routerinfo_t *router); uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router);

-const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl, - bandwidth_weight_rule_t rule); -double frac_nodes_with_descriptors(const smartlist_t *sl, - bandwidth_weight_rule_t rule, - int for_direct_conn); - -const node_t *router_choose_random_node(smartlist_t *excludedsmartlist, - struct routerset_t *excludedset, - router_crn_flags_t flags); - -int router_digest_is_trusted_dir_type(const char *digest, - dirinfo_type_t type); -#define router_digest_is_trusted_dir(d) \ - router_digest_is_trusted_dir_type((d), NO_DIRINFO) - int hexdigest_to_digest(const char *hexdigest, char *digest); const routerinfo_t *router_get_by_id_digest(const char *digest); routerinfo_t *router_get_mutable_by_digest(const char *digest); @@ -194,13 +84,10 @@ void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, void routerlist_free_all(void); void routerlist_reset_warnings(void);

-MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void)); -MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id, - (const char *digest)); -MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id, - (const char *digest)); -MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk, - (const char *id_digest, const char *sk_digest)); +/* XXXX move this */ +void list_pending_downloads(digestmap_t *result, + digest256map_t *result256, + int purpose, const char *prefix);

static int WRA_WAS_ADDED(was_router_added_t s); static int WRA_WAS_OUTDATED(was_router_added_t s); @@ -269,21 +156,6 @@ void routerlist_retry_directory_downloads(time_t now);

int router_exit_policy_rejects_all(const routerinfo_t *router);

-dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, - uint16_t dir_port, uint16_t or_port, - const tor_addr_port_t *addrport_ipv6, - const char *digest, const char *v3_auth_digest, - dirinfo_type_t type, double weight); -dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, - uint16_t dir_port, uint16_t or_port, - const tor_addr_port_t *addrport_ipv6, - const char *id_digest, double weight); -void dir_server_add(dir_server_t *ent); - -void authority_cert_free_(authority_cert_t *cert); -#define authority_cert_free(cert) \ - FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert)) -void clear_dir_servers(void); void update_consensus_router_descriptor_downloads(time_t now, int is_vote, networkstatus_t *consensus); void update_router_descriptor_downloads(time_t now); @@ -321,16 +193,6 @@ int hex_digest_nickname_matches(const char *hexdigest, const char *nickname);

#ifdef ROUTERLIST_PRIVATE -STATIC int choose_array_element_by_weight(const uint64_t *entries, - int n_entries); -STATIC void scale_array_elements_to_u64(uint64_t *entries_out, - const double *entries_in, - int n_entries, - uint64_t *total_out); -STATIC const routerstatus_t *router_pick_directory_server_impl( - dirinfo_type_t auth, int flags, - int *n_busy_out); - MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router, int seconds)); MOCK_DECL(STATIC was_router_added_t, extrainfo_insert, @@ -339,8 +201,6 @@ MOCK_DECL(STATIC was_router_added_t, extrainfo_insert, MOCK_DECL(STATIC void, initiate_descriptor_downloads, (const routerstatus_t *source, int purpose, smartlist_t *digests, int lo, int hi, int pds_flags)); -STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap, - int serverdesc, int microdesc);

#endif /* defined(ROUTERLIST_PRIVATE) */

diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index b76b2974f..5b82c2adf 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -75,6 +75,7 @@ #include "feature/relay/routerkeys.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" +#include "feature/nodelist/authcert.h" #include "lib/sandbox/sandbox.h" #include "feature/hs_common/shared_random_client.h" #include "feature/nodelist/torcert.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 1f316ebf0..d136eaeb6 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -30,6 +30,8 @@ #include "feature/stats/rephist.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "app/config/statefile.h" diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 37e604d07..f9508f464 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -33,7 +33,7 @@ #include "core/or/relay.h" #include "feature/stats/rephist.h" #include "feature/hs_common/replaycache.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" #include "lib/encoding/confline.h" diff --git a/src/test/test_config.c b/src/test/test_config.c index bf21a8d51..f224ddde3 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -38,6 +38,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" #include "feature/relay/router.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" #include "app/config/statefile.h" diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 0428ac6fc..4f5a9f58d 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -10,7 +10,7 @@ #include "feature/hs/hs_common.h" #include "feature/nodelist/networkstatus.h" #include "feature/rend/rendservice.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/authcert.h" #include "feature/nodelist/nodelist.h" #include "test/test.h" #include "test/test_helpers.h" diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 723799ee8..363539ce4 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -15,6 +15,7 @@ #define ROUTERPARSE_PRIVATE #define HIBERNATE_PRIVATE #define NETWORKSTATUS_PRIVATE +#define NODE_SELECT_PRIVATE #define RELAY_PRIVATE

#include "core/or/or.h" @@ -38,6 +39,9 @@ #include "feature/nodelist/networkstatus.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerparse.h" #include "feature/nodelist/routerset.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 09799a0e5..2cfed16b5 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -19,6 +19,8 @@ #include "feature/rend/rendcommon.h" #include "feature/rend/rendcache.h" #include "feature/relay/router.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerlist.h" #include "test/rend_test_helpers.h" #include "feature/nodelist/microdesc.h" diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 89d1f4f90..f629596c5 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -12,6 +12,7 @@ #define HIBERNATE_PRIVATE #define NETWORKSTATUS_PRIVATE #define ROUTERLIST_PRIVATE +#define NODE_SELECT_PRIVATE #define TOR_UNIT_TESTING #include "core/or/or.h" #include "app/config/config.h" @@ -27,6 +28,8 @@ #include "feature/nodelist/nodelist.h" #include "core/or/policies.h" #include "feature/relay/router.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" #include "feature/nodelist/routerparse.h" diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 70adf580a..725724aa5 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -17,7 +17,8 @@ #include "feature/nodelist/networkstatus.h" #include "feature/relay/router.h" #include "feature/relay/routerkeys.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/routerparse.h" #include "feature/hs_common/shared_random_client.h" #include "feature/dircommon/voting_schedule.h"