[tor-commits] [tor/master] Split routerlist.c into 4 separate modules
nickm at torproject.org
nickm at torproject.org
Thu Sep 20 15:07:57 UTC 2018
commit 08e3b88f0774fedb41a6b1c170a710dd12a7cb90
Author: Nick Mathewson <nickm at torproject.org>
Date: Wed Sep 19 14:10:15 2018 -0400
Split routerlist.c into 4 separate modules
There are now separate modules for:
* the list of router descriptors
* the list of authorities and fallbacks
* managing authority certificates
* selecting random nodes
---
src/app/config/config.c | 2 +-
src/core/include.am | 6 +
src/core/mainloop/main.c | 2 +
src/core/or/channeltls.c | 2 +-
src/core/or/circuitbuild.c | 2 +-
src/core/or/connection_or.c | 1 +
src/core/or/reasons.c | 2 +-
src/feature/client/bridges.c | 2 +
src/feature/client/entrynodes.c | 2 +-
src/feature/control/control.c | 2 +
src/feature/dirauth/dirvote.c | 2 +
src/feature/dirauth/shared_random.c | 3 +-
src/feature/dircache/directory.c | 4 +
src/feature/dircache/dirserv.c | 2 +
src/feature/hs/hs_service.c | 2 +-
src/feature/nodelist/authcert.c | 1205 +++++++++++++++
src/feature/nodelist/authcert.h | 60 +
src/feature/nodelist/dirlist.c | 421 ++++++
src/feature/nodelist/dirlist.h | 47 +
src/feature/nodelist/microdesc.c | 1 +
src/feature/nodelist/networkstatus.c | 3 +
src/feature/nodelist/node_select.c | 1108 ++++++++++++++
src/feature/nodelist/node_select.h | 102 ++
src/feature/nodelist/nodelist.c | 2 +
src/feature/nodelist/routerlist.c | 2777 +---------------------------------
src/feature/nodelist/routerlist.h | 154 +-
src/feature/nodelist/routerparse.c | 1 +
src/feature/relay/router.c | 2 +
src/feature/rend/rendservice.c | 2 +-
src/test/test_config.c | 1 +
src/test/test_controller.c | 2 +-
src/test/test_dir.c | 4 +
src/test/test_dir_handle_get.c | 2 +
src/test/test_routerlist.c | 3 +
src/test/test_shared_random.c | 3 +-
35 files changed, 3074 insertions(+), 2862 deletions(-)
diff --git a/src/app/config/config.c b/src/app/config/config.c
index 253d7ac9e..a3e775f48 100644
--- a/src/app/config/config.c
+++ b/src/app/config/config.c
@@ -105,7 +105,7 @@
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
#include "lib/sandbox/sandbox.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerset.h"
#include "core/or/scheduler.h"
#include "app/config/statefile.h"
diff --git a/src/core/include.am b/src/core/include.am
index 4ccabe438..7adcc0d9d 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -80,9 +80,12 @@ LIBTOR_APP_A_SOURCES = \
src/feature/hs/hs_stats.c \
src/feature/hs_common/replaycache.c \
src/feature/hs_common/shared_random_client.c \
+ src/feature/nodelist/authcert.c \
+ src/feature/nodelist/dirlist.c \
src/feature/nodelist/microdesc.c \
src/feature/nodelist/networkstatus.c \
src/feature/nodelist/nodelist.c \
+ src/feature/nodelist/node_select.c \
src/feature/nodelist/parsecommon.c \
src/feature/nodelist/routerlist.c \
src/feature/nodelist/routerparse.c \
@@ -256,8 +259,10 @@ noinst_HEADERS += \
src/feature/hs/hsdir_index_st.h \
src/feature/hs_common/replaycache.h \
src/feature/hs_common/shared_random_client.h \
+ src/feature/nodelist/authcert.h \
src/feature/nodelist/authority_cert_st.h \
src/feature/nodelist/desc_store_st.h \
+ src/feature/nodelist/dirlist.h \
src/feature/nodelist/document_signature_st.h \
src/feature/nodelist/extrainfo_st.h \
src/feature/nodelist/microdesc.h \
@@ -268,6 +273,7 @@ noinst_HEADERS += \
src/feature/nodelist/networkstatus_voter_info_st.h \
src/feature/nodelist/node_st.h \
src/feature/nodelist/nodelist.h \
+ src/feature/nodelist/node_select.h \
src/feature/nodelist/parsecommon.h \
src/feature/nodelist/routerinfo_st.h \
src/feature/nodelist/routerlist.h \
diff --git a/src/core/mainloop/main.c b/src/core/mainloop/main.c
index 0a31a3145..6884021bb 100644
--- a/src/core/mainloop/main.c
+++ b/src/core/mainloop/main.c
@@ -101,6 +101,8 @@
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "core/or/scheduler.h"
diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c
index b90a2ea87..c81a00b49 100644
--- a/src/core/or/channeltls.c
+++ b/src/core/or/channeltls.c
@@ -53,7 +53,7 @@
#include "core/or/relay.h"
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/dirlist.h"
#include "core/or/scheduler.h"
#include "feature/nodelist/torcert.h"
#include "feature/nodelist/networkstatus.h"
diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c
index b1dcfb853..ea23c1dfd 100644
--- a/src/core/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -60,6 +60,7 @@
#include "feature/rend/rendcommon.h"
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
@@ -3007,4 +3008,3 @@ circuit_upgrade_circuits_from_guard_wait(void)
smartlist_free(to_upgrade);
}
-
diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c
index e559e6b98..de6066020 100644
--- a/src/core/or/connection_or.c
+++ b/src/core/or/connection_or.c
@@ -56,6 +56,7 @@
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/relay/ext_orport.h"
#include "core/or/scheduler.h"
diff --git a/src/core/or/reasons.c b/src/core/or/reasons.c
index 7a1e09e9f..610338ee1 100644
--- a/src/core/or/reasons.c
+++ b/src/core/or/reasons.c
@@ -17,7 +17,7 @@
#include "core/or/or.h"
#include "app/config/config.h"
#include "core/or/reasons.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
#include "lib/tls/tortls.h"
/***************************** Edge (stream) reasons **********************/
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
index 7f4422f78..5ab0f6f4f 100644
--- a/src/feature/client/bridges.c
+++ b/src/feature/client/bridges.c
@@ -23,7 +23,9 @@
#include "feature/nodelist/nodelist.h"
#include "core/or/policies.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
+
#include "feature/nodelist/routerset.h"
#include "feature/client/transports.h"
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
index 494ad3352..f16de722d 100644
--- a/src/feature/client/entrynodes.c
+++ b/src/feature/client/entrynodes.c
@@ -133,7 +133,7 @@
#include "feature/nodelist/nodelist.h"
#include "core/or/policies.h"
#include "feature/relay/router.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
#include "feature/client/transports.h"
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index 5ac7f6d99..cff3c414c 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -77,6 +77,8 @@
#include "feature/rend/rendservice.h"
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/hs_common/shared_random_client.h"
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index 14357c770..76877490b 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -19,6 +19,8 @@
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/client/entrynodes.h" /* needed for guardfraction methods */
diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c
index 8ae267977..85ec82ac0 100644
--- a/src/feature/dirauth/shared_random.c
+++ b/src/feature/dirauth/shared_random.c
@@ -96,7 +96,7 @@
#include "feature/nodelist/networkstatus.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/hs_common/shared_random_client.h"
#include "feature/dirauth/shared_random_state.h"
#include "feature/dircommon/voting_schedule.h"
@@ -1288,4 +1288,3 @@ set_num_srv_agreements(int32_t value)
}
#endif /* defined(TOR_UNIT_TESTS) */
-
diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c
index b94c5317a..8628a35c5 100644
--- a/src/feature/dircache/directory.c
+++ b/src/feature/dircache/directory.c
@@ -40,7 +40,11 @@
#include "feature/rend/rendservice.h"
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
+
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
#include "lib/encoding/confline.h"
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
index b85db8324..b3cd83e21 100644
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@ -28,7 +28,9 @@
#include "core/or/protover.h"
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
+
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
#include "feature/nodelist/torcert.h"
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
index 243ecb9b5..83607e624 100644
--- a/src/feature/hs/hs_service.c
+++ b/src/feature/hs/hs_service.c
@@ -27,7 +27,7 @@
#include "feature/rend/rendservice.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/hs_common/shared_random_client.h"
#include "app/config/statefile.h"
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
new file mode 100644
index 000000000..c4a037590
--- /dev/null
+++ b/src/feature/nodelist/authcert.c
@@ -0,0 +1,1205 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert.c
+ * \brief Code to maintain directory authorities' certificates.
+ *
+ * Authority certificates are signed with authority identity keys; they
+ * are used to authenticate shorter-term authority signing keys. We
+ * fetch them when we find a consensus or a vote that has been signed
+ * with a signing key we don't recognize. We cache them on disk and
+ * load them on startup. Authority operators generate them with the
+ * "tor-gencert" utility.
+ */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/main.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/dircache/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerparse.h"
+#include "feature/relay/router.h"
+
+#include "core/or/connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/node_st.h"
+
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+#define DSMAP_FOREACH(map, keyvar, valvar) \
+ DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
+ valvar)
+#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
+
+/* Forward declaration for cert_list_t */
+typedef struct cert_list_t cert_list_t;
+
+static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
+ const char *digest);
+static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+ const char *digest,
+ time_t now);
+static void list_pending_fpsk_downloads(fp_pair_map_t *result);
+
+/** List of certificates for a single authority, and download status for
+ * latest certificate.
+ */
+struct cert_list_t {
+ /*
+ * The keys of download status map are cert->signing_key_digest for pending
+ * downloads by (identity digest/signing key digest) pair; functions such
+ * as authority_cert_get_by_digest() already assume these are unique.
+ */
+ struct digest_ds_map_t *dl_status_map;
+ /* There is also a dlstatus for the download by identity key only */
+ download_status_t dl_status_by_id;
+ smartlist_t *certs;
+};
+/** Map from v3 identity key digest to cert_list_t. */
+static digestmap_t *trusted_dir_certs = NULL;
+
+/** True iff any key certificate in at least one member of
+ * <b>trusted_dir_certs</b> has changed since we last flushed the
+ * certificates to disk. */
+static int trusted_dir_servers_certs_changed = 0;
+
+/** Initialise schedule, want_authority, and increment_on in the download
+ * status dlstatus, then call download_status_reset() on it.
+ * It is safe to call this function or download_status_reset() multiple times
+ * on a new dlstatus. But it should *not* be called after a dlstatus has been
+ * used to count download attempts or failures. */
+static void
+download_status_cert_init(download_status_t *dlstatus)
+{
+ dlstatus->schedule = DL_SCHED_CONSENSUS;
+ dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
+ dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
+ dlstatus->last_backoff_position = 0;
+ dlstatus->last_delay_used = 0;
+
+ /* Use the new schedule to set next_attempt_at */
+ download_status_reset(dlstatus);
+}
+
+/** Reset the download status of a specified element in a dsmap */
+static void
+download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
+{
+ download_status_t *dlstatus = NULL;
+
+ tor_assert(cl);
+ tor_assert(digest);
+
+ /* Make sure we have a dsmap */
+ if (!(cl->dl_status_map)) {
+ cl->dl_status_map = dsmap_new();
+ }
+ /* Look for a download_status_t in the map with this digest */
+ dlstatus = dsmap_get(cl->dl_status_map, digest);
+ /* Got one? */
+ if (!dlstatus) {
+ /* Insert before we reset */
+ dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+ dsmap_set(cl->dl_status_map, digest, dlstatus);
+ download_status_cert_init(dlstatus);
+ }
+ tor_assert(dlstatus);
+ /* Go ahead and reset it */
+ download_status_reset(dlstatus);
+}
+
+/**
+ * Return true if the download for this signing key digest in cl is ready
+ * to be re-attempted.
+ */
+static int
+download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+ const char *digest,
+ time_t now)
+{
+ int rv = 0;
+ download_status_t *dlstatus = NULL;
+
+ tor_assert(cl);
+ tor_assert(digest);
+
+ /* Make sure we have a dsmap */
+ if (!(cl->dl_status_map)) {
+ cl->dl_status_map = dsmap_new();
+ }
+ /* Look for a download_status_t in the map with this digest */
+ dlstatus = dsmap_get(cl->dl_status_map, digest);
+ /* Got one? */
+ if (dlstatus) {
+ /* Use download_status_is_ready() */
+ rv = download_status_is_ready(dlstatus, now);
+ } else {
+ /*
+ * If we don't know anything about it, return 1, since we haven't
+ * tried this one before. We need to create a new entry here,
+ * too.
+ */
+ dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+ download_status_cert_init(dlstatus);
+ dsmap_set(cl->dl_status_map, digest, dlstatus);
+ rv = 1;
+ }
+
+ return rv;
+}
+
+/** Helper: Return the cert_list_t for an authority whose authority ID is
+ * <b>id_digest</b>, allocating a new list if necessary. */
+static cert_list_t *
+get_cert_list(const char *id_digest)
+{
+ cert_list_t *cl;
+ if (!trusted_dir_certs)
+ trusted_dir_certs = digestmap_new();
+ cl = digestmap_get(trusted_dir_certs, id_digest);
+ if (!cl) {
+ cl = tor_malloc_zero(sizeof(cert_list_t));
+ download_status_cert_init(&cl->dl_status_by_id);
+ cl->certs = smartlist_new();
+ cl->dl_status_map = dsmap_new();
+ digestmap_set(trusted_dir_certs, id_digest, cl);
+ }
+ return cl;
+}
+
+/** Return a list of authority ID digests with potentially enumerable lists
+ * of download_status_t objects; used by controller GETINFO queries.
+ */
+
+MOCK_IMPL(smartlist_t *,
+list_authority_ids_with_downloads, (void))
+{
+ smartlist_t *ids = smartlist_new();
+ digestmap_iter_t *i;
+ const char *digest;
+ char *tmp;
+ void *cl;
+
+ if (trusted_dir_certs) {
+ for (i = digestmap_iter_init(trusted_dir_certs);
+ !(digestmap_iter_done(i));
+ i = digestmap_iter_next(trusted_dir_certs, i)) {
+ /*
+ * We always have at least dl_status_by_id to query, so no need to
+ * probe deeper than the existence of a cert_list_t.
+ */
+ digestmap_iter_get(i, &digest, &cl);
+ tmp = tor_malloc(DIGEST_LEN);
+ memcpy(tmp, digest, DIGEST_LEN);
+ smartlist_add(ids, tmp);
+ }
+ }
+ /* else definitely no downloads going since nothing even has a cert list */
+
+ return ids;
+}
+
+/** Given an authority ID digest, return a pointer to the default download
+ * status, or NULL if there is no such entry in trusted_dir_certs */
+
+MOCK_IMPL(download_status_t *,
+id_only_download_status_for_authority_id, (const char *digest))
+{
+ download_status_t *dl = NULL;
+ cert_list_t *cl;
+
+ if (trusted_dir_certs) {
+ cl = digestmap_get(trusted_dir_certs, digest);
+ if (cl) {
+ dl = &(cl->dl_status_by_id);
+ }
+ }
+
+ return dl;
+}
+
+/** Given an authority ID digest, return a smartlist of signing key digests
+ * for which download_status_t is potentially queryable, or NULL if no such
+ * authority ID digest is known. */
+
+MOCK_IMPL(smartlist_t *,
+list_sk_digests_for_authority_id, (const char *digest))
+{
+ smartlist_t *sks = NULL;
+ cert_list_t *cl;
+ dsmap_iter_t *i;
+ const char *sk_digest;
+ char *tmp;
+ download_status_t *dl;
+
+ if (trusted_dir_certs) {
+ cl = digestmap_get(trusted_dir_certs, digest);
+ if (cl) {
+ sks = smartlist_new();
+ if (cl->dl_status_map) {
+ for (i = dsmap_iter_init(cl->dl_status_map);
+ !(dsmap_iter_done(i));
+ i = dsmap_iter_next(cl->dl_status_map, i)) {
+ /* Pull the digest out and add it to the list */
+ dsmap_iter_get(i, &sk_digest, &dl);
+ tmp = tor_malloc(DIGEST_LEN);
+ memcpy(tmp, sk_digest, DIGEST_LEN);
+ smartlist_add(sks, tmp);
+ }
+ }
+ }
+ }
+
+ return sks;
+}
+
+/** Given an authority ID digest and a signing key digest, return the
+ * download_status_t or NULL if none exists. */
+
+MOCK_IMPL(download_status_t *,
+download_status_for_authority_id_and_sk,(const char *id_digest,
+ const char *sk_digest))
+{
+ download_status_t *dl = NULL;
+ cert_list_t *cl = NULL;
+
+ if (trusted_dir_certs) {
+ cl = digestmap_get(trusted_dir_certs, id_digest);
+ if (cl && cl->dl_status_map) {
+ dl = dsmap_get(cl->dl_status_map, sk_digest);
+ }
+ }
+
+ return dl;
+}
+
+#define cert_list_free(val) \
+ FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
+
+/** Release all space held by a cert_list_t */
+static void
+cert_list_free_(cert_list_t *cl)
+{
+ if (!cl)
+ return;
+
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+ authority_cert_free(cert));
+ smartlist_free(cl->certs);
+ dsmap_free(cl->dl_status_map, tor_free_);
+ tor_free(cl);
+}
+
+/** Wrapper for cert_list_free so we can pass it to digestmap_free */
+static void
+cert_list_free_void(void *cl)
+{
+ cert_list_free_(cl);
+}
+
+/** Reload the cached v3 key certificates from the cached-certs file in
+ * the data directory. Return 0 on success, -1 on failure. */
+int
+trusted_dirs_reload_certs(void)
+{
+ char *filename;
+ char *contents;
+ int r;
+
+ filename = get_cachedir_fname("cached-certs");
+ contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+ tor_free(filename);
+ if (!contents)
+ return 0;
+ r = trusted_dirs_load_certs_from_string(
+ contents,
+ TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
+ tor_free(contents);
+ return r;
+}
+
+/** Helper: return true iff we already have loaded the exact cert
+ * <b>cert</b>. */
+static inline int
+already_have_cert(authority_cert_t *cert)
+{
+ cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
+
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+ {
+ if (tor_memeq(c->cache_info.signed_descriptor_digest,
+ cert->cache_info.signed_descriptor_digest,
+ DIGEST_LEN))
+ return 1;
+ });
+ return 0;
+}
+
+/** Load a bunch of new key certificates from the string <b>contents</b>. If
+ * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
+ * from the cache, and we don't need to flush them to disk. If we are a
+ * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
+ * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
+ * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST. If <b>flush</b> is true, we
+ * need to flush any changed certificates to disk now. Return 0 on success,
+ * -1 if any certs fail to parse.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+int
+trusted_dirs_load_certs_from_string(const char *contents, int source,
+ int flush, const char *source_dir)
+{
+ dir_server_t *ds;
+ const char *s, *eos;
+ int failure_code = 0;
+ int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
+ int added_trusted_cert = 0;
+
+ for (s = contents; *s; s = eos) {
+ authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+ cert_list_t *cl;
+ if (!cert) {
+ failure_code = -1;
+ break;
+ }
+ ds = trusteddirserver_get_by_v3_auth_digest(
+ cert->cache_info.identity_digest);
+ log_debug(LD_DIR, "Parsed certificate for %s",
+ ds ? ds->nickname : "unknown authority");
+
+ if (already_have_cert(cert)) {
+ /* we already have this one. continue. */
+ log_info(LD_DIR, "Skipping %s certificate for %s that we "
+ "already have.",
+ from_store ? "cached" : "downloaded",
+ ds ? ds->nickname : "an old or new authority");
+
+ /*
+ * A duplicate on download should be treated as a failure, so we call
+ * authority_cert_dl_failed() to reset the download status to make sure
+ * we can't try again. Since we've implemented the fp-sk mechanism
+ * to download certs by signing key, this should be much rarer than it
+ * was and is perhaps cause for concern.
+ */
+ if (!from_store) {
+ if (authdir_mode(get_options())) {
+ log_warn(LD_DIR,
+ "Got a certificate for %s, but we already have it. "
+ "Maybe they haven't updated it. Waiting for a while.",
+ ds ? ds->nickname : "an old or new authority");
+ } else {
+ log_info(LD_DIR,
+ "Got a certificate for %s, but we already have it. "
+ "Maybe they haven't updated it. Waiting for a while.",
+ ds ? ds->nickname : "an old or new authority");
+ }
+
+ /*
+ * This is where we care about the source; authority_cert_dl_failed()
+ * needs to know whether the download was by fp or (fp,sk) pair to
+ * twiddle the right bit in the download map.
+ */
+ if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
+ authority_cert_dl_failed(cert->cache_info.identity_digest,
+ NULL, 404);
+ } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
+ authority_cert_dl_failed(cert->cache_info.identity_digest,
+ cert->signing_key_digest, 404);
+ }
+ }
+
+ authority_cert_free(cert);
+ continue;
+ }
+
+ if (ds) {
+ added_trusted_cert = 1;
+ log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
+ "signing key %s", from_store ? "cached" : "downloaded",
+ ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
+ } else {
+ int adding = we_want_to_fetch_unknown_auth_certs(get_options());
+ log_info(LD_DIR, "%s %s certificate for unrecognized directory "
+ "authority with signing key %s",
+ adding ? "Adding" : "Not adding",
+ from_store ? "cached" : "downloaded",
+ hex_str(cert->signing_key_digest,DIGEST_LEN));
+ if (!adding) {
+ authority_cert_free(cert);
+ continue;
+ }
+ }
+
+ cl = get_cert_list(cert->cache_info.identity_digest);
+ smartlist_add(cl->certs, cert);
+ if (ds && cert->cache_info.published_on > ds->addr_current_at) {
+ /* Check to see whether we should update our view of the authority's
+ * address. */
+ if (cert->addr && cert->dir_port &&
+ (ds->addr != cert->addr ||
+ ds->dir_port != cert->dir_port)) {
+ char *a = tor_dup_ip(cert->addr);
+ log_notice(LD_DIR, "Updating address for directory authority %s "
+ "from %s:%d to %s:%d based on certificate.",
+ ds->nickname, ds->address, (int)ds->dir_port,
+ a, cert->dir_port);
+ tor_free(a);
+ ds->addr = cert->addr;
+ ds->dir_port = cert->dir_port;
+ }
+ ds->addr_current_at = cert->cache_info.published_on;
+ }
+
+ if (!from_store)
+ trusted_dir_servers_certs_changed = 1;
+ }
+
+ if (flush)
+ trusted_dirs_flush_certs_to_disk();
+
+ /* call this even if failure_code is <0, since some certs might have
+ * succeeded, but only pass source_dir if there were no failures,
+ * and at least one more authority certificate was added to the store.
+ * This avoids retrying a directory that's serving bad or entirely duplicate
+ * certificates. */
+ if (failure_code == 0 && added_trusted_cert) {
+ networkstatus_note_certs_arrived(source_dir);
+ } else {
+ networkstatus_note_certs_arrived(NULL);
+ }
+
+ return failure_code;
+}
+
+/** Save all v3 key certificates to the cached-certs file. */
+void
+trusted_dirs_flush_certs_to_disk(void)
+{
+ char *filename;
+ smartlist_t *chunks;
+
+ if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
+ return;
+
+ chunks = smartlist_new();
+ DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+ {
+ sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
+ c->bytes = cert->cache_info.signed_descriptor_body;
+ c->len = cert->cache_info.signed_descriptor_len;
+ smartlist_add(chunks, c);
+ });
+ } DIGESTMAP_FOREACH_END;
+
+ filename = get_cachedir_fname("cached-certs");
+ if (write_chunks_to_file(filename, chunks, 0, 0)) {
+ log_warn(LD_FS, "Error writing certificates to disk.");
+ }
+ tor_free(filename);
+ SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
+ smartlist_free(chunks);
+
+ trusted_dir_servers_certs_changed = 0;
+}
+
+static int
+compare_certs_by_pubdates(const void **_a, const void **_b)
+{
+ const authority_cert_t *cert1 = *_a, *cert2=*_b;
+
+ if (cert1->cache_info.published_on < cert2->cache_info.published_on)
+ return -1;
+ else if (cert1->cache_info.published_on > cert2->cache_info.published_on)
+ return 1;
+ else
+ return 0;
+}
+
+/** Remove all expired v3 authority certificates that have been superseded for
+ * more than 48 hours or, if not expired, that were published more than 7 days
+ * before being superseded. (If the most recent cert was published more than 48
+ * hours ago, then we aren't going to get any consensuses signed with older
+ * keys.) */
+void
+trusted_dirs_remove_old_certs(void)
+{
+ time_t now = time(NULL);
+#define DEAD_CERT_LIFETIME (2*24*60*60)
+#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
+ if (!trusted_dir_certs)
+ return;
+
+ DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+ /* Sort the list from first-published to last-published */
+ smartlist_sort(cl->certs, compare_certs_by_pubdates);
+
+ SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+ if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
+ /* This is the most recently published cert. Keep it. */
+ continue;
+ }
+ authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
+ const time_t next_cert_published = next_cert->cache_info.published_on;
+ if (next_cert_published > now) {
+ /* All later certs are published in the future. Keep everything
+ * we didn't discard. */
+ break;
+ }
+ int should_remove = 0;
+ if (cert->expires + DEAD_CERT_LIFETIME < now) {
+ /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
+ * Remove it. */
+ should_remove = 1;
+ } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
+ /* Certificate has been superseded for OLD_CERT_LIFETIME.
+ * Remove it.
+ */
+ should_remove = 1;
+ }
+ if (should_remove) {
+ SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
+ authority_cert_free(cert);
+ trusted_dir_servers_certs_changed = 1;
+ }
+ } SMARTLIST_FOREACH_END(cert);
+
+ } DIGESTMAP_FOREACH_END;
+#undef DEAD_CERT_LIFETIME
+#undef OLD_CERT_LIFETIME
+
+ trusted_dirs_flush_certs_to_disk();
+}
+
+/** Return the newest v3 authority certificate whose v3 authority identity key
+ * has digest <b>id_digest</b>. Return NULL if no such authority is known,
+ * or it has no certificate. */
+authority_cert_t *
+authority_cert_get_newest_by_id(const char *id_digest)
+{
+ cert_list_t *cl;
+ authority_cert_t *best = NULL;
+ if (!trusted_dir_certs ||
+ !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+ return NULL;
+
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+ {
+ if (!best || cert->cache_info.published_on > best->cache_info.published_on)
+ best = cert;
+ });
+ return best;
+}
+
+/** Return the newest v3 authority certificate whose directory signing key has
+ * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
+ */
+authority_cert_t *
+authority_cert_get_by_sk_digest(const char *sk_digest)
+{
+ authority_cert_t *c;
+ if (!trusted_dir_certs)
+ return NULL;
+
+ if ((c = get_my_v3_authority_cert()) &&
+ tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+ return c;
+ if ((c = get_my_v3_legacy_cert()) &&
+ tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+ return c;
+
+ DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+ {
+ if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+ return cert;
+ });
+ } DIGESTMAP_FOREACH_END;
+ return NULL;
+}
+
+/** Return the v3 authority certificate with signing key matching
+ * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
+ * Return NULL if no such authority is known. */
+authority_cert_t *
+authority_cert_get_by_digests(const char *id_digest,
+ const char *sk_digest)
+{
+ cert_list_t *cl;
+ if (!trusted_dir_certs ||
+ !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+ return NULL;
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+ if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+ return cert; );
+
+ return NULL;
+}
+
+/** Add every known authority_cert_t to <b>certs_out</b>. */
+void
+authority_cert_get_all(smartlist_t *certs_out)
+{
+ tor_assert(certs_out);
+ if (!trusted_dir_certs)
+ return;
+
+ DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+ SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+ smartlist_add(certs_out, c));
+ } DIGESTMAP_FOREACH_END;
+}
+
+/** Called when an attempt to download a certificate with the authority with
+ * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
+ * fails with HTTP response code <b>status</b>: remember the failure, so we
+ * don't try again immediately. */
+void
+authority_cert_dl_failed(const char *id_digest,
+ const char *signing_key_digest, int status)
+{
+ cert_list_t *cl;
+ download_status_t *dlstatus = NULL;
+ char id_digest_str[2*DIGEST_LEN+1];
+ char sk_digest_str[2*DIGEST_LEN+1];
+
+ if (!trusted_dir_certs ||
+ !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+ return;
+
+ /*
+ * Are we noting a failed download of the latest cert for the id digest,
+ * or of a download by (id, signing key) digest pair?
+ */
+ if (!signing_key_digest) {
+ /* Just by id digest */
+ download_status_failed(&cl->dl_status_by_id, status);
+ } else {
+ /* Reset by (id, signing key) digest pair
+ *
+ * Look for a download_status_t in the map with this digest
+ */
+ dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
+ /* Got one? */
+ if (dlstatus) {
+ download_status_failed(dlstatus, status);
+ } else {
+ /*
+ * Do this rather than hex_str(), since hex_str clobbers
+ * old results and we call twice in the param list.
+ */
+ base16_encode(id_digest_str, sizeof(id_digest_str),
+ id_digest, DIGEST_LEN);
+ base16_encode(sk_digest_str, sizeof(sk_digest_str),
+ signing_key_digest, DIGEST_LEN);
+ log_warn(LD_BUG,
+ "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
+ "status %d, but knew nothing about the download.",
+ id_digest_str, sk_digest_str, status);
+ }
+ }
+}
+
+static const char *BAD_SIGNING_KEYS[] = {
+ "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
+ "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
+ "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
+ "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
+ "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
+ "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
+ "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
+ "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
+ "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
+ "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
+ NULL,
+};
+
+/** Return true iff <b>cert</b> authenticates some atuhority signing key
+ * which, because of the old openssl heartbleed vulnerability, should
+ * never be trusted. */
+int
+authority_cert_is_blacklisted(const authority_cert_t *cert)
+{
+ char hex_digest[HEX_DIGEST_LEN+1];
+ int i;
+ base16_encode(hex_digest, sizeof(hex_digest),
+ cert->signing_key_digest, sizeof(cert->signing_key_digest));
+
+ for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
+ if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/** Return true iff when we've been getting enough failures when trying to
+ * download the certificate with ID digest <b>id_digest</b> that we're willing
+ * to start bugging the user about it. */
+int
+authority_cert_dl_looks_uncertain(const char *id_digest)
+{
+#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
+ cert_list_t *cl;
+ int n_failures;
+ if (!trusted_dir_certs ||
+ !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+ return 0;
+
+ n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
+ return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
+}
+
+/* Fetch the authority certificates specified in resource.
+ * If we are a bridge client, and node is a configured bridge, fetch from node
+ * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
+ * rs. Otherwise, fetch from a random directory mirror. */
+static void
+authority_certs_fetch_resource_impl(const char *resource,
+ const char *dir_hint,
+ const node_t *node,
+ const routerstatus_t *rs)
+{
+ const or_options_t *options = get_options();
+ int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+ resource);
+
+ /* Make sure bridge clients never connect to anything but a bridge */
+ if (options->UseBridges) {
+ if (node && !node_is_a_configured_bridge(node)) {
+ /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
+ get_via_tor = 1;
+ } else if (!node) {
+ /* If we're using bridges, and there's no node, use a 3-hop path. */
+ get_via_tor = 1;
+ }
+ }
+
+ const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
+ : DIRIND_ONEHOP;
+
+ directory_request_t *req = NULL;
+ /* If we've just downloaded a consensus from a bridge, re-use that
+ * bridge */
+ if (options->UseBridges && node && node->ri && !get_via_tor) {
+ /* clients always make OR connections to bridges */
+ tor_addr_port_t or_ap;
+ /* we are willing to use a non-preferred address if we need to */
+ fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+ &or_ap);
+
+ req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+ directory_request_set_or_addr_port(req, &or_ap);
+ if (dir_hint)
+ directory_request_set_directory_id_digest(req, dir_hint);
+ } else if (rs) {
+ /* And if we've just downloaded a consensus from a directory, re-use that
+ * directory */
+ req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+ directory_request_set_routerstatus(req, rs);
+ }
+
+ if (req) {
+ /* We've set up a request object -- fill in the other request fields, and
+ * send the request. */
+ directory_request_set_indirection(req, indirection);
+ directory_request_set_resource(req, resource);
+ directory_initiate_request(req);
+ directory_request_free(req);
+ return;
+ }
+
+ /* Otherwise, we want certs from a random fallback or directory
+ * mirror, because they will almost always succeed. */
+ directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+ resource, PDS_RETRY_IF_NO_SERVERS,
+ DL_WANT_ANY_DIRSERVER);
+}
+
+/** Try to download any v3 authority certificates that we may be missing. If
+ * <b>status</b> is provided, try to get all the ones that were used to sign
+ * <b>status</b>. Additionally, try to have a non-expired certificate for
+ * every V3 authority in trusted_dir_servers. Don't fetch certificates we
+ * already have.
+ *
+ * If dir_hint is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ **/
+void
+authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+ const char *dir_hint)
+{
+ /*
+ * The pending_id digestmap tracks pending certificate downloads by
+ * identity digest; the pending_cert digestmap tracks pending downloads
+ * by (identity digest, signing key digest) pairs.
+ */
+ digestmap_t *pending_id;
+ fp_pair_map_t *pending_cert;
+ /*
+ * The missing_id_digests smartlist will hold a list of id digests
+ * we want to fetch the newest cert for; the missing_cert_digests
+ * smartlist will hold a list of fp_pair_t with an identity and
+ * signing key digest.
+ */
+ smartlist_t *missing_cert_digests, *missing_id_digests;
+ char *resource = NULL;
+ cert_list_t *cl;
+ const or_options_t *options = get_options();
+ const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
+ fp_pair_t *fp_tmp = NULL;
+ char id_digest_str[2*DIGEST_LEN+1];
+ char sk_digest_str[2*DIGEST_LEN+1];
+
+ if (should_delay_dir_fetches(options, NULL))
+ return;
+
+ pending_cert = fp_pair_map_new();
+ pending_id = digestmap_new();
+ missing_cert_digests = smartlist_new();
+ missing_id_digests = smartlist_new();
+
+ /*
+ * First, we get the lists of already pending downloads so we don't
+ * duplicate effort.
+ */
+ list_pending_downloads(pending_id, NULL,
+ DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
+ list_pending_fpsk_downloads(pending_cert);
+
+ /*
+ * Now, we download any trusted authority certs we don't have by
+ * identity digest only. This gets the latest cert for that authority.
+ */
+ SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+ dir_server_t *, ds) {
+ int found = 0;
+ if (!(ds->type & V3_DIRINFO))
+ continue;
+ if (smartlist_contains_digest(missing_id_digests,
+ ds->v3_identity_digest))
+ continue;
+ cl = get_cert_list(ds->v3_identity_digest);
+ SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+ if (now < cert->expires) {
+ /* It's not expired, and we weren't looking for something to
+ * verify a consensus with. Call it done. */
+ download_status_reset(&(cl->dl_status_by_id));
+ /* No sense trying to download it specifically by signing key hash */
+ download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
+ found = 1;
+ break;
+ }
+ } SMARTLIST_FOREACH_END(cert);
+ if (!found &&
+ download_status_is_ready(&(cl->dl_status_by_id), now) &&
+ !digestmap_get(pending_id, ds->v3_identity_digest)) {
+ log_info(LD_DIR,
+ "No current certificate known for authority %s "
+ "(ID digest %s); launching request.",
+ ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
+ smartlist_add(missing_id_digests, ds->v3_identity_digest);
+ }
+ } SMARTLIST_FOREACH_END(ds);
+
+ /*
+ * Next, if we have a consensus, scan through it and look for anything
+ * signed with a key from a cert we don't have. Those get downloaded
+ * by (fp,sk) pair, but if we don't know any certs at all for the fp
+ * (identity digest), and it's one of the trusted dir server certs
+ * we started off above or a pending download in pending_id, don't
+ * try to get it yet. Most likely, the one we'll get for that will
+ * have the right signing key too, and we'd just be downloading
+ * redundantly.
+ */
+ if (status) {
+ SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
+ voter) {
+ if (!smartlist_len(voter->sigs))
+ continue; /* This authority never signed this consensus, so don't
+ * go looking for a cert with key digest 0000000000. */
+ if (!keep_unknown &&
+ !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
+ continue; /* We don't want unknown certs, and we don't know this
+ * authority.*/
+
+ /*
+ * If we don't know *any* cert for this authority, and a download by ID
+ * is pending or we added it to missing_id_digests above, skip this
+ * one for now to avoid duplicate downloads.
+ */
+ cl = get_cert_list(voter->identity_digest);
+ if (smartlist_len(cl->certs) == 0) {
+ /* We have no certs at all for this one */
+
+ /* Do we have a download of one pending? */
+ if (digestmap_get(pending_id, voter->identity_digest))
+ continue;
+
+ /*
+ * Are we about to launch a download of one due to the trusted
+ * dir server check above?
+ */
+ if (smartlist_contains_digest(missing_id_digests,
+ voter->identity_digest))
+ continue;
+ }
+
+ SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+ authority_cert_t *cert =
+ authority_cert_get_by_digests(voter->identity_digest,
+ sig->signing_key_digest);
+ if (cert) {
+ if (now < cert->expires)
+ download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
+ continue;
+ }
+ if (download_status_is_ready_by_sk_in_cl(
+ cl, sig->signing_key_digest, now) &&
+ !fp_pair_map_get_by_digests(pending_cert,
+ voter->identity_digest,
+ sig->signing_key_digest)) {
+ /*
+ * Do this rather than hex_str(), since hex_str clobbers
+ * old results and we call twice in the param list.
+ */
+ base16_encode(id_digest_str, sizeof(id_digest_str),
+ voter->identity_digest, DIGEST_LEN);
+ base16_encode(sk_digest_str, sizeof(sk_digest_str),
+ sig->signing_key_digest, DIGEST_LEN);
+
+ if (voter->nickname) {
+ log_info(LD_DIR,
+ "We're missing a certificate from authority %s "
+ "(ID digest %s) with signing key %s: "
+ "launching request.",
+ voter->nickname, id_digest_str, sk_digest_str);
+ } else {
+ log_info(LD_DIR,
+ "We're missing a certificate from authority ID digest "
+ "%s with signing key %s: launching request.",
+ id_digest_str, sk_digest_str);
+ }
+
+ /* Allocate a new fp_pair_t to append */
+ fp_tmp = tor_malloc(sizeof(*fp_tmp));
+ memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
+ memcpy(fp_tmp->second, sig->signing_key_digest,
+ sizeof(fp_tmp->second));
+ smartlist_add(missing_cert_digests, fp_tmp);
+ }
+ } SMARTLIST_FOREACH_END(sig);
+ } SMARTLIST_FOREACH_END(voter);
+ }
+
+ /* Bridge clients look up the node for the dir_hint */
+ const node_t *node = NULL;
+ /* All clients, including bridge clients, look up the routerstatus for the
+ * dir_hint */
+ const routerstatus_t *rs = NULL;
+
+ /* If we still need certificates, try the directory that just successfully
+ * served us a consensus or certificates.
+ * As soon as the directory fails to provide additional certificates, we try
+ * another, randomly selected directory. This avoids continual retries.
+ * (We only ever have one outstanding request per certificate.)
+ */
+ if (dir_hint) {
+ if (options->UseBridges) {
+ /* Bridge clients try the nodelist. If the dir_hint is from an authority,
+ * or something else fetched over tor, we won't find the node here, but
+ * we will find the rs. */
+ node = node_get_by_id(dir_hint);
+ }
+
+ /* All clients try the consensus routerstatus, then the fallback
+ * routerstatus */
+ rs = router_get_consensus_status_by_id(dir_hint);
+ if (!rs) {
+ /* This will also find authorities */
+ const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
+ dir_hint);
+ if (ds) {
+ rs = &ds->fake_status;
+ }
+ }
+
+ if (!node && !rs) {
+ log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
+ "no routerstatus could be found for it.",
+ options->UseBridges ? "no node and " : "",
+ hex_str(dir_hint, DIGEST_LEN));
+ }
+ }
+
+ /* Do downloads by identity digest */
+ if (smartlist_len(missing_id_digests) > 0) {
+ int need_plus = 0;
+ smartlist_t *fps = smartlist_new();
+
+ smartlist_add_strdup(fps, "fp/");
+
+ SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
+ char *fp = NULL;
+
+ if (digestmap_get(pending_id, d))
+ continue;
+
+ base16_encode(id_digest_str, sizeof(id_digest_str),
+ d, DIGEST_LEN);
+
+ if (need_plus) {
+ tor_asprintf(&fp, "+%s", id_digest_str);
+ } else {
+ /* No need for tor_asprintf() in this case; first one gets no '+' */
+ fp = tor_strdup(id_digest_str);
+ need_plus = 1;
+ }
+
+ smartlist_add(fps, fp);
+ } SMARTLIST_FOREACH_END(d);
+
+ if (smartlist_len(fps) > 1) {
+ resource = smartlist_join_strings(fps, "", 0, NULL);
+ /* node and rs are directories that just gave us a consensus or
+ * certificates */
+ authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+ tor_free(resource);
+ }
+ /* else we didn't add any: they were all pending */
+
+ SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
+ smartlist_free(fps);
+ }
+
+ /* Do downloads by identity digest/signing key pair */
+ if (smartlist_len(missing_cert_digests) > 0) {
+ int need_plus = 0;
+ smartlist_t *fp_pairs = smartlist_new();
+
+ smartlist_add_strdup(fp_pairs, "fp-sk/");
+
+ SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
+ char *fp_pair = NULL;
+
+ if (fp_pair_map_get(pending_cert, d))
+ continue;
+
+ /* Construct string encodings of the digests */
+ base16_encode(id_digest_str, sizeof(id_digest_str),
+ d->first, DIGEST_LEN);
+ base16_encode(sk_digest_str, sizeof(sk_digest_str),
+ d->second, DIGEST_LEN);
+
+ /* Now tor_asprintf() */
+ if (need_plus) {
+ tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
+ } else {
+ /* First one in the list doesn't get a '+' */
+ tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
+ need_plus = 1;
+ }
+
+ /* Add it to the list of pairs to request */
+ smartlist_add(fp_pairs, fp_pair);
+ } SMARTLIST_FOREACH_END(d);
+
+ if (smartlist_len(fp_pairs) > 1) {
+ resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
+ /* node and rs are directories that just gave us a consensus or
+ * certificates */
+ authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+ tor_free(resource);
+ }
+ /* else they were all pending */
+
+ SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
+ smartlist_free(fp_pairs);
+ }
+
+ smartlist_free(missing_id_digests);
+ SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
+ smartlist_free(missing_cert_digests);
+ digestmap_free(pending_id, NULL);
+ fp_pair_map_free(pending_cert, NULL);
+}
+
+void
+authcert_free_all(void)
+{
+ if (trusted_dir_certs) {
+ digestmap_free(trusted_dir_certs, cert_list_free_void);
+ trusted_dir_certs = NULL;
+ }
+}
+
+/** Free storage held in <b>cert</b>. */
+void
+authority_cert_free_(authority_cert_t *cert)
+{
+ if (!cert)
+ return;
+
+ tor_free(cert->cache_info.signed_descriptor_body);
+ crypto_pk_free(cert->signing_key);
+ crypto_pk_free(cert->identity_key);
+
+ tor_free(cert);
+}
+
+/** For every certificate we are currently downloading by (identity digest,
+ * signing key digest) pair, set result[fp_pair] to (void *1).
+ */
+static void
+list_pending_fpsk_downloads(fp_pair_map_t *result)
+{
+ const char *pfx = "fp-sk/";
+ smartlist_t *tmp;
+ smartlist_t *conns;
+ const char *resource;
+
+ tor_assert(result);
+
+ tmp = smartlist_new();
+ conns = get_connection_array();
+
+ SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+ if (conn->type == CONN_TYPE_DIR &&
+ conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
+ !conn->marked_for_close) {
+ resource = TO_DIR_CONN(conn)->requested_resource;
+ if (!strcmpstart(resource, pfx))
+ dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
+ tmp);
+ }
+ } SMARTLIST_FOREACH_END(conn);
+
+ SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
+ fp_pair_map_set(result, fp, (void*)1);
+ tor_free(fp);
+ } SMARTLIST_FOREACH_END(fp);
+
+ smartlist_free(tmp);
+}
diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h
new file mode 100644
index 000000000..48326d7bd
--- /dev/null
+++ b/src/feature/nodelist/authcert.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert.h
+ * \brief Header file for authcert.c
+ **/
+
+#ifndef TOR_AUTHCERT_H
+#define TOR_AUTHCERT_H
+
+#include "lib/testsupport/testsupport.h"
+
+int trusted_dirs_reload_certs(void);
+
+/*
+ * Pass one of these as source to trusted_dirs_load_certs_from_string()
+ * to indicate whence string originates; this controls error handling
+ * behavior such as marking downloads as failed.
+ */
+
+#define TRUSTED_DIRS_CERTS_SRC_SELF 0
+#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
+#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
+
+int trusted_dirs_load_certs_from_string(const char *contents, int source,
+ int flush, const char *source_dir);
+void trusted_dirs_remove_old_certs(void);
+void trusted_dirs_flush_certs_to_disk(void);
+authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
+authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
+authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
+ const char *sk_digest);
+void authority_cert_get_all(smartlist_t *certs_out);
+void authority_cert_dl_failed(const char *id_digest,
+ const char *signing_key_digest, int status);
+void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+ const char *dir_hint);
+int authority_cert_dl_looks_uncertain(const char *id_digest);
+int authority_cert_is_blacklisted(const authority_cert_t *cert);
+
+void authority_cert_free_(authority_cert_t *cert);
+#define authority_cert_free(cert) \
+ FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
+
+MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
+MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
+ (const char *digest));
+MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
+ (const char *digest));
+MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
+ (const char *id_digest, const char *sk_digest));
+
+void authcert_free_all(void);
+
+#endif
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
new file mode 100644
index 000000000..8f78df521
--- /dev/null
+++ b/src/feature/nodelist/dirlist.c
@@ -0,0 +1,421 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirlist.c
+ * \brief Code to maintain our lists of directory authorities and
+ * fallback directories.
+ *
+ * For the directory authorities, we have a list containing the public
+ * identity key, and contact points, for each authority. The
+ * authorities receive descriptors from relays, and publish consensuses,
+ * descriptors, and microdescriptors. This list is pre-configured.
+ *
+ * Fallback directories are well-known, stable, but untrusted directory
+ * caches that clients which have not yet bootstrapped can use to get
+ * their first networkstatus consensus, in order to find out where the
+ * Tor network really is. This list is pre-configured in
+ * fallback_dirs.inc. Every authority also serves as a fallback.
+ *
+ * Both fallback directories and directory authorities are are
+ * represented by a dir_server_t.
+ */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/control/control.h"
+#include "feature/dircache/directory.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "lib/net/resolve.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/node_st.h"
+
+/** Global list of a dir_server_t object for each directory
+ * authority. */
+static smartlist_t *trusted_dir_servers = NULL;
+/** Global list of dir_server_t objects for all directory authorities
+ * and all fallback directory servers. */
+static smartlist_t *fallback_dir_servers = NULL;
+
+/** Return the number of directory authorities whose type matches some bit set
+ * in <b>type</b> */
+int
+get_n_authorities(dirinfo_type_t type)
+{
+ int n = 0;
+ if (!trusted_dir_servers)
+ return 0;
+ SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+ if (ds->type & type)
+ ++n);
+ return n;
+}
+
+/** Return a smartlist containing a list of dir_server_t * for all
+ * known trusted dirservers. Callers must not modify the list or its
+ * contents.
+ */
+smartlist_t *
+router_get_trusted_dir_servers_mutable(void)
+{
+ if (!trusted_dir_servers)
+ trusted_dir_servers = smartlist_new();
+
+ return trusted_dir_servers;
+}
+
+smartlist_t *
+router_get_fallback_dir_servers_mutable(void)
+{
+ if (!fallback_dir_servers)
+ fallback_dir_servers = smartlist_new();
+
+ return fallback_dir_servers;
+}
+
+const smartlist_t *
+router_get_trusted_dir_servers(void)
+{
+ return router_get_trusted_dir_servers_mutable();
+}
+
+const smartlist_t *
+router_get_fallback_dir_servers(void)
+{
+ return router_get_fallback_dir_servers_mutable();
+}
+
+/** Reset all internal variables used to count failed downloads of network
+ * status objects. */
+void
+router_reset_status_download_failures(void)
+{
+ mark_all_dirservers_up(fallback_dir_servers);
+}
+
+/** Return the dir_server_t for the directory authority whose identity
+ * key hashes to <b>digest</b>, or NULL if no such authority is known.
+ */
+dir_server_t *
+router_get_trusteddirserver_by_digest(const char *digest)
+{
+ if (!trusted_dir_servers)
+ return NULL;
+
+ SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+ {
+ if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+ return ds;
+ });
+
+ return NULL;
+}
+
+/** Return the dir_server_t for the fallback dirserver whose identity
+ * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
+ * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
+ * and UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+dir_server_t *
+router_get_fallback_dirserver_by_digest(const char *digest)
+{
+ if (!fallback_dir_servers)
+ return NULL;
+
+ if (!digest)
+ return NULL;
+
+ SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
+ {
+ if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+ return ds;
+ });
+
+ return NULL;
+}
+
+/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
+ * or 0 if no such fallback is in the list of fallback_dir_servers.
+ * (fallback_dir_servers is affected by the FallbackDir and
+ * UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+int
+router_digest_is_fallback_dir(const char *digest)
+{
+ return (router_get_fallback_dirserver_by_digest(digest) != NULL);
+}
+
+/** Return the dir_server_t for the directory authority whose
+ * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
+ * is known.
+ */
+MOCK_IMPL(dir_server_t *,
+trusteddirserver_get_by_v3_auth_digest, (const char *digest))
+{
+ if (!trusted_dir_servers)
+ return NULL;
+
+ SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+ {
+ if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
+ (ds->type & V3_DIRINFO))
+ return ds;
+ });
+
+ return NULL;
+}
+
+/** Mark as running every dir_server_t in <b>server_list</b>. */
+void
+mark_all_dirservers_up(smartlist_t *server_list)
+{
+ if (server_list) {
+ SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
+ routerstatus_t *rs;
+ node_t *node;
+ dir->is_running = 1;
+ node = node_get_mutable_by_id(dir->digest);
+ if (node)
+ node->is_running = 1;
+ rs = router_get_mutable_consensus_status_by_id(dir->digest);
+ if (rs) {
+ rs->last_dir_503_at = 0;
+ control_event_networkstatus_changed_single(rs);
+ }
+ } SMARTLIST_FOREACH_END(dir);
+ }
+ router_dir_info_changed();
+}
+
+/** Return true iff <b>digest</b> is the digest of the identity key of a
+ * trusted directory matching at least one bit of <b>type</b>. If <b>type</b>
+ * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
+int
+router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
+{
+ if (!trusted_dir_servers)
+ return 0;
+ if (authdir_mode(get_options()) && router_digest_is_me(digest))
+ return 1;
+ SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
+ if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
+ return (!type) || ((type & ent->type) != 0);
+ });
+ return 0;
+}
+
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
+ * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL,
+ * add ourself. If <b>is_authority</b>, this is a directory authority. Return
+ * the new directory server entry on success or NULL on failure. */
+static dir_server_t *
+dir_server_new(int is_authority,
+ const char *nickname,
+ const tor_addr_t *addr,
+ const char *hostname,
+ uint16_t dir_port, uint16_t or_port,
+ const tor_addr_port_t *addrport_ipv6,
+ const char *digest, const char *v3_auth_digest,
+ dirinfo_type_t type,
+ double weight)
+{
+ dir_server_t *ent;
+ uint32_t a;
+ char *hostname_ = NULL;
+
+ tor_assert(digest);
+
+ if (weight < 0)
+ return NULL;
+
+ if (tor_addr_family(addr) == AF_INET)
+ a = tor_addr_to_ipv4h(addr);
+ else
+ return NULL;
+
+ if (!hostname)
+ hostname_ = tor_addr_to_str_dup(addr);
+ else
+ hostname_ = tor_strdup(hostname);
+
+ ent = tor_malloc_zero(sizeof(dir_server_t));
+ ent->nickname = nickname ? tor_strdup(nickname) : NULL;
+ ent->address = hostname_;
+ ent->addr = a;
+ ent->dir_port = dir_port;
+ ent->or_port = or_port;
+ ent->is_running = 1;
+ ent->is_authority = is_authority;
+ ent->type = type;
+ ent->weight = weight;
+ if (addrport_ipv6) {
+ if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
+ log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
+ tor_addr_make_unspec(&ent->ipv6_addr);
+ } else {
+ tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
+ ent->ipv6_orport = addrport_ipv6->port;
+ }
+ } else {
+ tor_addr_make_unspec(&ent->ipv6_addr);
+ }
+
+ memcpy(ent->digest, digest, DIGEST_LEN);
+ if (v3_auth_digest && (type & V3_DIRINFO))
+ memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
+
+ if (nickname)
+ tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
+ nickname, hostname_, (int)dir_port);
+ else
+ tor_asprintf(&ent->description, "directory server at %s:%d",
+ hostname_, (int)dir_port);
+
+ ent->fake_status.addr = ent->addr;
+ tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
+ memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
+ if (nickname)
+ strlcpy(ent->fake_status.nickname, nickname,
+ sizeof(ent->fake_status.nickname));
+ else
+ ent->fake_status.nickname[0] = '\0';
+ ent->fake_status.dir_port = ent->dir_port;
+ ent->fake_status.or_port = ent->or_port;
+ ent->fake_status.ipv6_orport = ent->ipv6_orport;
+
+ return ent;
+}
+
+/** Create an authoritative directory server at
+ * <b>address</b>:<b>port</b>, with identity key <b>digest</b>. If
+ * <b>address</b> is NULL, add ourself. Return the new trusted directory
+ * server entry on success or NULL if we couldn't add it. */
+dir_server_t *
+trusted_dir_server_new(const char *nickname, const char *address,
+ uint16_t dir_port, uint16_t or_port,
+ const tor_addr_port_t *ipv6_addrport,
+ const char *digest, const char *v3_auth_digest,
+ dirinfo_type_t type, double weight)
+{
+ uint32_t a;
+ tor_addr_t addr;
+ char *hostname=NULL;
+ dir_server_t *result;
+
+ if (!address) { /* The address is us; we should guess. */
+ if (resolve_my_address(LOG_WARN, get_options(),
+ &a, NULL, &hostname) < 0) {
+ log_warn(LD_CONFIG,
+ "Couldn't find a suitable address when adding ourself as a "
+ "trusted directory server.");
+ return NULL;
+ }
+ if (!hostname)
+ hostname = tor_dup_ip(a);
+ } else {
+ if (tor_lookup_hostname(address, &a)) {
+ log_warn(LD_CONFIG,
+ "Unable to lookup address for directory server at '%s'",
+ address);
+ return NULL;
+ }
+ hostname = tor_strdup(address);
+ }
+ tor_addr_from_ipv4h(&addr, a);
+
+ result = dir_server_new(1, nickname, &addr, hostname,
+ dir_port, or_port,
+ ipv6_addrport,
+ digest,
+ v3_auth_digest, type, weight);
+ tor_free(hostname);
+ return result;
+}
+
+/** Return a new dir_server_t for a fallback directory server at
+ * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
+ * <b>id_digest</b> */
+dir_server_t *
+fallback_dir_server_new(const tor_addr_t *addr,
+ uint16_t dir_port, uint16_t or_port,
+ const tor_addr_port_t *addrport_ipv6,
+ const char *id_digest, double weight)
+{
+ return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
+ addrport_ipv6,
+ id_digest,
+ NULL, ALL_DIRINFO, weight);
+}
+
+/** Add a directory server to the global list(s). */
+void
+dir_server_add(dir_server_t *ent)
+{
+ if (!trusted_dir_servers)
+ trusted_dir_servers = smartlist_new();
+ if (!fallback_dir_servers)
+ fallback_dir_servers = smartlist_new();
+
+ if (ent->is_authority)
+ smartlist_add(trusted_dir_servers, ent);
+
+ smartlist_add(fallback_dir_servers, ent);
+ router_dir_info_changed();
+}
+
+#define dir_server_free(val) \
+ FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
+
+/** Free storage held in <b>ds</b>. */
+static void
+dir_server_free_(dir_server_t *ds)
+{
+ if (!ds)
+ return;
+
+ tor_free(ds->nickname);
+ tor_free(ds->description);
+ tor_free(ds->address);
+ tor_free(ds);
+}
+
+/** Remove all members from the list of dir servers. */
+void
+clear_dir_servers(void)
+{
+ if (fallback_dir_servers) {
+ SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
+ dir_server_free(ent));
+ smartlist_clear(fallback_dir_servers);
+ } else {
+ fallback_dir_servers = smartlist_new();
+ }
+ if (trusted_dir_servers) {
+ smartlist_clear(trusted_dir_servers);
+ } else {
+ trusted_dir_servers = smartlist_new();
+ }
+ router_dir_info_changed();
+}
+
+void
+dirlist_free_all(void)
+{
+ clear_dir_servers();
+ smartlist_free(trusted_dir_servers);
+ smartlist_free(fallback_dir_servers);
+ trusted_dir_servers = fallback_dir_servers = NULL;
+}
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
new file mode 100644
index 000000000..6baa5686c
--- /dev/null
+++ b/src/feature/nodelist/dirlist.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirlist.h
+ * \brief Header file for dirlist.c
+ **/
+
+#ifndef TOR_DIRLIST_H
+#define TOR_DIRLIST_H
+
+int get_n_authorities(dirinfo_type_t type);
+const smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_fallback_dir_servers(void);
+smartlist_t *router_get_trusted_dir_servers_mutable(void);
+smartlist_t *router_get_fallback_dir_servers_mutable(void);
+void mark_all_dirservers_up(smartlist_t *server_list);
+
+dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
+dir_server_t *router_get_fallback_dirserver_by_digest(
+ const char *digest);
+int router_digest_is_fallback_dir(const char *digest);
+MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
+ (const char *d));
+
+int router_digest_is_trusted_dir_type(const char *digest,
+ dirinfo_type_t type);
+#define router_digest_is_trusted_dir(d) \
+ router_digest_is_trusted_dir_type((d), NO_DIRINFO)
+
+dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
+ uint16_t dir_port, uint16_t or_port,
+ const tor_addr_port_t *addrport_ipv6,
+ const char *digest, const char *v3_auth_digest,
+ dirinfo_type_t type, double weight);
+dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
+ uint16_t dir_port, uint16_t or_port,
+ const tor_addr_port_t *addrport_ipv6,
+ const char *id_digest, double weight);
+void dir_server_add(dir_server_t *ent);
+
+void clear_dir_servers(void);
+void dirlist_free_all(void);
+
+#endif
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
index aa5e6b1db..203d079b1 100644
--- a/src/feature/nodelist/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -22,6 +22,7 @@
#include "feature/nodelist/nodelist.h"
#include "core/or/policies.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index 81da9756a..d4340b6ca 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -63,6 +63,9 @@
#include "core/or/protover.h"
#include "core/or/relay.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "core/or/scheduler.h"
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
new file mode 100644
index 000000000..a9f045d45
--- /dev/null
+++ b/src/feature/nodelist/node_select.c
@@ -0,0 +1,1108 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file node_select.c
+ * \brief Code to choose nodes randomly based on restrictions and
+ * weighted probabilities.
+ **/
+
+#define NODE_SELECT_PRIVATE
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dircache/directory.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/math/fp.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+static int compute_weighted_bandwidths(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule,
+ double **bandwidths_out,
+ double *total_bandwidth_out);
+static const routerstatus_t *router_pick_trusteddirserver_impl(
+ const smartlist_t *sourcelist, dirinfo_type_t auth,
+ int flags, int *n_busy_out);
+static const routerstatus_t *router_pick_dirserver_generic(
+ smartlist_t *sourcelist,
+ dirinfo_type_t type, int flags);
+
+/** Try to find a running dirserver that supports operations of <b>type</b>.
+ *
+ * If there are no running dirservers in our routerlist and the
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
+ *
+ * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
+ * dirservers that we can't reach.
+ *
+ * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
+ * (if we're a dirserver).
+ *
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
+ */
+const routerstatus_t *
+router_pick_directory_server(dirinfo_type_t type, int flags)
+{
+ int busy = 0;
+ const routerstatus_t *choice;
+
+ choice = router_pick_directory_server_impl(type, flags, &busy);
+ if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+ return choice;
+
+ if (busy) {
+ /* If the reason that we got no server is that servers are "busy",
+ * we must be excluding good servers because we already have serverdesc
+ * fetches with them. Do not mark down servers up because of this. */
+ tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+ PDS_NO_EXISTING_MICRODESC_FETCH)));
+ return NULL;
+ }
+
+ log_info(LD_DIR,
+ "No reachable router entries for dirservers. "
+ "Trying them all again.");
+ /* mark all fallback directory mirrors as up again */
+ router_reset_status_download_failures();
+ /* try again */
+ choice = router_pick_directory_server_impl(type, flags, NULL);
+ return choice;
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_dirserver_generic(smartlist_t *sourcelist,
+ dirinfo_type_t type, int flags)
+{
+ const routerstatus_t *choice;
+ int busy = 0;
+
+ if (smartlist_len(sourcelist) == 1) {
+ /* If there's only one choice, then we should disable the logic that
+ * would otherwise prevent us from choosing ourself. */
+ flags |= PDS_ALLOW_SELF;
+ }
+
+ choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
+ if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+ return choice;
+ if (busy) {
+ /* If the reason that we got no server is that servers are "busy",
+ * we must be excluding good servers because we already have serverdesc
+ * fetches with them. Do not mark down servers up because of this. */
+ tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+ PDS_NO_EXISTING_MICRODESC_FETCH)));
+ return NULL;
+ }
+
+ log_info(LD_DIR,
+ "No dirservers are reachable. Trying them all again.");
+ mark_all_dirservers_up(sourcelist);
+ return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
+}
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
+ * Must be called before RETRY_WITHOUT_EXCLUDE().
+ *
+ * If we got no result, and we are applying IP preferences, and we are a
+ * client that could use an alternate IP version, try again with the
+ * opposite preferences. */
+#define RETRY_ALTERNATE_IP_VERSION(retry_label) \
+ STMT_BEGIN \
+ if (result == NULL && try_ip_pref && options->ClientUseIPv4 \
+ && fascist_firewall_use_ipv6(options) && !server_mode(options) \
+ && !n_busy) { \
+ n_excluded = 0; \
+ n_busy = 0; \
+ try_ip_pref = 0; \
+ goto retry_label; \
+ } \
+ STMT_END \
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
+ * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
+ *
+ * If we got no result, and we are excluding nodes, and StrictNodes is
+ * not set, try again without excluding nodes. */
+#define RETRY_WITHOUT_EXCLUDE(retry_label) \
+ STMT_BEGIN \
+ if (result == NULL && try_excluding && !options->StrictNodes \
+ && n_excluded && !n_busy) { \
+ try_excluding = 0; \
+ n_excluded = 0; \
+ n_busy = 0; \
+ try_ip_pref = 1; \
+ goto retry_label; \
+ } \
+ STMT_END
+
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity) \
+ STMT_BEGIN \
+ int is_trusted_extrainfo = router_digest_is_trusted_dir_type( \
+ (identity), EXTRAINFO_DIRINFO); \
+ if (((type) & EXTRAINFO_DIRINFO) && \
+ !router_supports_extrainfo((identity), is_trusted_extrainfo)) \
+ continue; \
+ STMT_END
+
+#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
+#endif
+
+/* Log a message if rs is not found or not a preferred address */
+static void
+router_picked_poor_directory_log(const routerstatus_t *rs)
+{
+ const networkstatus_t *usable_consensus;
+ usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
+ usable_consensus_flavor());
+
+#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+ /* Don't log early in the bootstrap process, it's normal to pick from a
+ * small pool of nodes. Of course, this won't help if we're trying to
+ * diagnose bootstrap issues. */
+ if (!smartlist_len(nodelist_get_list()) || !usable_consensus
+ || !router_have_minimum_dir_info()) {
+ return;
+ }
+#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
+
+ /* We couldn't find a node, or the one we have doesn't fit our preferences.
+ * Sometimes this is normal, sometimes it can be a reachability issue. */
+ if (!rs) {
+ /* This happens a lot, so it's at debug level */
+ log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
+ "we couldn't find a directory that fit our criteria. "
+ "Perhaps we will succeed next time with less strict criteria.");
+ } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
+ && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
+ ) {
+ /* This is rare, and might be interesting to users trying to diagnose
+ * connection issues on dual-stack machines. */
+ log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
+ "addresses for launching an outgoing connection: "
+ "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
+ routerstatus_describe(rs),
+ fmt_addr32(rs->addr), rs->or_port,
+ rs->dir_port, fmt_addr(&rs->ipv6_addr),
+ rs->ipv6_orport, rs->dir_port);
+ }
+}
+
+#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+
+/* Check if we already have a directory fetch from ap, for serverdesc
+ * (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
+ */
+STATIC int
+router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
+ int microdesc)
+{
+ if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
+ return 0;
+ }
+
+ /* XX/teor - we're not checking tunnel connections here, see #17848
+ */
+ if (serverdesc && (
+ connection_get_by_type_addr_port_purpose(
+ CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
+ || connection_get_by_type_addr_port_purpose(
+ CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
+ return 1;
+ }
+
+ if (microdesc && (
+ connection_get_by_type_addr_port_purpose(
+ CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
+ return 1;
+ }
+
+ return 0;
+}
+
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ */
+static int
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+ const tor_addr_t *ipv6_addr,
+ uint16_t dir_port,
+ int serverdesc,
+ int microdesc)
+{
+ tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
+
+ /* Assume IPv6 DirPort is the same as IPv4 DirPort */
+ tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+ ipv4_dir_ap.port = dir_port;
+ tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+ ipv6_dir_ap.port = dir_port;
+
+ return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
+ || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
+}
+
+/** Pick a random running valid directory server/mirror from our
+ * routerlist. Arguments are as for router_pick_directory_server(), except:
+ *
+ * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
+ * directories that we excluded for no other reason than
+ * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
+ */
+STATIC const routerstatus_t *
+router_pick_directory_server_impl(dirinfo_type_t type, int flags,
+ int *n_busy_out)
+{
+ const or_options_t *options = get_options();
+ const node_t *result;
+ smartlist_t *direct, *tunnel;
+ smartlist_t *trusted_direct, *trusted_tunnel;
+ smartlist_t *overloaded_direct, *overloaded_tunnel;
+ time_t now = time(NULL);
+ const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+ const int requireother = ! (flags & PDS_ALLOW_SELF);
+ const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+ const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+ const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+ int try_excluding = 1, n_excluded = 0, n_busy = 0;
+ int try_ip_pref = 1;
+
+ if (!consensus)
+ return NULL;
+
+ retry_search:
+
+ direct = smartlist_new();
+ tunnel = smartlist_new();
+ trusted_direct = smartlist_new();
+ trusted_tunnel = smartlist_new();
+ overloaded_direct = smartlist_new();
+ overloaded_tunnel = smartlist_new();
+
+ const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+ const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+ const int must_have_or = directory_must_use_begindir(options);
+
+ /* Find all the running dirservers we know about. */
+ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+ int is_trusted;
+ int is_overloaded;
+ const routerstatus_t *status = node->rs;
+ const country_t country = node->country;
+ if (!status)
+ continue;
+
+ if (!node->is_running || !node_is_dir(node) || !node->is_valid)
+ continue;
+ if (requireother && router_digest_is_me(node->identity))
+ continue;
+
+ SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
+ if (try_excluding &&
+ routerset_contains_routerstatus(options->ExcludeNodes, status,
+ country)) {
+ ++n_excluded;
+ continue;
+ }
+
+ if (router_is_already_dir_fetching_(status->addr,
+ &status->ipv6_addr,
+ status->dir_port,
+ no_serverdesc_fetching,
+ no_microdesc_fetching)) {
+ ++n_busy;
+ continue;
+ }
+
+ is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+ is_trusted = router_digest_is_trusted_dir(node->identity);
+
+ /* Clients use IPv6 addresses if the server has one and the client
+ * prefers IPv6.
+ * Add the router if its preferred address and port are reachable.
+ * If we don't get any routers, we'll try again with the non-preferred
+ * address for each router (if any). (To ensure correct load-balancing
+ * we try routers that only have one address both times.)
+ */
+ if (!fascistfirewall || skip_or_fw ||
+ fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
+ try_ip_pref))
+ smartlist_add(is_trusted ? trusted_tunnel :
+ is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
+ else if (!must_have_or && (skip_dir_fw ||
+ fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
+ try_ip_pref)))
+ smartlist_add(is_trusted ? trusted_direct :
+ is_overloaded ? overloaded_direct : direct, (void*)node);
+ } SMARTLIST_FOREACH_END(node);
+
+ if (smartlist_len(tunnel)) {
+ result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
+ } else if (smartlist_len(overloaded_tunnel)) {
+ result = node_sl_choose_by_bandwidth(overloaded_tunnel,
+ WEIGHT_FOR_DIR);
+ } else if (smartlist_len(trusted_tunnel)) {
+ /* FFFF We don't distinguish between trusteds and overloaded trusteds
+ * yet. Maybe one day we should. */
+ /* FFFF We also don't load balance over authorities yet. I think this
+ * is a feature, but it could easily be a bug. -RD */
+ result = smartlist_choose(trusted_tunnel);
+ } else if (smartlist_len(direct)) {
+ result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
+ } else if (smartlist_len(overloaded_direct)) {
+ result = node_sl_choose_by_bandwidth(overloaded_direct,
+ WEIGHT_FOR_DIR);
+ } else {
+ result = smartlist_choose(trusted_direct);
+ }
+ smartlist_free(direct);
+ smartlist_free(tunnel);
+ smartlist_free(trusted_direct);
+ smartlist_free(trusted_tunnel);
+ smartlist_free(overloaded_direct);
+ smartlist_free(overloaded_tunnel);
+
+ RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+ RETRY_WITHOUT_EXCLUDE(retry_search);
+
+ if (n_busy_out)
+ *n_busy_out = n_busy;
+
+ router_picked_poor_directory_log(result ? result->rs : NULL);
+
+ return result ? result->rs : NULL;
+}
+
+/** Given an array of double/uint64_t unions that are currently being used as
+ * doubles, convert them to uint64_t, and try to scale them linearly so as to
+ * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
+ * the sum of all elements in the array _before_ scaling. */
+STATIC void
+scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
+ int n_entries,
+ uint64_t *total_out)
+{
+ double total = 0.0;
+ double scale_factor = 0.0;
+ int i;
+
+ for (i = 0; i < n_entries; ++i)
+ total += entries_in[i];
+
+ if (total > 0.0) {
+ scale_factor = ((double)INT64_MAX) / total;
+ scale_factor /= 4.0; /* make sure we're very far away from overflowing */
+ }
+
+ for (i = 0; i < n_entries; ++i)
+ entries_out[i] = tor_llround(entries_in[i] * scale_factor);
+
+ if (total_out)
+ *total_out = (uint64_t) total;
+}
+
+/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
+ * choosing each element with a probability proportional to its (uint64_t)
+ * value, and return the index of that element. If all elements are 0, choose
+ * an index at random. Return -1 on error.
+ */
+STATIC int
+choose_array_element_by_weight(const uint64_t *entries, int n_entries)
+{
+ int i;
+ uint64_t rand_val;
+ uint64_t total = 0;
+
+ for (i = 0; i < n_entries; ++i)
+ total += entries[i];
+
+ if (n_entries < 1)
+ return -1;
+
+ if (total == 0)
+ return crypto_rand_int(n_entries);
+
+ tor_assert(total < INT64_MAX);
+
+ rand_val = crypto_rand_uint64(total);
+
+ return select_array_member_cumulative_timei(
+ entries, n_entries, total, rand_val);
+}
+
+/** Return bw*1000, unless bw*1000 would overflow, in which case return
+ * INT32_MAX. */
+static inline int32_t
+kb_to_bytes(uint32_t bw)
+{
+ return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
+}
+
+/** Helper function:
+ * choose a random element of smartlist <b>sl</b> of nodes, weighted by
+ * the advertised bandwidth of each element using the consensus
+ * bandwidth weights.
+ *
+ * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
+ * nodes' bandwidth equally regardless of their Exit status, since there may
+ * be some in the list because they exit to obscure ports. If
+ * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
+ * exit-node's bandwidth less depending on the smallness of the fraction of
+ * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
+ * guard node: consider all guard's bandwidth equally. Otherwise, weight
+ * guards proportionally less.
+ */
+static const node_t *
+smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule)
+{
+ double *bandwidths_dbl=NULL;
+ uint64_t *bandwidths_u64=NULL;
+
+ if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
+ return NULL;
+
+ bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
+ scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
+ smartlist_len(sl), NULL);
+
+ {
+ int idx = choose_array_element_by_weight(bandwidths_u64,
+ smartlist_len(sl));
+ tor_free(bandwidths_dbl);
+ tor_free(bandwidths_u64);
+ return idx < 0 ? NULL : smartlist_get(sl, idx);
+ }
+}
+
+/** When weighting bridges, enforce these values as lower and upper
+ * bound for believable bandwidth, because there is no way for us
+ * to verify a bridge's bandwidth currently. */
+#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */
+#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, making sure to stay within the
+ * interval between bridge-min-believe-bw and
+ * bridge-max-believe-bw. */
+static uint32_t
+bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
+{
+ uint32_t result = router->bandwidthcapacity;
+ if (result > router->bandwidthrate)
+ result = router->bandwidthrate;
+ if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
+ result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
+ else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
+ result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
+ return result;
+}
+
+/** Given a list of routers and a weighting rule as in
+ * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
+ * values for each node and store them in a freshly allocated
+ * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
+ * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
+ * of all the bandwidths.
+ * Return 0 on success, -1 on failure. */
+static int
+compute_weighted_bandwidths(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule,
+ double **bandwidths_out,
+ double *total_bandwidth_out)
+{
+ int64_t weight_scale;
+ double Wg = -1, Wm = -1, We = -1, Wd = -1;
+ double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
+ guardfraction_bandwidth_t guardfraction_bw;
+ double *bandwidths = NULL;
+ double total_bandwidth = 0.0;
+
+ tor_assert(sl);
+ tor_assert(bandwidths_out);
+
+ /* Can't choose exit and guard at same time */
+ tor_assert(rule == NO_WEIGHTING ||
+ rule == WEIGHT_FOR_EXIT ||
+ rule == WEIGHT_FOR_GUARD ||
+ rule == WEIGHT_FOR_MID ||
+ rule == WEIGHT_FOR_DIR);
+
+ *bandwidths_out = NULL;
+
+ if (total_bandwidth_out) {
+ *total_bandwidth_out = 0.0;
+ }
+
+ if (smartlist_len(sl) == 0) {
+ log_info(LD_CIRC,
+ "Empty routerlist passed in to consensus weight node "
+ "selection for rule %s",
+ bandwidth_weight_rule_to_string(rule));
+ return -1;
+ }
+
+ weight_scale = networkstatus_get_weight_scale_param(NULL);
+
+ if (rule == WEIGHT_FOR_GUARD) {
+ Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
+ Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
+ We = 0;
+ Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
+
+ Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+ Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+ Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+ Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+ } else if (rule == WEIGHT_FOR_MID) {
+ Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
+ Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
+ We = networkstatus_get_bw_weight(NULL, "Wme", -1);
+ Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
+
+ Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+ Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+ Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+ Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+ } else if (rule == WEIGHT_FOR_EXIT) {
+ // Guards CAN be exits if they have weird exit policies
+ // They are d then I guess...
+ We = networkstatus_get_bw_weight(NULL, "Wee", -1);
+ Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
+ Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
+ Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
+
+ Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+ Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+ Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+ Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+ } else if (rule == WEIGHT_FOR_DIR) {
+ We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
+ Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
+ Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
+ Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
+
+ Wgb = Wmb = Web = Wdb = weight_scale;
+ } else if (rule == NO_WEIGHTING) {
+ Wg = Wm = We = Wd = weight_scale;
+ Wgb = Wmb = Web = Wdb = weight_scale;
+ }
+
+ if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
+ || Web < 0) {
+ log_debug(LD_CIRC,
+ "Got negative bandwidth weights. Defaulting to naive selection"
+ " algorithm.");
+ Wg = Wm = We = Wd = weight_scale;
+ Wgb = Wmb = Web = Wdb = weight_scale;
+ }
+
+ Wg /= weight_scale;
+ Wm /= weight_scale;
+ We /= weight_scale;
+ Wd /= weight_scale;
+
+ Wgb /= weight_scale;
+ Wmb /= weight_scale;
+ Web /= weight_scale;
+ Wdb /= weight_scale;
+
+ bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
+
+ // Cycle through smartlist and total the bandwidth.
+ static int warned_missing_bw = 0;
+ SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+ int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
+ double weight = 1;
+ double weight_without_guard_flag = 0; /* Used for guardfraction */
+ double final_weight = 0;
+ is_exit = node->is_exit && ! node->is_bad_exit;
+ is_guard = node->is_possible_guard;
+ is_dir = node_is_dir(node);
+ if (node->rs) {
+ if (!node->rs->has_bandwidth) {
+ /* This should never happen, unless all the authorities downgrade
+ * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
+ if (! warned_missing_bw) {
+ log_warn(LD_BUG,
+ "Consensus is missing some bandwidths. Using a naive "
+ "router selection algorithm");
+ warned_missing_bw = 1;
+ }
+ this_bw = 30000; /* Chosen arbitrarily */
+ } else {
+ this_bw = kb_to_bytes(node->rs->bandwidth_kb);
+ }
+ } else if (node->ri) {
+ /* bridge or other descriptor not in our consensus */
+ this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
+ } else {
+ /* We can't use this one. */
+ continue;
+ }
+
+ if (is_guard && is_exit) {
+ weight = (is_dir ? Wdb*Wd : Wd);
+ weight_without_guard_flag = (is_dir ? Web*We : We);
+ } else if (is_guard) {
+ weight = (is_dir ? Wgb*Wg : Wg);
+ weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
+ } else if (is_exit) {
+ weight = (is_dir ? Web*We : We);
+ } else { // middle
+ weight = (is_dir ? Wmb*Wm : Wm);
+ }
+ /* These should be impossible; but overflows here would be bad, so let's
+ * make sure. */
+ if (this_bw < 0)
+ this_bw = 0;
+ if (weight < 0.0)
+ weight = 0.0;
+ if (weight_without_guard_flag < 0.0)
+ weight_without_guard_flag = 0.0;
+
+ /* If guardfraction information is available in the consensus, we
+ * want to calculate this router's bandwidth according to its
+ * guardfraction. Quoting from proposal236:
+ *
+ * Let Wpf denote the weight from the 'bandwidth-weights' line a
+ * client would apply to N for position p if it had the guard
+ * flag, Wpn the weight if it did not have the guard flag, and B the
+ * measured bandwidth of N in the consensus. Then instead of choosing
+ * N for position p proportionally to Wpf*B or Wpn*B, clients should
+ * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+ */
+ if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
+ /* XXX The assert should actually check for is_guard. However,
+ * that crashes dirauths because of #13297. This should be
+ * equivalent: */
+ tor_assert(node->rs->is_possible_guard);
+
+ guard_get_guardfraction_bandwidth(&guardfraction_bw,
+ this_bw,
+ node->rs->guardfraction_percentage);
+
+ /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
+ final_weight =
+ guardfraction_bw.guard_bw * weight +
+ guardfraction_bw.non_guard_bw * weight_without_guard_flag;
+
+ log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
+ node->rs->nickname, final_weight, weight*this_bw,
+ bandwidth_weight_rule_to_string(rule));
+ } else { /* no guardfraction information. calculate the weight normally. */
+ final_weight = weight*this_bw;
+ }
+
+ bandwidths[node_sl_idx] = final_weight;
+ total_bandwidth += final_weight;
+ } SMARTLIST_FOREACH_END(node);
+
+ log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
+ "on weights "
+ "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
+ bandwidth_weight_rule_to_string(rule),
+ Wg, Wm, We, Wd, total_bandwidth);
+
+ *bandwidths_out = bandwidths;
+
+ if (total_bandwidth_out) {
+ *total_bandwidth_out = total_bandwidth;
+ }
+
+ return 0;
+}
+
+/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
+ * by their weighted bandwidths with rule <b>rule</b>, for which we have
+ * descriptors.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect
+ * to it indirectly, or use it as if we were connecting to it indirectly. */
+double
+frac_nodes_with_descriptors(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule,
+ int for_direct_conn)
+{
+ double *bandwidths = NULL;
+ double total, present;
+
+ if (smartlist_len(sl) == 0)
+ return 0.0;
+
+ if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
+ total <= 0.0) {
+ int n_with_descs = 0;
+ SMARTLIST_FOREACH(sl, const node_t *, node, {
+ if (node_has_preferred_descriptor(node, for_direct_conn))
+ n_with_descs++;
+ });
+ tor_free(bandwidths);
+ return ((double)n_with_descs) / smartlist_len(sl);
+ }
+
+ present = 0.0;
+ SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+ if (node_has_preferred_descriptor(node, for_direct_conn))
+ present += bandwidths[node_sl_idx];
+ } SMARTLIST_FOREACH_END(node);
+
+ tor_free(bandwidths);
+
+ return present / total;
+}
+
+/** Choose a random element of status list <b>sl</b>, weighted by
+ * the advertised bandwidth of each node */
+const node_t *
+node_sl_choose_by_bandwidth(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule)
+{ /*XXXX MOVE */
+ return smartlist_choose_node_by_bandwidth_weights(sl, rule);
+}
+
+/** Given a <b>router</b>, add every node_t in its family (including the
+ * node itself!) to <b>sl</b>.
+ *
+ * Note the type mismatch: This function takes a routerinfo, but adds nodes
+ * to the smartlist!
+ */
+static void
+routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
+{
+ /* XXXX MOVE ? */
+ node_t fake_node;
+ const node_t *node = node_get_by_id(router->cache_info.identity_digest);
+ if (node == NULL) {
+ memset(&fake_node, 0, sizeof(fake_node));
+ fake_node.ri = (routerinfo_t *)router;
+ memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
+ node = &fake_node;
+ }
+ nodelist_add_node_and_family(sl, node);
+}
+
+/** Return a random running node from the nodelist. Never
+ * pick a node that is in
+ * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
+ * even if they are the only nodes available.
+ * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
+ * a minimum uptime, return one of those.
+ * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
+ * advertised capacity of each router.
+ * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
+ * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
+ * picking an exit node, otherwise we weight bandwidths for picking a relay
+ * node (that is, possibly discounting exit nodes).
+ * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
+ * have a routerinfo or microdescriptor -- that is, enough info to be
+ * used to build a circuit.
+ * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
+ * have an address that is preferred by the ClientPreferIPv6ORPort setting
+ * (regardless of this flag, we exclude nodes that aren't allowed by the
+ * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
+ */
+const node_t *
+router_choose_random_node(smartlist_t *excludedsmartlist,
+ routerset_t *excludedset,
+ router_crn_flags_t flags)
+{ /* XXXX MOVE */
+ const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+ const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+ const int need_guard = (flags & CRN_NEED_GUARD) != 0;
+ const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
+ const int need_desc = (flags & CRN_NEED_DESC) != 0;
+ const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
+ const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+ const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
+
+ smartlist_t *sl=smartlist_new(),
+ *excludednodes=smartlist_new();
+ const node_t *choice = NULL;
+ const routerinfo_t *r;
+ bandwidth_weight_rule_t rule;
+
+ tor_assert(!(weight_for_exit && need_guard));
+ rule = weight_for_exit ? WEIGHT_FOR_EXIT :
+ (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
+
+ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+ if (node_allows_single_hop_exits(node)) {
+ /* Exclude relays that allow single hop exit circuits. This is an
+ * obsolete option since 0.2.9.2-alpha and done by default in
+ * 0.3.1.0-alpha. */
+ smartlist_add(excludednodes, node);
+ } else if (rendezvous_v3 &&
+ !node_supports_v3_rendezvous_point(node)) {
+ /* Exclude relays that do not support to rendezvous for a hidden service
+ * version 3. */
+ smartlist_add(excludednodes, node);
+ }
+ } SMARTLIST_FOREACH_END(node);
+
+ /* If the node_t is not found we won't be to exclude ourself but we
+ * won't be able to pick ourself in router_choose_random_node() so
+ * this is fine to at least try with our routerinfo_t object. */
+ if ((r = router_get_my_routerinfo()))
+ routerlist_add_node_and_family(excludednodes, r);
+
+ router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
+ need_guard, need_desc, pref_addr,
+ direct_conn);
+ log_debug(LD_CIRC,
+ "We found %d running nodes.",
+ smartlist_len(sl));
+
+ smartlist_subtract(sl,excludednodes);
+ log_debug(LD_CIRC,
+ "We removed %d excludednodes, leaving %d nodes.",
+ smartlist_len(excludednodes),
+ smartlist_len(sl));
+
+ if (excludedsmartlist) {
+ smartlist_subtract(sl,excludedsmartlist);
+ log_debug(LD_CIRC,
+ "We removed %d excludedsmartlist, leaving %d nodes.",
+ smartlist_len(excludedsmartlist),
+ smartlist_len(sl));
+ }
+ if (excludedset) {
+ routerset_subtract_nodes(sl,excludedset);
+ log_debug(LD_CIRC,
+ "We removed excludedset, leaving %d nodes.",
+ smartlist_len(sl));
+ }
+
+ // Always weight by bandwidth
+ choice = node_sl_choose_by_bandwidth(sl, rule);
+
+ smartlist_free(sl);
+ if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
+ /* try once more -- recurse but with fewer restrictions. */
+ log_info(LD_CIRC,
+ "We couldn't find any live%s%s%s routers; falling back "
+ "to list of all routers.",
+ need_capacity?", fast":"",
+ need_uptime?", stable":"",
+ need_guard?", guard":"");
+ flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
+ CRN_PREF_ADDR);
+ choice = router_choose_random_node(
+ excludedsmartlist, excludedset, flags);
+ }
+ smartlist_free(excludednodes);
+ if (!choice) {
+ log_warn(LD_CIRC,
+ "No available nodes when trying to choose node. Failing.");
+ }
+ return choice;
+}
+
+/** Try to find a running directory authority. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_trusteddirserver(dirinfo_type_t type, int flags)
+{
+ return router_pick_dirserver_generic(
+ router_get_trusted_dir_servers_mutable(),
+ type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
+{
+ return router_pick_dirserver_generic(
+ router_get_fallback_dir_servers_mutable(),
+ type, flags);
+}
+
+/** Pick a random element from a list of dir_server_t, weighting by their
+ * <b>weight</b> field. */
+static const dir_server_t *
+dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
+{
+ int n = smartlist_len(servers);
+ int i;
+ double *weights_dbl;
+ uint64_t *weights_u64;
+ const dir_server_t *ds;
+
+ weights_dbl = tor_calloc(n, sizeof(double));
+ weights_u64 = tor_calloc(n, sizeof(uint64_t));
+ for (i = 0; i < n; ++i) {
+ ds = smartlist_get(servers, i);
+ weights_dbl[i] = ds->weight;
+ if (ds->is_authority)
+ weights_dbl[i] *= authority_weight;
+ }
+
+ scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
+ i = choose_array_element_by_weight(weights_u64, n);
+ tor_free(weights_dbl);
+ tor_free(weights_u64);
+ return (i < 0) ? NULL : smartlist_get(servers, i);
+}
+
+/** Choose randomly from among the dir_server_ts in sourcelist that
+ * are up. Flags are as for router_pick_directory_server_impl().
+ */
+static const routerstatus_t *
+router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
+ dirinfo_type_t type, int flags,
+ int *n_busy_out)
+{
+ const or_options_t *options = get_options();
+ smartlist_t *direct, *tunnel;
+ smartlist_t *overloaded_direct, *overloaded_tunnel;
+ const routerinfo_t *me = router_get_my_routerinfo();
+ const routerstatus_t *result = NULL;
+ time_t now = time(NULL);
+ const int requireother = ! (flags & PDS_ALLOW_SELF);
+ const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+ const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+ const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+ const double auth_weight =
+ (sourcelist == router_get_fallback_dir_servers()) ?
+ options->DirAuthorityFallbackRate : 1.0;
+ smartlist_t *pick_from;
+ int n_busy = 0;
+ int try_excluding = 1, n_excluded = 0;
+ int try_ip_pref = 1;
+
+ if (!sourcelist)
+ return NULL;
+
+ retry_search:
+
+ direct = smartlist_new();
+ tunnel = smartlist_new();
+ overloaded_direct = smartlist_new();
+ overloaded_tunnel = smartlist_new();
+
+ const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+ const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+ const int must_have_or = directory_must_use_begindir(options);
+
+ SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
+ {
+ int is_overloaded =
+ d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
+ if (!d->is_running) continue;
+ if ((type & d->type) == 0)
+ continue;
+
+ SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
+ if (requireother && me && router_digest_is_me(d->digest))
+ continue;
+ if (try_excluding &&
+ routerset_contains_routerstatus(options->ExcludeNodes,
+ &d->fake_status, -1)) {
+ ++n_excluded;
+ continue;
+ }
+
+ if (router_is_already_dir_fetching_(d->addr,
+ &d->ipv6_addr,
+ d->dir_port,
+ no_serverdesc_fetching,
+ no_microdesc_fetching)) {
+ ++n_busy;
+ continue;
+ }
+
+ /* Clients use IPv6 addresses if the server has one and the client
+ * prefers IPv6.
+ * Add the router if its preferred address and port are reachable.
+ * If we don't get any routers, we'll try again with the non-preferred
+ * address for each router (if any). (To ensure correct load-balancing
+ * we try routers that only have one address both times.)
+ */
+ if (!fascistfirewall || skip_or_fw ||
+ fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
+ try_ip_pref))
+ smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
+ else if (!must_have_or && (skip_dir_fw ||
+ fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
+ try_ip_pref)))
+ smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
+ }
+ SMARTLIST_FOREACH_END(d);
+
+ if (smartlist_len(tunnel)) {
+ pick_from = tunnel;
+ } else if (smartlist_len(overloaded_tunnel)) {
+ pick_from = overloaded_tunnel;
+ } else if (smartlist_len(direct)) {
+ pick_from = direct;
+ } else {
+ pick_from = overloaded_direct;
+ }
+
+ {
+ const dir_server_t *selection =
+ dirserver_choose_by_weight(pick_from, auth_weight);
+
+ if (selection)
+ result = &selection->fake_status;
+ }
+
+ smartlist_free(direct);
+ smartlist_free(tunnel);
+ smartlist_free(overloaded_direct);
+ smartlist_free(overloaded_tunnel);
+
+ RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+ RETRY_WITHOUT_EXCLUDE(retry_search);
+
+ router_picked_poor_directory_log(result);
+
+ if (n_busy_out)
+ *n_busy_out = n_busy;
+ return result;
+}
diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h
new file mode 100644
index 000000000..05dabd123
--- /dev/null
+++ b/src/feature/nodelist/node_select.h
@@ -0,0 +1,102 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file node_select.h
+ * \brief Header file for node_select.c
+ **/
+
+#ifndef TOR_NODE_SELECT_H
+#define TOR_NODE_SELECT_H
+
+/** Flags to be passed to control router_choose_random_node() to indicate what
+ * kind of nodes to pick according to what algorithm. */
+typedef enum router_crn_flags_t {
+ CRN_NEED_UPTIME = 1<<0,
+ CRN_NEED_CAPACITY = 1<<1,
+ CRN_NEED_GUARD = 1<<2,
+ /* XXXX not used, apparently. */
+ CRN_WEIGHT_AS_EXIT = 1<<5,
+ CRN_NEED_DESC = 1<<6,
+ /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
+ CRN_PREF_ADDR = 1<<7,
+ /* On clients, only provide nodes that we can connect to directly, based on
+ * our firewall rules */
+ CRN_DIRECT_CONN = 1<<8,
+ /* On clients, only provide nodes with HSRend >= 2 protocol version which
+ * is required for hidden service version >= 3. */
+ CRN_RENDEZVOUS_V3 = 1<<9,
+} router_crn_flags_t;
+
+/** Possible ways to weight routers when choosing one randomly. See
+ * routerlist_sl_choose_by_bandwidth() for more information.*/
+typedef enum bandwidth_weight_rule_t {
+ NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
+ WEIGHT_FOR_DIR
+} bandwidth_weight_rule_t;
+
+/* Flags for pick_directory_server() and pick_trusteddirserver(). */
+/** Flag to indicate that we should not automatically be willing to use
+ * ourself to answer a directory request.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_ALLOW_SELF (1<<0)
+/** Flag to indicate that if no servers seem to be up, we should mark all
+ * directory servers as up and try again.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_RETRY_IF_NO_SERVERS (1<<1)
+/** Flag to indicate that we should not exclude directory servers that
+ * our ReachableAddress settings would exclude. This usually means that
+ * we're going to connect to the server over Tor, and so we don't need to
+ * worry about our firewall telling us we can't.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_IGNORE_FASCISTFIREWALL (1<<2)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading server descriptors
+ * or extrainfo documents.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading microdescs.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
+const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
+ int flags);
+
+int router_get_my_share_of_directory_requests(double *v3_share_out);
+
+const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule);
+double frac_nodes_with_descriptors(const smartlist_t *sl,
+ bandwidth_weight_rule_t rule,
+ int for_direct_conn);
+const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
+ struct routerset_t *excludedset,
+ router_crn_flags_t flags);
+
+const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
+ int flags);
+const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
+ int flags);
+
+#ifdef NODE_SELECT_PRIVATE
+STATIC int choose_array_element_by_weight(const uint64_t *entries,
+ int n_entries);
+STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
+ const double *entries_in,
+ int n_entries,
+ uint64_t *total_out);
+STATIC const routerstatus_t *router_pick_directory_server_impl(
+ dirinfo_type_t auth, int flags,
+ int *n_busy_out);
+STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
+ int serverdesc, int microdesc);
+#endif
+
+#endif
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
index 50dc8f7d3..c990e6940 100644
--- a/src/feature/nodelist/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -59,6 +59,8 @@
#include "core/or/protover.h"
#include "feature/rend/rendservice.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
index bcc5c1f07..547b54bb9 100644
--- a/src/feature/nodelist/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -57,78 +57,35 @@
* nodes according to different rules and weights. Historically, they
* were all in this module. Now, they are spread across this module,
* nodelist.c, and networkstatus.c. (TODO: Fix that.)
- *
- * <br>
- *
- * (For historical reasons) this module also contains code for handling
- * the list of fallback directories, the list of directory authorities,
- * and the list of authority certificates.
- *
- * For the directory authorities, we have a list containing the public
- * identity key, and contact points, for each authority. The
- * authorities receive descriptors from relays, and publish consensuses,
- * descriptors, and microdescriptors. This list is pre-configured.
- *
- * Fallback directories are well-known, stable, but untrusted directory
- * caches that clients which have not yet bootstrapped can use to get
- * their first networkstatus consensus, in order to find out where the
- * Tor network really is. This list is pre-configured in
- * fallback_dirs.inc. Every authority also serves as a fallback.
- *
- * Both fallback directories and directory authorities are are
- * represented by a dir_server_t.
- *
- * Authority certificates are signed with authority identity keys; they
- * are used to authenticate shorter-term authority signing keys. We
- * fetch them when we find a consensus or a vote that has been signed
- * with a signing key we don't recognize. We cache them on disk and
- * load them on startup. Authority operators generate them with the
- * "tor-gencert" utility.
- *
- * TODO: Authority certificates should be a separate module.
- *
- * TODO: dir_server_t stuff should be in a separate module.
**/
#define ROUTERLIST_PRIVATE
#include "core/or/or.h"
-#include "lib/err/backtrace.h"
-#include "feature/client/bridges.h"
-#include "lib/crypt_ops/crypto_ed25519.h"
-#include "lib/crypt_ops/crypto_format.h"
-#include "core/or/circuitstats.h"
+
#include "app/config/config.h"
#include "core/mainloop/connection.h"
+#include "core/mainloop/main.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
#include "feature/control/control.h"
-#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/dirauth/mode.h"
#include "feature/dircache/directory.h"
#include "feature/dircache/dirserv.h"
-#include "feature/client/entrynodes.h"
-#include "feature/dircommon/fp_pair.h"
-#include "feature/stats/geoip.h"
-#include "feature/hibernate/hibernate.h"
-#include "core/mainloop/main.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
-#include "core/or/policies.h"
-#include "core/or/reasons.h"
-#include "feature/rend/rendcommon.h"
-#include "feature/rend/rendservice.h"
-#include "feature/stats/rephist.h"
-#include "feature/relay/router.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
-#include "lib/sandbox/sandbox.h"
#include "feature/nodelist/torcert.h"
-#include "lib/math/fp.h"
-#include "lib/net/resolve.h"
-
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/mode.h"
+#include "feature/relay/router.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/nodelist/authority_cert_st.h"
#include "feature/dircommon/dir_connection_st.h"
#include "feature/dirclient/dir_server_st.h"
#include "feature/nodelist/document_signature_st.h"
@@ -156,7 +113,6 @@
DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
-DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
#define SDMAP_FOREACH(map, keyvar, valvar) \
DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \
valvar)
@@ -164,1164 +120,31 @@ DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar)
#define EIMAP_FOREACH(map, keyvar, valvar) \
DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar)
-#define DSMAP_FOREACH(map, keyvar, valvar) \
- DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
- valvar)
#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn))
#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn))
-#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn))
-/* Forward declaration for cert_list_t */
-typedef struct cert_list_t cert_list_t;
-
/* static function prototypes */
-static int compute_weighted_bandwidths(const smartlist_t *sl,
- bandwidth_weight_rule_t rule,
- double **bandwidths_out,
- double *total_bandwidth_out);
-static const routerstatus_t *router_pick_trusteddirserver_impl(
- const smartlist_t *sourcelist, dirinfo_type_t auth,
- int flags, int *n_busy_out);
-static const routerstatus_t *router_pick_dirserver_generic(
- smartlist_t *sourcelist,
- dirinfo_type_t type, int flags);
-static void mark_all_dirservers_up(smartlist_t *server_list);
static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
-static const char *signed_descriptor_get_body_impl(
- const signed_descriptor_t *desc,
- int with_annotations);
-static void list_pending_downloads(digestmap_t *result,
- digest256map_t *result256,
- int purpose, const char *prefix);
-static void list_pending_fpsk_downloads(fp_pair_map_t *result);
-static void launch_dummy_descriptor_download_as_needed(time_t now,
- const or_options_t *options);
-static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
- const char *digest);
-static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
- const char *digest,
- time_t now);
-
-/****************************************************************************/
-
-/** Global list of a dir_server_t object for each directory
- * authority. */
-static smartlist_t *trusted_dir_servers = NULL;
-/** Global list of dir_server_t objects for all directory authorities
- * and all fallback directory servers. */
-static smartlist_t *fallback_dir_servers = NULL;
-
-/** List of certificates for a single authority, and download status for
- * latest certificate.
- */
-struct cert_list_t {
- /*
- * The keys of download status map are cert->signing_key_digest for pending
- * downloads by (identity digest/signing key digest) pair; functions such
- * as authority_cert_get_by_digest() already assume these are unique.
- */
- struct digest_ds_map_t *dl_status_map;
- /* There is also a dlstatus for the download by identity key only */
- download_status_t dl_status_by_id;
- smartlist_t *certs;
-};
-/** Map from v3 identity key digest to cert_list_t. */
-static digestmap_t *trusted_dir_certs = NULL;
-/** True iff any key certificate in at least one member of
- * <b>trusted_dir_certs</b> has changed since we last flushed the
- * certificates to disk. */
-static int trusted_dir_servers_certs_changed = 0;
-
-/** Global list of all of the routers that we know about. */
-static routerlist_t *routerlist = NULL;
-
-/** List of strings for nicknames we've already warned about and that are
- * still unknown / unavailable. */
-static smartlist_t *warned_nicknames = NULL;
-
-/** The last time we tried to download any routerdesc, or 0 for "never". We
- * use this to rate-limit download attempts when the number of routerdescs to
- * download is low. */
-static time_t last_descriptor_download_attempted = 0;
-
-/** Return the number of directory authorities whose type matches some bit set
- * in <b>type</b> */
-int
-get_n_authorities(dirinfo_type_t type)
-{
- int n = 0;
- if (!trusted_dir_servers)
- return 0;
- SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
- if (ds->type & type)
- ++n);
- return n;
-}
-
-/** Initialise schedule, want_authority, and increment_on in the download
- * status dlstatus, then call download_status_reset() on it.
- * It is safe to call this function or download_status_reset() multiple times
- * on a new dlstatus. But it should *not* be called after a dlstatus has been
- * used to count download attempts or failures. */
-static void
-download_status_cert_init(download_status_t *dlstatus)
-{
- dlstatus->schedule = DL_SCHED_CONSENSUS;
- dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
- dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
- dlstatus->last_backoff_position = 0;
- dlstatus->last_delay_used = 0;
-
- /* Use the new schedule to set next_attempt_at */
- download_status_reset(dlstatus);
-}
-
-/** Reset the download status of a specified element in a dsmap */
-static void
-download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
-{
- download_status_t *dlstatus = NULL;
-
- tor_assert(cl);
- tor_assert(digest);
-
- /* Make sure we have a dsmap */
- if (!(cl->dl_status_map)) {
- cl->dl_status_map = dsmap_new();
- }
- /* Look for a download_status_t in the map with this digest */
- dlstatus = dsmap_get(cl->dl_status_map, digest);
- /* Got one? */
- if (!dlstatus) {
- /* Insert before we reset */
- dlstatus = tor_malloc_zero(sizeof(*dlstatus));
- dsmap_set(cl->dl_status_map, digest, dlstatus);
- download_status_cert_init(dlstatus);
- }
- tor_assert(dlstatus);
- /* Go ahead and reset it */
- download_status_reset(dlstatus);
-}
-
-/**
- * Return true if the download for this signing key digest in cl is ready
- * to be re-attempted.
- */
-static int
-download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
- const char *digest,
- time_t now)
-{
- int rv = 0;
- download_status_t *dlstatus = NULL;
-
- tor_assert(cl);
- tor_assert(digest);
-
- /* Make sure we have a dsmap */
- if (!(cl->dl_status_map)) {
- cl->dl_status_map = dsmap_new();
- }
- /* Look for a download_status_t in the map with this digest */
- dlstatus = dsmap_get(cl->dl_status_map, digest);
- /* Got one? */
- if (dlstatus) {
- /* Use download_status_is_ready() */
- rv = download_status_is_ready(dlstatus, now);
- } else {
- /*
- * If we don't know anything about it, return 1, since we haven't
- * tried this one before. We need to create a new entry here,
- * too.
- */
- dlstatus = tor_malloc_zero(sizeof(*dlstatus));
- download_status_cert_init(dlstatus);
- dsmap_set(cl->dl_status_map, digest, dlstatus);
- rv = 1;
- }
-
- return rv;
-}
-
-/** Helper: Return the cert_list_t for an authority whose authority ID is
- * <b>id_digest</b>, allocating a new list if necessary. */
-static cert_list_t *
-get_cert_list(const char *id_digest)
-{
- cert_list_t *cl;
- if (!trusted_dir_certs)
- trusted_dir_certs = digestmap_new();
- cl = digestmap_get(trusted_dir_certs, id_digest);
- if (!cl) {
- cl = tor_malloc_zero(sizeof(cert_list_t));
- download_status_cert_init(&cl->dl_status_by_id);
- cl->certs = smartlist_new();
- cl->dl_status_map = dsmap_new();
- digestmap_set(trusted_dir_certs, id_digest, cl);
- }
- return cl;
-}
-
-/** Return a list of authority ID digests with potentially enumerable lists
- * of download_status_t objects; used by controller GETINFO queries.
- */
-
-MOCK_IMPL(smartlist_t *,
-list_authority_ids_with_downloads, (void))
-{
- smartlist_t *ids = smartlist_new();
- digestmap_iter_t *i;
- const char *digest;
- char *tmp;
- void *cl;
-
- if (trusted_dir_certs) {
- for (i = digestmap_iter_init(trusted_dir_certs);
- !(digestmap_iter_done(i));
- i = digestmap_iter_next(trusted_dir_certs, i)) {
- /*
- * We always have at least dl_status_by_id to query, so no need to
- * probe deeper than the existence of a cert_list_t.
- */
- digestmap_iter_get(i, &digest, &cl);
- tmp = tor_malloc(DIGEST_LEN);
- memcpy(tmp, digest, DIGEST_LEN);
- smartlist_add(ids, tmp);
- }
- }
- /* else definitely no downloads going since nothing even has a cert list */
-
- return ids;
-}
-
-/** Given an authority ID digest, return a pointer to the default download
- * status, or NULL if there is no such entry in trusted_dir_certs */
-
-MOCK_IMPL(download_status_t *,
-id_only_download_status_for_authority_id, (const char *digest))
-{
- download_status_t *dl = NULL;
- cert_list_t *cl;
-
- if (trusted_dir_certs) {
- cl = digestmap_get(trusted_dir_certs, digest);
- if (cl) {
- dl = &(cl->dl_status_by_id);
- }
- }
-
- return dl;
-}
-
-/** Given an authority ID digest, return a smartlist of signing key digests
- * for which download_status_t is potentially queryable, or NULL if no such
- * authority ID digest is known. */
-
-MOCK_IMPL(smartlist_t *,
-list_sk_digests_for_authority_id, (const char *digest))
-{
- smartlist_t *sks = NULL;
- cert_list_t *cl;
- dsmap_iter_t *i;
- const char *sk_digest;
- char *tmp;
- download_status_t *dl;
-
- if (trusted_dir_certs) {
- cl = digestmap_get(trusted_dir_certs, digest);
- if (cl) {
- sks = smartlist_new();
- if (cl->dl_status_map) {
- for (i = dsmap_iter_init(cl->dl_status_map);
- !(dsmap_iter_done(i));
- i = dsmap_iter_next(cl->dl_status_map, i)) {
- /* Pull the digest out and add it to the list */
- dsmap_iter_get(i, &sk_digest, &dl);
- tmp = tor_malloc(DIGEST_LEN);
- memcpy(tmp, sk_digest, DIGEST_LEN);
- smartlist_add(sks, tmp);
- }
- }
- }
- }
-
- return sks;
-}
-
-/** Given an authority ID digest and a signing key digest, return the
- * download_status_t or NULL if none exists. */
-
-MOCK_IMPL(download_status_t *,
-download_status_for_authority_id_and_sk,(const char *id_digest,
- const char *sk_digest))
-{
- download_status_t *dl = NULL;
- cert_list_t *cl = NULL;
-
- if (trusted_dir_certs) {
- cl = digestmap_get(trusted_dir_certs, id_digest);
- if (cl && cl->dl_status_map) {
- dl = dsmap_get(cl->dl_status_map, sk_digest);
- }
- }
-
- return dl;
-}
-
-#define cert_list_free(val) \
- FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
-
-/** Release all space held by a cert_list_t */
-static void
-cert_list_free_(cert_list_t *cl)
-{
- if (!cl)
- return;
-
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
- authority_cert_free(cert));
- smartlist_free(cl->certs);
- dsmap_free(cl->dl_status_map, tor_free_);
- tor_free(cl);
-}
-
-/** Wrapper for cert_list_free so we can pass it to digestmap_free */
-static void
-cert_list_free_void(void *cl)
-{
- cert_list_free_(cl);
-}
-
-/** Reload the cached v3 key certificates from the cached-certs file in
- * the data directory. Return 0 on success, -1 on failure. */
-int
-trusted_dirs_reload_certs(void)
-{
- char *filename;
- char *contents;
- int r;
-
- filename = get_cachedir_fname("cached-certs");
- contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
- tor_free(filename);
- if (!contents)
- return 0;
- r = trusted_dirs_load_certs_from_string(
- contents,
- TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
- tor_free(contents);
- return r;
-}
-
-/** Helper: return true iff we already have loaded the exact cert
- * <b>cert</b>. */
-static inline int
-already_have_cert(authority_cert_t *cert)
-{
- cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
-
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
- {
- if (tor_memeq(c->cache_info.signed_descriptor_digest,
- cert->cache_info.signed_descriptor_digest,
- DIGEST_LEN))
- return 1;
- });
- return 0;
-}
-
-/** Load a bunch of new key certificates from the string <b>contents</b>. If
- * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
- * from the cache, and we don't need to flush them to disk. If we are a
- * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
- * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
- * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST. If <b>flush</b> is true, we
- * need to flush any changed certificates to disk now. Return 0 on success,
- * -1 if any certs fail to parse.
- *
- * If source_dir is non-NULL, it's the identity digest for a directory that
- * we've just successfully retrieved certificates from, so try it first to
- * fetch any missing certificates.
- */
-int
-trusted_dirs_load_certs_from_string(const char *contents, int source,
- int flush, const char *source_dir)
-{
- dir_server_t *ds;
- const char *s, *eos;
- int failure_code = 0;
- int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
- int added_trusted_cert = 0;
-
- for (s = contents; *s; s = eos) {
- authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
- cert_list_t *cl;
- if (!cert) {
- failure_code = -1;
- break;
- }
- ds = trusteddirserver_get_by_v3_auth_digest(
- cert->cache_info.identity_digest);
- log_debug(LD_DIR, "Parsed certificate for %s",
- ds ? ds->nickname : "unknown authority");
-
- if (already_have_cert(cert)) {
- /* we already have this one. continue. */
- log_info(LD_DIR, "Skipping %s certificate for %s that we "
- "already have.",
- from_store ? "cached" : "downloaded",
- ds ? ds->nickname : "an old or new authority");
-
- /*
- * A duplicate on download should be treated as a failure, so we call
- * authority_cert_dl_failed() to reset the download status to make sure
- * we can't try again. Since we've implemented the fp-sk mechanism
- * to download certs by signing key, this should be much rarer than it
- * was and is perhaps cause for concern.
- */
- if (!from_store) {
- if (authdir_mode(get_options())) {
- log_warn(LD_DIR,
- "Got a certificate for %s, but we already have it. "
- "Maybe they haven't updated it. Waiting for a while.",
- ds ? ds->nickname : "an old or new authority");
- } else {
- log_info(LD_DIR,
- "Got a certificate for %s, but we already have it. "
- "Maybe they haven't updated it. Waiting for a while.",
- ds ? ds->nickname : "an old or new authority");
- }
-
- /*
- * This is where we care about the source; authority_cert_dl_failed()
- * needs to know whether the download was by fp or (fp,sk) pair to
- * twiddle the right bit in the download map.
- */
- if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
- authority_cert_dl_failed(cert->cache_info.identity_digest,
- NULL, 404);
- } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
- authority_cert_dl_failed(cert->cache_info.identity_digest,
- cert->signing_key_digest, 404);
- }
- }
-
- authority_cert_free(cert);
- continue;
- }
-
- if (ds) {
- added_trusted_cert = 1;
- log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
- "signing key %s", from_store ? "cached" : "downloaded",
- ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
- } else {
- int adding = we_want_to_fetch_unknown_auth_certs(get_options());
- log_info(LD_DIR, "%s %s certificate for unrecognized directory "
- "authority with signing key %s",
- adding ? "Adding" : "Not adding",
- from_store ? "cached" : "downloaded",
- hex_str(cert->signing_key_digest,DIGEST_LEN));
- if (!adding) {
- authority_cert_free(cert);
- continue;
- }
- }
-
- cl = get_cert_list(cert->cache_info.identity_digest);
- smartlist_add(cl->certs, cert);
- if (ds && cert->cache_info.published_on > ds->addr_current_at) {
- /* Check to see whether we should update our view of the authority's
- * address. */
- if (cert->addr && cert->dir_port &&
- (ds->addr != cert->addr ||
- ds->dir_port != cert->dir_port)) {
- char *a = tor_dup_ip(cert->addr);
- log_notice(LD_DIR, "Updating address for directory authority %s "
- "from %s:%d to %s:%d based on certificate.",
- ds->nickname, ds->address, (int)ds->dir_port,
- a, cert->dir_port);
- tor_free(a);
- ds->addr = cert->addr;
- ds->dir_port = cert->dir_port;
- }
- ds->addr_current_at = cert->cache_info.published_on;
- }
-
- if (!from_store)
- trusted_dir_servers_certs_changed = 1;
- }
-
- if (flush)
- trusted_dirs_flush_certs_to_disk();
-
- /* call this even if failure_code is <0, since some certs might have
- * succeeded, but only pass source_dir if there were no failures,
- * and at least one more authority certificate was added to the store.
- * This avoids retrying a directory that's serving bad or entirely duplicate
- * certificates. */
- if (failure_code == 0 && added_trusted_cert) {
- networkstatus_note_certs_arrived(source_dir);
- } else {
- networkstatus_note_certs_arrived(NULL);
- }
-
- return failure_code;
-}
-
-/** Save all v3 key certificates to the cached-certs file. */
-void
-trusted_dirs_flush_certs_to_disk(void)
-{
- char *filename;
- smartlist_t *chunks;
-
- if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
- return;
-
- chunks = smartlist_new();
- DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
- {
- sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
- c->bytes = cert->cache_info.signed_descriptor_body;
- c->len = cert->cache_info.signed_descriptor_len;
- smartlist_add(chunks, c);
- });
- } DIGESTMAP_FOREACH_END;
-
- filename = get_cachedir_fname("cached-certs");
- if (write_chunks_to_file(filename, chunks, 0, 0)) {
- log_warn(LD_FS, "Error writing certificates to disk.");
- }
- tor_free(filename);
- SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
- smartlist_free(chunks);
-
- trusted_dir_servers_certs_changed = 0;
-}
-
-static int
-compare_certs_by_pubdates(const void **_a, const void **_b)
-{
- const authority_cert_t *cert1 = *_a, *cert2=*_b;
-
- if (cert1->cache_info.published_on < cert2->cache_info.published_on)
- return -1;
- else if (cert1->cache_info.published_on > cert2->cache_info.published_on)
- return 1;
- else
- return 0;
-}
-
-/** Remove all expired v3 authority certificates that have been superseded for
- * more than 48 hours or, if not expired, that were published more than 7 days
- * before being superseded. (If the most recent cert was published more than 48
- * hours ago, then we aren't going to get any consensuses signed with older
- * keys.) */
-static void
-trusted_dirs_remove_old_certs(void)
-{
- time_t now = time(NULL);
-#define DEAD_CERT_LIFETIME (2*24*60*60)
-#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
- if (!trusted_dir_certs)
- return;
-
- DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
- /* Sort the list from first-published to last-published */
- smartlist_sort(cl->certs, compare_certs_by_pubdates);
-
- SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
- if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
- /* This is the most recently published cert. Keep it. */
- continue;
- }
- authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
- const time_t next_cert_published = next_cert->cache_info.published_on;
- if (next_cert_published > now) {
- /* All later certs are published in the future. Keep everything
- * we didn't discard. */
- break;
- }
- int should_remove = 0;
- if (cert->expires + DEAD_CERT_LIFETIME < now) {
- /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
- * Remove it. */
- should_remove = 1;
- } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
- /* Certificate has been superseded for OLD_CERT_LIFETIME.
- * Remove it.
- */
- should_remove = 1;
- }
- if (should_remove) {
- SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
- authority_cert_free(cert);
- trusted_dir_servers_certs_changed = 1;
- }
- } SMARTLIST_FOREACH_END(cert);
-
- } DIGESTMAP_FOREACH_END;
-#undef DEAD_CERT_LIFETIME
-#undef OLD_CERT_LIFETIME
-
- trusted_dirs_flush_certs_to_disk();
-}
-
-/** Return the newest v3 authority certificate whose v3 authority identity key
- * has digest <b>id_digest</b>. Return NULL if no such authority is known,
- * or it has no certificate. */
-authority_cert_t *
-authority_cert_get_newest_by_id(const char *id_digest)
-{
- cert_list_t *cl;
- authority_cert_t *best = NULL;
- if (!trusted_dir_certs ||
- !(cl = digestmap_get(trusted_dir_certs, id_digest)))
- return NULL;
-
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
- {
- if (!best || cert->cache_info.published_on > best->cache_info.published_on)
- best = cert;
- });
- return best;
-}
-
-/** Return the newest v3 authority certificate whose directory signing key has
- * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
- */
-authority_cert_t *
-authority_cert_get_by_sk_digest(const char *sk_digest)
-{
- authority_cert_t *c;
- if (!trusted_dir_certs)
- return NULL;
-
- if ((c = get_my_v3_authority_cert()) &&
- tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
- return c;
- if ((c = get_my_v3_legacy_cert()) &&
- tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
- return c;
-
- DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
- {
- if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
- return cert;
- });
- } DIGESTMAP_FOREACH_END;
- return NULL;
-}
-
-/** Return the v3 authority certificate with signing key matching
- * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
- * Return NULL if no such authority is known. */
-authority_cert_t *
-authority_cert_get_by_digests(const char *id_digest,
- const char *sk_digest)
-{
- cert_list_t *cl;
- if (!trusted_dir_certs ||
- !(cl = digestmap_get(trusted_dir_certs, id_digest)))
- return NULL;
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
- if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
- return cert; );
-
- return NULL;
-}
-
-/** Add every known authority_cert_t to <b>certs_out</b>. */
-void
-authority_cert_get_all(smartlist_t *certs_out)
-{
- tor_assert(certs_out);
- if (!trusted_dir_certs)
- return;
-
- DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
- SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
- smartlist_add(certs_out, c));
- } DIGESTMAP_FOREACH_END;
-}
-
-/** Called when an attempt to download a certificate with the authority with
- * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
- * fails with HTTP response code <b>status</b>: remember the failure, so we
- * don't try again immediately. */
-void
-authority_cert_dl_failed(const char *id_digest,
- const char *signing_key_digest, int status)
-{
- cert_list_t *cl;
- download_status_t *dlstatus = NULL;
- char id_digest_str[2*DIGEST_LEN+1];
- char sk_digest_str[2*DIGEST_LEN+1];
-
- if (!trusted_dir_certs ||
- !(cl = digestmap_get(trusted_dir_certs, id_digest)))
- return;
-
- /*
- * Are we noting a failed download of the latest cert for the id digest,
- * or of a download by (id, signing key) digest pair?
- */
- if (!signing_key_digest) {
- /* Just by id digest */
- download_status_failed(&cl->dl_status_by_id, status);
- } else {
- /* Reset by (id, signing key) digest pair
- *
- * Look for a download_status_t in the map with this digest
- */
- dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
- /* Got one? */
- if (dlstatus) {
- download_status_failed(dlstatus, status);
- } else {
- /*
- * Do this rather than hex_str(), since hex_str clobbers
- * old results and we call twice in the param list.
- */
- base16_encode(id_digest_str, sizeof(id_digest_str),
- id_digest, DIGEST_LEN);
- base16_encode(sk_digest_str, sizeof(sk_digest_str),
- signing_key_digest, DIGEST_LEN);
- log_warn(LD_BUG,
- "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
- "status %d, but knew nothing about the download.",
- id_digest_str, sk_digest_str, status);
- }
- }
-}
-
-static const char *BAD_SIGNING_KEYS[] = {
- "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
- "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
- "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
- "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
- "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
- "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
- "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
- "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
- "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
- "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
- NULL,
-};
-
-/** Return true iff <b>cert</b> authenticates some atuhority signing key
- * which, because of the old openssl heartbleed vulnerability, should
- * never be trusted. */
-int
-authority_cert_is_blacklisted(const authority_cert_t *cert)
-{
- char hex_digest[HEX_DIGEST_LEN+1];
- int i;
- base16_encode(hex_digest, sizeof(hex_digest),
- cert->signing_key_digest, sizeof(cert->signing_key_digest));
-
- for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
- if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
- return 1;
- }
- }
- return 0;
-}
-
-/** Return true iff when we've been getting enough failures when trying to
- * download the certificate with ID digest <b>id_digest</b> that we're willing
- * to start bugging the user about it. */
-int
-authority_cert_dl_looks_uncertain(const char *id_digest)
-{
-#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
- cert_list_t *cl;
- int n_failures;
- if (!trusted_dir_certs ||
- !(cl = digestmap_get(trusted_dir_certs, id_digest)))
- return 0;
-
- n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
- return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
-}
-
-/* Fetch the authority certificates specified in resource.
- * If we are a bridge client, and node is a configured bridge, fetch from node
- * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
- * rs. Otherwise, fetch from a random directory mirror. */
-static void
-authority_certs_fetch_resource_impl(const char *resource,
- const char *dir_hint,
- const node_t *node,
- const routerstatus_t *rs)
-{
- const or_options_t *options = get_options();
- int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
- resource);
-
- /* Make sure bridge clients never connect to anything but a bridge */
- if (options->UseBridges) {
- if (node && !node_is_a_configured_bridge(node)) {
- /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
- get_via_tor = 1;
- } else if (!node) {
- /* If we're using bridges, and there's no node, use a 3-hop path. */
- get_via_tor = 1;
- }
- }
-
- const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
- : DIRIND_ONEHOP;
-
- directory_request_t *req = NULL;
- /* If we've just downloaded a consensus from a bridge, re-use that
- * bridge */
- if (options->UseBridges && node && node->ri && !get_via_tor) {
- /* clients always make OR connections to bridges */
- tor_addr_port_t or_ap;
- /* we are willing to use a non-preferred address if we need to */
- fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
- &or_ap);
-
- req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
- directory_request_set_or_addr_port(req, &or_ap);
- if (dir_hint)
- directory_request_set_directory_id_digest(req, dir_hint);
- } else if (rs) {
- /* And if we've just downloaded a consensus from a directory, re-use that
- * directory */
- req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
- directory_request_set_routerstatus(req, rs);
- }
-
- if (req) {
- /* We've set up a request object -- fill in the other request fields, and
- * send the request. */
- directory_request_set_indirection(req, indirection);
- directory_request_set_resource(req, resource);
- directory_initiate_request(req);
- directory_request_free(req);
- return;
- }
-
- /* Otherwise, we want certs from a random fallback or directory
- * mirror, because they will almost always succeed. */
- directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
- resource, PDS_RETRY_IF_NO_SERVERS,
- DL_WANT_ANY_DIRSERVER);
-}
-
-/** Try to download any v3 authority certificates that we may be missing. If
- * <b>status</b> is provided, try to get all the ones that were used to sign
- * <b>status</b>. Additionally, try to have a non-expired certificate for
- * every V3 authority in trusted_dir_servers. Don't fetch certificates we
- * already have.
- *
- * If dir_hint is non-NULL, it's the identity digest for a directory that
- * we've just successfully retrieved a consensus or certificates from, so try
- * it first to fetch any missing certificates.
- **/
-void
-authority_certs_fetch_missing(networkstatus_t *status, time_t now,
- const char *dir_hint)
-{
- /*
- * The pending_id digestmap tracks pending certificate downloads by
- * identity digest; the pending_cert digestmap tracks pending downloads
- * by (identity digest, signing key digest) pairs.
- */
- digestmap_t *pending_id;
- fp_pair_map_t *pending_cert;
- /*
- * The missing_id_digests smartlist will hold a list of id digests
- * we want to fetch the newest cert for; the missing_cert_digests
- * smartlist will hold a list of fp_pair_t with an identity and
- * signing key digest.
- */
- smartlist_t *missing_cert_digests, *missing_id_digests;
- char *resource = NULL;
- cert_list_t *cl;
- const or_options_t *options = get_options();
- const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
- fp_pair_t *fp_tmp = NULL;
- char id_digest_str[2*DIGEST_LEN+1];
- char sk_digest_str[2*DIGEST_LEN+1];
-
- if (should_delay_dir_fetches(options, NULL))
- return;
-
- pending_cert = fp_pair_map_new();
- pending_id = digestmap_new();
- missing_cert_digests = smartlist_new();
- missing_id_digests = smartlist_new();
-
- /*
- * First, we get the lists of already pending downloads so we don't
- * duplicate effort.
- */
- list_pending_downloads(pending_id, NULL,
- DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
- list_pending_fpsk_downloads(pending_cert);
-
- /*
- * Now, we download any trusted authority certs we don't have by
- * identity digest only. This gets the latest cert for that authority.
- */
- SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) {
- int found = 0;
- if (!(ds->type & V3_DIRINFO))
- continue;
- if (smartlist_contains_digest(missing_id_digests,
- ds->v3_identity_digest))
- continue;
- cl = get_cert_list(ds->v3_identity_digest);
- SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
- if (now < cert->expires) {
- /* It's not expired, and we weren't looking for something to
- * verify a consensus with. Call it done. */
- download_status_reset(&(cl->dl_status_by_id));
- /* No sense trying to download it specifically by signing key hash */
- download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
- found = 1;
- break;
- }
- } SMARTLIST_FOREACH_END(cert);
- if (!found &&
- download_status_is_ready(&(cl->dl_status_by_id), now) &&
- !digestmap_get(pending_id, ds->v3_identity_digest)) {
- log_info(LD_DIR,
- "No current certificate known for authority %s "
- "(ID digest %s); launching request.",
- ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
- smartlist_add(missing_id_digests, ds->v3_identity_digest);
- }
- } SMARTLIST_FOREACH_END(ds);
-
- /*
- * Next, if we have a consensus, scan through it and look for anything
- * signed with a key from a cert we don't have. Those get downloaded
- * by (fp,sk) pair, but if we don't know any certs at all for the fp
- * (identity digest), and it's one of the trusted dir server certs
- * we started off above or a pending download in pending_id, don't
- * try to get it yet. Most likely, the one we'll get for that will
- * have the right signing key too, and we'd just be downloading
- * redundantly.
- */
- if (status) {
- SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
- voter) {
- if (!smartlist_len(voter->sigs))
- continue; /* This authority never signed this consensus, so don't
- * go looking for a cert with key digest 0000000000. */
- if (!keep_unknown &&
- !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
- continue; /* We don't want unknown certs, and we don't know this
- * authority.*/
-
- /*
- * If we don't know *any* cert for this authority, and a download by ID
- * is pending or we added it to missing_id_digests above, skip this
- * one for now to avoid duplicate downloads.
- */
- cl = get_cert_list(voter->identity_digest);
- if (smartlist_len(cl->certs) == 0) {
- /* We have no certs at all for this one */
-
- /* Do we have a download of one pending? */
- if (digestmap_get(pending_id, voter->identity_digest))
- continue;
-
- /*
- * Are we about to launch a download of one due to the trusted
- * dir server check above?
- */
- if (smartlist_contains_digest(missing_id_digests,
- voter->identity_digest))
- continue;
- }
-
- SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
- authority_cert_t *cert =
- authority_cert_get_by_digests(voter->identity_digest,
- sig->signing_key_digest);
- if (cert) {
- if (now < cert->expires)
- download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
- continue;
- }
- if (download_status_is_ready_by_sk_in_cl(
- cl, sig->signing_key_digest, now) &&
- !fp_pair_map_get_by_digests(pending_cert,
- voter->identity_digest,
- sig->signing_key_digest)) {
- /*
- * Do this rather than hex_str(), since hex_str clobbers
- * old results and we call twice in the param list.
- */
- base16_encode(id_digest_str, sizeof(id_digest_str),
- voter->identity_digest, DIGEST_LEN);
- base16_encode(sk_digest_str, sizeof(sk_digest_str),
- sig->signing_key_digest, DIGEST_LEN);
-
- if (voter->nickname) {
- log_info(LD_DIR,
- "We're missing a certificate from authority %s "
- "(ID digest %s) with signing key %s: "
- "launching request.",
- voter->nickname, id_digest_str, sk_digest_str);
- } else {
- log_info(LD_DIR,
- "We're missing a certificate from authority ID digest "
- "%s with signing key %s: launching request.",
- id_digest_str, sk_digest_str);
- }
-
- /* Allocate a new fp_pair_t to append */
- fp_tmp = tor_malloc(sizeof(*fp_tmp));
- memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
- memcpy(fp_tmp->second, sig->signing_key_digest,
- sizeof(fp_tmp->second));
- smartlist_add(missing_cert_digests, fp_tmp);
- }
- } SMARTLIST_FOREACH_END(sig);
- } SMARTLIST_FOREACH_END(voter);
- }
-
- /* Bridge clients look up the node for the dir_hint */
- const node_t *node = NULL;
- /* All clients, including bridge clients, look up the routerstatus for the
- * dir_hint */
- const routerstatus_t *rs = NULL;
-
- /* If we still need certificates, try the directory that just successfully
- * served us a consensus or certificates.
- * As soon as the directory fails to provide additional certificates, we try
- * another, randomly selected directory. This avoids continual retries.
- * (We only ever have one outstanding request per certificate.)
- */
- if (dir_hint) {
- if (options->UseBridges) {
- /* Bridge clients try the nodelist. If the dir_hint is from an authority,
- * or something else fetched over tor, we won't find the node here, but
- * we will find the rs. */
- node = node_get_by_id(dir_hint);
- }
-
- /* All clients try the consensus routerstatus, then the fallback
- * routerstatus */
- rs = router_get_consensus_status_by_id(dir_hint);
- if (!rs) {
- /* This will also find authorities */
- const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
- dir_hint);
- if (ds) {
- rs = &ds->fake_status;
- }
- }
-
- if (!node && !rs) {
- log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
- "no routerstatus could be found for it.",
- options->UseBridges ? "no node and " : "",
- hex_str(dir_hint, DIGEST_LEN));
- }
- }
-
- /* Do downloads by identity digest */
- if (smartlist_len(missing_id_digests) > 0) {
- int need_plus = 0;
- smartlist_t *fps = smartlist_new();
-
- smartlist_add_strdup(fps, "fp/");
-
- SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
- char *fp = NULL;
-
- if (digestmap_get(pending_id, d))
- continue;
-
- base16_encode(id_digest_str, sizeof(id_digest_str),
- d, DIGEST_LEN);
-
- if (need_plus) {
- tor_asprintf(&fp, "+%s", id_digest_str);
- } else {
- /* No need for tor_asprintf() in this case; first one gets no '+' */
- fp = tor_strdup(id_digest_str);
- need_plus = 1;
- }
-
- smartlist_add(fps, fp);
- } SMARTLIST_FOREACH_END(d);
-
- if (smartlist_len(fps) > 1) {
- resource = smartlist_join_strings(fps, "", 0, NULL);
- /* node and rs are directories that just gave us a consensus or
- * certificates */
- authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
- tor_free(resource);
- }
- /* else we didn't add any: they were all pending */
-
- SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
- smartlist_free(fps);
- }
-
- /* Do downloads by identity digest/signing key pair */
- if (smartlist_len(missing_cert_digests) > 0) {
- int need_plus = 0;
- smartlist_t *fp_pairs = smartlist_new();
-
- smartlist_add_strdup(fp_pairs, "fp-sk/");
-
- SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
- char *fp_pair = NULL;
-
- if (fp_pair_map_get(pending_cert, d))
- continue;
-
- /* Construct string encodings of the digests */
- base16_encode(id_digest_str, sizeof(id_digest_str),
- d->first, DIGEST_LEN);
- base16_encode(sk_digest_str, sizeof(sk_digest_str),
- d->second, DIGEST_LEN);
-
- /* Now tor_asprintf() */
- if (need_plus) {
- tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
- } else {
- /* First one in the list doesn't get a '+' */
- tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
- need_plus = 1;
- }
+static const char *signed_descriptor_get_body_impl(
+ const signed_descriptor_t *desc,
+ int with_annotations);
+static void launch_dummy_descriptor_download_as_needed(time_t now,
+ const or_options_t *options);
- /* Add it to the list of pairs to request */
- smartlist_add(fp_pairs, fp_pair);
- } SMARTLIST_FOREACH_END(d);
+/****************************************************************************/
- if (smartlist_len(fp_pairs) > 1) {
- resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
- /* node and rs are directories that just gave us a consensus or
- * certificates */
- authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
- tor_free(resource);
- }
- /* else they were all pending */
+/** Global list of all of the routers that we know about. */
+static routerlist_t *routerlist = NULL;
- SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
- smartlist_free(fp_pairs);
- }
+/** List of strings for nicknames we've already warned about and that are
+ * still unknown / unavailable. */
+static smartlist_t *warned_nicknames = NULL;
- smartlist_free(missing_id_digests);
- SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
- smartlist_free(missing_cert_digests);
- digestmap_free(pending_id, NULL);
- fp_pair_map_free(pending_cert, NULL);
-}
+/** The last time we tried to download any routerdesc, or 0 for "never". We
+ * use this to rate-limit download attempts when the number of routerdescs to
+ * download is low. */
+static time_t last_descriptor_download_attempted = 0;
/* Router descriptor storage.
*
@@ -1603,713 +426,60 @@ router_reload_router_list_impl(desc_store_t *store)
else
router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL,
NULL, 0, NULL);
- store->journal_len = (size_t) st.st_size;
- tor_free(contents);
- }
-
- tor_free(fname);
-
- if (store->journal_len) {
- /* Always clear the journal on startup.*/
- router_rebuild_store(RRS_FORCE, store);
- } else if (!extrainfo) {
- /* Don't cache expired routers. (This is in an else because
- * router_rebuild_store() also calls remove_old_routers().) */
- routerlist_remove_old_routers();
- }
-
- return 0;
-}
-
-/** Load all cached router descriptors and extra-info documents from the
- * store. Return 0 on success and -1 on failure.
- */
-int
-router_reload_router_list(void)
-{
- routerlist_t *rl = router_get_routerlist();
- if (router_reload_router_list_impl(&rl->desc_store))
- return -1;
- if (router_reload_router_list_impl(&rl->extrainfo_store))
- return -1;
- return 0;
-}
-
-/** Return a smartlist containing a list of dir_server_t * for all
- * known trusted dirservers. Callers must not modify the list or its
- * contents.
- */
-const smartlist_t *
-router_get_trusted_dir_servers(void)
-{
- if (!trusted_dir_servers)
- trusted_dir_servers = smartlist_new();
-
- return trusted_dir_servers;
-}
-
-const smartlist_t *
-router_get_fallback_dir_servers(void)
-{
- if (!fallback_dir_servers)
- fallback_dir_servers = smartlist_new();
-
- return fallback_dir_servers;
-}
-
-/** Try to find a running dirserver that supports operations of <b>type</b>.
- *
- * If there are no running dirservers in our routerlist and the
- * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
- * (including authorities) as running again, and pick one.
- *
- * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
- * dirservers that we can't reach.
- *
- * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
- * (if we're a dirserver).
- *
- * Don't pick a fallback directory mirror if any non-fallback is viable;
- * (the fallback directory mirrors include the authorities)
- * try to avoid using servers that have returned 503 recently.
- */
-const routerstatus_t *
-router_pick_directory_server(dirinfo_type_t type, int flags)
-{
- int busy = 0;
- const routerstatus_t *choice;
-
- if (!routerlist)
- return NULL;
-
- choice = router_pick_directory_server_impl(type, flags, &busy);
- if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
- return choice;
-
- if (busy) {
- /* If the reason that we got no server is that servers are "busy",
- * we must be excluding good servers because we already have serverdesc
- * fetches with them. Do not mark down servers up because of this. */
- tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
- PDS_NO_EXISTING_MICRODESC_FETCH)));
- return NULL;
- }
-
- log_info(LD_DIR,
- "No reachable router entries for dirservers. "
- "Trying them all again.");
- /* mark all fallback directory mirrors as up again */
- mark_all_dirservers_up(fallback_dir_servers);
- /* try again */
- choice = router_pick_directory_server_impl(type, flags, NULL);
- return choice;
-}
-
-/** Return the dir_server_t for the directory authority whose identity
- * key hashes to <b>digest</b>, or NULL if no such authority is known.
- */
-dir_server_t *
-router_get_trusteddirserver_by_digest(const char *digest)
-{
- if (!trusted_dir_servers)
- return NULL;
-
- SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
- {
- if (tor_memeq(ds->digest, digest, DIGEST_LEN))
- return ds;
- });
-
- return NULL;
-}
-
-/** Return the dir_server_t for the fallback dirserver whose identity
- * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
- * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
- * and UseDefaultFallbackDirs torrc options.)
- * The list of fallback directories includes the list of authorities.
- */
-dir_server_t *
-router_get_fallback_dirserver_by_digest(const char *digest)
-{
- if (!fallback_dir_servers)
- return NULL;
-
- if (!digest)
- return NULL;
-
- SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
- {
- if (tor_memeq(ds->digest, digest, DIGEST_LEN))
- return ds;
- });
-
- return NULL;
-}
-
-/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
- * or 0 if no such fallback is in the list of fallback_dir_servers.
- * (fallback_dir_servers is affected by the FallbackDir and
- * UseDefaultFallbackDirs torrc options.)
- * The list of fallback directories includes the list of authorities.
- */
-int
-router_digest_is_fallback_dir(const char *digest)
-{
- return (router_get_fallback_dirserver_by_digest(digest) != NULL);
-}
-
-/** Return the dir_server_t for the directory authority whose
- * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
- * is known.
- */
-MOCK_IMPL(dir_server_t *,
-trusteddirserver_get_by_v3_auth_digest, (const char *digest))
-{
- if (!trusted_dir_servers)
- return NULL;
-
- SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
- {
- if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
- (ds->type & V3_DIRINFO))
- return ds;
- });
-
- return NULL;
-}
-
-/** Try to find a running directory authority. Flags are as for
- * router_pick_directory_server.
- */
-const routerstatus_t *
-router_pick_trusteddirserver(dirinfo_type_t type, int flags)
-{
- return router_pick_dirserver_generic(trusted_dir_servers, type, flags);
-}
-
-/** Try to find a running fallback directory. Flags are as for
- * router_pick_directory_server.
- */
-const routerstatus_t *
-router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
-{
- return router_pick_dirserver_generic(fallback_dir_servers, type, flags);
-}
-
-/** Try to find a running fallback directory. Flags are as for
- * router_pick_directory_server.
- */
-static const routerstatus_t *
-router_pick_dirserver_generic(smartlist_t *sourcelist,
- dirinfo_type_t type, int flags)
-{
- const routerstatus_t *choice;
- int busy = 0;
-
- if (smartlist_len(sourcelist) == 1) {
- /* If there's only one choice, then we should disable the logic that
- * would otherwise prevent us from choosing ourself. */
- flags |= PDS_ALLOW_SELF;
- }
-
- choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
- if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
- return choice;
- if (busy) {
- /* If the reason that we got no server is that servers are "busy",
- * we must be excluding good servers because we already have serverdesc
- * fetches with them. Do not mark down servers up because of this. */
- tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
- PDS_NO_EXISTING_MICRODESC_FETCH)));
- return NULL;
- }
-
- log_info(LD_DIR,
- "No dirservers are reachable. Trying them all again.");
- mark_all_dirservers_up(sourcelist);
- return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
-}
-
-/* Check if we already have a directory fetch from ap, for serverdesc
- * (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
- */
-STATIC int
-router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
- int microdesc)
-{
- if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
- return 0;
- }
-
- /* XX/teor - we're not checking tunnel connections here, see #17848
- */
- if (serverdesc && (
- connection_get_by_type_addr_port_purpose(
- CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
- || connection_get_by_type_addr_port_purpose(
- CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
- return 1;
- }
-
- if (microdesc && (
- connection_get_by_type_addr_port_purpose(
- CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
- return 1;
- }
-
- return 0;
-}
-
-/* Check if we already have a directory fetch from the ipv4 or ipv6
- * router, for serverdesc (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- */
-static int
-router_is_already_dir_fetching_(uint32_t ipv4_addr,
- const tor_addr_t *ipv6_addr,
- uint16_t dir_port,
- int serverdesc,
- int microdesc)
-{
- tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
-
- /* Assume IPv6 DirPort is the same as IPv4 DirPort */
- tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
- ipv4_dir_ap.port = dir_port;
- tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
- ipv6_dir_ap.port = dir_port;
-
- return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
- || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
-}
-
-#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
-#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
-#endif
-
-/* Log a message if rs is not found or not a preferred address */
-static void
-router_picked_poor_directory_log(const routerstatus_t *rs)
-{
- const networkstatus_t *usable_consensus;
- usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
- usable_consensus_flavor());
-
-#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
- /* Don't log early in the bootstrap process, it's normal to pick from a
- * small pool of nodes. Of course, this won't help if we're trying to
- * diagnose bootstrap issues. */
- if (!smartlist_len(nodelist_get_list()) || !usable_consensus
- || !router_have_minimum_dir_info()) {
- return;
- }
-#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
-
- /* We couldn't find a node, or the one we have doesn't fit our preferences.
- * Sometimes this is normal, sometimes it can be a reachability issue. */
- if (!rs) {
- /* This happens a lot, so it's at debug level */
- log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
- "we couldn't find a directory that fit our criteria. "
- "Perhaps we will succeed next time with less strict criteria.");
- } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
- && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
- ) {
- /* This is rare, and might be interesting to users trying to diagnose
- * connection issues on dual-stack machines. */
- log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
- "addresses for launching an outgoing connection: "
- "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
- routerstatus_describe(rs),
- fmt_addr32(rs->addr), rs->or_port,
- rs->dir_port, fmt_addr(&rs->ipv6_addr),
- rs->ipv6_orport, rs->dir_port);
- }
-}
-
-#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
-
-/** How long do we avoid using a directory server after it's given us a 503? */
-#define DIR_503_TIMEOUT (60*60)
-
-/* Common retry code for router_pick_directory_server_impl and
- * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
- * Must be called before RETRY_WITHOUT_EXCLUDE().
- *
- * If we got no result, and we are applying IP preferences, and we are a
- * client that could use an alternate IP version, try again with the
- * opposite preferences. */
-#define RETRY_ALTERNATE_IP_VERSION(retry_label) \
- STMT_BEGIN \
- if (result == NULL && try_ip_pref && options->ClientUseIPv4 \
- && fascist_firewall_use_ipv6(options) && !server_mode(options) \
- && !n_busy) { \
- n_excluded = 0; \
- n_busy = 0; \
- try_ip_pref = 0; \
- goto retry_label; \
- } \
- STMT_END \
-
-/* Common retry code for router_pick_directory_server_impl and
- * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
- * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
- *
- * If we got no result, and we are excluding nodes, and StrictNodes is
- * not set, try again without excluding nodes. */
-#define RETRY_WITHOUT_EXCLUDE(retry_label) \
- STMT_BEGIN \
- if (result == NULL && try_excluding && !options->StrictNodes \
- && n_excluded && !n_busy) { \
- try_excluding = 0; \
- n_excluded = 0; \
- n_busy = 0; \
- try_ip_pref = 1; \
- goto retry_label; \
- } \
- STMT_END
-
-/* Common code used in the loop within router_pick_directory_server_impl and
- * router_pick_trusteddirserver_impl.
- *
- * Check if the given <b>identity</b> supports extrainfo. If not, skip further
- * checks.
- */
-#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity) \
- STMT_BEGIN \
- int is_trusted_extrainfo = router_digest_is_trusted_dir_type( \
- (identity), EXTRAINFO_DIRINFO); \
- if (((type) & EXTRAINFO_DIRINFO) && \
- !router_supports_extrainfo((identity), is_trusted_extrainfo)) \
- continue; \
- STMT_END
-
-/* When iterating through the routerlist, can OR address/port preference
- * and reachability checks be skipped?
- */
-int
-router_skip_or_reachability(const or_options_t *options, int try_ip_pref)
-{
- /* Servers always have and prefer IPv4.
- * And if clients are checking against the firewall for reachability only,
- * but there's no firewall, don't bother checking */
- return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or());
-}
-
-/* When iterating through the routerlist, can Dir address/port preference
- * and reachability checks be skipped?
- */
-static int
-router_skip_dir_reachability(const or_options_t *options, int try_ip_pref)
-{
- /* Servers always have and prefer IPv4.
- * And if clients are checking against the firewall for reachability only,
- * but there's no firewall, don't bother checking */
- return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir());
-}
-
-/** Pick a random running valid directory server/mirror from our
- * routerlist. Arguments are as for router_pick_directory_server(), except:
- *
- * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
- * directories that we excluded for no other reason than
- * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
- */
-STATIC const routerstatus_t *
-router_pick_directory_server_impl(dirinfo_type_t type, int flags,
- int *n_busy_out)
-{
- const or_options_t *options = get_options();
- const node_t *result;
- smartlist_t *direct, *tunnel;
- smartlist_t *trusted_direct, *trusted_tunnel;
- smartlist_t *overloaded_direct, *overloaded_tunnel;
- time_t now = time(NULL);
- const networkstatus_t *consensus = networkstatus_get_latest_consensus();
- const int requireother = ! (flags & PDS_ALLOW_SELF);
- const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
- const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
- const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
- int try_excluding = 1, n_excluded = 0, n_busy = 0;
- int try_ip_pref = 1;
-
- if (!consensus)
- return NULL;
-
- retry_search:
-
- direct = smartlist_new();
- tunnel = smartlist_new();
- trusted_direct = smartlist_new();
- trusted_tunnel = smartlist_new();
- overloaded_direct = smartlist_new();
- overloaded_tunnel = smartlist_new();
-
- const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
- const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
- const int must_have_or = directory_must_use_begindir(options);
-
- /* Find all the running dirservers we know about. */
- SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
- int is_trusted;
- int is_overloaded;
- const routerstatus_t *status = node->rs;
- const country_t country = node->country;
- if (!status)
- continue;
-
- if (!node->is_running || !node_is_dir(node) || !node->is_valid)
- continue;
- if (requireother && router_digest_is_me(node->identity))
- continue;
-
- SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
-
- if (try_excluding &&
- routerset_contains_routerstatus(options->ExcludeNodes, status,
- country)) {
- ++n_excluded;
- continue;
- }
-
- if (router_is_already_dir_fetching_(status->addr,
- &status->ipv6_addr,
- status->dir_port,
- no_serverdesc_fetching,
- no_microdesc_fetching)) {
- ++n_busy;
- continue;
- }
-
- is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
- is_trusted = router_digest_is_trusted_dir(node->identity);
-
- /* Clients use IPv6 addresses if the server has one and the client
- * prefers IPv6.
- * Add the router if its preferred address and port are reachable.
- * If we don't get any routers, we'll try again with the non-preferred
- * address for each router (if any). (To ensure correct load-balancing
- * we try routers that only have one address both times.)
- */
- if (!fascistfirewall || skip_or_fw ||
- fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
- try_ip_pref))
- smartlist_add(is_trusted ? trusted_tunnel :
- is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
- else if (!must_have_or && (skip_dir_fw ||
- fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
- try_ip_pref)))
- smartlist_add(is_trusted ? trusted_direct :
- is_overloaded ? overloaded_direct : direct, (void*)node);
- } SMARTLIST_FOREACH_END(node);
-
- if (smartlist_len(tunnel)) {
- result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
- } else if (smartlist_len(overloaded_tunnel)) {
- result = node_sl_choose_by_bandwidth(overloaded_tunnel,
- WEIGHT_FOR_DIR);
- } else if (smartlist_len(trusted_tunnel)) {
- /* FFFF We don't distinguish between trusteds and overloaded trusteds
- * yet. Maybe one day we should. */
- /* FFFF We also don't load balance over authorities yet. I think this
- * is a feature, but it could easily be a bug. -RD */
- result = smartlist_choose(trusted_tunnel);
- } else if (smartlist_len(direct)) {
- result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
- } else if (smartlist_len(overloaded_direct)) {
- result = node_sl_choose_by_bandwidth(overloaded_direct,
- WEIGHT_FOR_DIR);
- } else {
- result = smartlist_choose(trusted_direct);
- }
- smartlist_free(direct);
- smartlist_free(tunnel);
- smartlist_free(trusted_direct);
- smartlist_free(trusted_tunnel);
- smartlist_free(overloaded_direct);
- smartlist_free(overloaded_tunnel);
-
- RETRY_ALTERNATE_IP_VERSION(retry_search);
-
- RETRY_WITHOUT_EXCLUDE(retry_search);
-
- if (n_busy_out)
- *n_busy_out = n_busy;
-
- router_picked_poor_directory_log(result ? result->rs : NULL);
-
- return result ? result->rs : NULL;
-}
-
-/** Pick a random element from a list of dir_server_t, weighting by their
- * <b>weight</b> field. */
-static const dir_server_t *
-dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
-{
- int n = smartlist_len(servers);
- int i;
- double *weights_dbl;
- uint64_t *weights_u64;
- const dir_server_t *ds;
-
- weights_dbl = tor_calloc(n, sizeof(double));
- weights_u64 = tor_calloc(n, sizeof(uint64_t));
- for (i = 0; i < n; ++i) {
- ds = smartlist_get(servers, i);
- weights_dbl[i] = ds->weight;
- if (ds->is_authority)
- weights_dbl[i] *= authority_weight;
- }
-
- scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
- i = choose_array_element_by_weight(weights_u64, n);
- tor_free(weights_dbl);
- tor_free(weights_u64);
- return (i < 0) ? NULL : smartlist_get(servers, i);
-}
-
-/** Choose randomly from among the dir_server_ts in sourcelist that
- * are up. Flags are as for router_pick_directory_server_impl().
- */
-static const routerstatus_t *
-router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
- dirinfo_type_t type, int flags,
- int *n_busy_out)
-{
- const or_options_t *options = get_options();
- smartlist_t *direct, *tunnel;
- smartlist_t *overloaded_direct, *overloaded_tunnel;
- const routerinfo_t *me = router_get_my_routerinfo();
- const routerstatus_t *result = NULL;
- time_t now = time(NULL);
- const int requireother = ! (flags & PDS_ALLOW_SELF);
- const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
- const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
- const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
- const double auth_weight = (sourcelist == fallback_dir_servers) ?
- options->DirAuthorityFallbackRate : 1.0;
- smartlist_t *pick_from;
- int n_busy = 0;
- int try_excluding = 1, n_excluded = 0;
- int try_ip_pref = 1;
-
- if (!sourcelist)
- return NULL;
-
- retry_search:
-
- direct = smartlist_new();
- tunnel = smartlist_new();
- overloaded_direct = smartlist_new();
- overloaded_tunnel = smartlist_new();
-
- const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
- const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
- const int must_have_or = directory_must_use_begindir(options);
-
- SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
- {
- int is_overloaded =
- d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
- if (!d->is_running) continue;
- if ((type & d->type) == 0)
- continue;
-
- SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
-
- if (requireother && me && router_digest_is_me(d->digest))
- continue;
- if (try_excluding &&
- routerset_contains_routerstatus(options->ExcludeNodes,
- &d->fake_status, -1)) {
- ++n_excluded;
- continue;
- }
-
- if (router_is_already_dir_fetching_(d->addr,
- &d->ipv6_addr,
- d->dir_port,
- no_serverdesc_fetching,
- no_microdesc_fetching)) {
- ++n_busy;
- continue;
- }
-
- /* Clients use IPv6 addresses if the server has one and the client
- * prefers IPv6.
- * Add the router if its preferred address and port are reachable.
- * If we don't get any routers, we'll try again with the non-preferred
- * address for each router (if any). (To ensure correct load-balancing
- * we try routers that only have one address both times.)
- */
- if (!fascistfirewall || skip_or_fw ||
- fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
- try_ip_pref))
- smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
- else if (!must_have_or && (skip_dir_fw ||
- fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
- try_ip_pref)))
- smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
- }
- SMARTLIST_FOREACH_END(d);
-
- if (smartlist_len(tunnel)) {
- pick_from = tunnel;
- } else if (smartlist_len(overloaded_tunnel)) {
- pick_from = overloaded_tunnel;
- } else if (smartlist_len(direct)) {
- pick_from = direct;
- } else {
- pick_from = overloaded_direct;
+ store->journal_len = (size_t) st.st_size;
+ tor_free(contents);
}
- {
- const dir_server_t *selection =
- dirserver_choose_by_weight(pick_from, auth_weight);
+ tor_free(fname);
- if (selection)
- result = &selection->fake_status;
+ if (store->journal_len) {
+ /* Always clear the journal on startup.*/
+ router_rebuild_store(RRS_FORCE, store);
+ } else if (!extrainfo) {
+ /* Don't cache expired routers. (This is in an else because
+ * router_rebuild_store() also calls remove_old_routers().) */
+ routerlist_remove_old_routers();
}
- smartlist_free(direct);
- smartlist_free(tunnel);
- smartlist_free(overloaded_direct);
- smartlist_free(overloaded_tunnel);
-
- RETRY_ALTERNATE_IP_VERSION(retry_search);
-
- RETRY_WITHOUT_EXCLUDE(retry_search);
+ return 0;
+}
- router_picked_poor_directory_log(result);
+/** Load all cached router descriptors and extra-info documents from the
+ * store. Return 0 on success and -1 on failure.
+ */
+int
+router_reload_router_list(void)
+{
+ routerlist_t *rl = router_get_routerlist();
+ if (router_reload_router_list_impl(&rl->desc_store))
+ return -1;
+ if (router_reload_router_list_impl(&rl->extrainfo_store))
+ return -1;
+ return 0;
+}
- if (n_busy_out)
- *n_busy_out = n_busy;
- return result;
+/* When iterating through the routerlist, can OR address/port preference
+ * and reachability checks be skipped?
+ */
+int
+router_skip_or_reachability(const or_options_t *options, int try_ip_pref)
+{
+ /* Servers always have and prefer IPv4.
+ * And if clients are checking against the firewall for reachability only,
+ * but there's no firewall, don't bother checking */
+ return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or());
}
-/** Mark as running every dir_server_t in <b>server_list</b>. */
-static void
-mark_all_dirservers_up(smartlist_t *server_list)
+/* When iterating through the routerlist, can Dir address/port preference
+ * and reachability checks be skipped?
+ */
+int
+router_skip_dir_reachability(const or_options_t *options, int try_ip_pref)
{
- if (server_list) {
- SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
- routerstatus_t *rs;
- node_t *node;
- dir->is_running = 1;
- node = node_get_mutable_by_id(dir->digest);
- if (node)
- node->is_running = 1;
- rs = router_get_mutable_consensus_status_by_id(dir->digest);
- if (rs) {
- rs->last_dir_503_at = 0;
- control_event_networkstatus_changed_single(rs);
- }
- } SMARTLIST_FOREACH_END(dir);
- }
- router_dir_info_changed();
+ /* Servers always have and prefer IPv4.
+ * And if clients are checking against the firewall for reachability only,
+ * but there's no firewall, don't bother checking */
+ return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir());
}
/** Return true iff r1 and r2 have the same address and OR port. */
@@ -2321,35 +491,6 @@ routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2)
r1->ipv6_orport == r2->ipv6_orport;
}
-/** Reset all internal variables used to count failed downloads of network
- * status objects. */
-void
-router_reset_status_download_failures(void)
-{
- mark_all_dirservers_up(fallback_dir_servers);
-}
-
-/** Given a <b>router</b>, add every node_t in its family (including the
- * node itself!) to <b>sl</b>.
- *
- * Note the type mismatch: This function takes a routerinfo, but adds nodes
- * to the smartlist!
- */
-static void
-routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
-{
- /* XXXX MOVE ? */
- node_t fake_node;
- const node_t *node = node_get_by_id(router->cache_info.identity_digest);
- if (node == NULL) {
- memset(&fake_node, 0, sizeof(fake_node));
- fake_node.ri = (routerinfo_t *)router;
- memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
- node = &fake_node;
- }
- nodelist_add_node_and_family(sl, node);
-}
-
/** Add every suitable node from our nodelist to <b>sl</b>, so that
* we can pick a node for a circuit.
*/
@@ -2431,511 +572,6 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router)
return result;
}
-/** Given an array of double/uint64_t unions that are currently being used as
- * doubles, convert them to uint64_t, and try to scale them linearly so as to
- * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
- * the sum of all elements in the array _before_ scaling. */
-STATIC void
-scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
- int n_entries,
- uint64_t *total_out)
-{
- double total = 0.0;
- double scale_factor = 0.0;
- int i;
-
- for (i = 0; i < n_entries; ++i)
- total += entries_in[i];
-
- if (total > 0.0) {
- scale_factor = ((double)INT64_MAX) / total;
- scale_factor /= 4.0; /* make sure we're very far away from overflowing */
- }
-
- for (i = 0; i < n_entries; ++i)
- entries_out[i] = tor_llround(entries_in[i] * scale_factor);
-
- if (total_out)
- *total_out = (uint64_t) total;
-}
-
-/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
- * choosing each element with a probability proportional to its (uint64_t)
- * value, and return the index of that element. If all elements are 0, choose
- * an index at random. Return -1 on error.
- */
-STATIC int
-choose_array_element_by_weight(const uint64_t *entries, int n_entries)
-{
- int i;
- uint64_t rand_val;
- uint64_t total = 0;
-
- for (i = 0; i < n_entries; ++i)
- total += entries[i];
-
- if (n_entries < 1)
- return -1;
-
- if (total == 0)
- return crypto_rand_int(n_entries);
-
- tor_assert(total < INT64_MAX);
-
- rand_val = crypto_rand_uint64(total);
-
- return select_array_member_cumulative_timei(
- entries, n_entries, total, rand_val);
-}
-
-/** When weighting bridges, enforce these values as lower and upper
- * bound for believable bandwidth, because there is no way for us
- * to verify a bridge's bandwidth currently. */
-#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */
-#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
-
-/** Return the smaller of the router's configured BandwidthRate
- * and its advertised capacity, making sure to stay within the
- * interval between bridge-min-believe-bw and
- * bridge-max-believe-bw. */
-static uint32_t
-bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
-{
- uint32_t result = router->bandwidthcapacity;
- if (result > router->bandwidthrate)
- result = router->bandwidthrate;
- if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
- result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
- else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
- result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
- return result;
-}
-
-/** Return bw*1000, unless bw*1000 would overflow, in which case return
- * INT32_MAX. */
-static inline int32_t
-kb_to_bytes(uint32_t bw)
-{
- return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
-}
-
-/** Helper function:
- * choose a random element of smartlist <b>sl</b> of nodes, weighted by
- * the advertised bandwidth of each element using the consensus
- * bandwidth weights.
- *
- * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
- * nodes' bandwidth equally regardless of their Exit status, since there may
- * be some in the list because they exit to obscure ports. If
- * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
- * exit-node's bandwidth less depending on the smallness of the fraction of
- * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
- * guard node: consider all guard's bandwidth equally. Otherwise, weight
- * guards proportionally less.
- */
-static const node_t *
-smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
- bandwidth_weight_rule_t rule)
-{
- double *bandwidths_dbl=NULL;
- uint64_t *bandwidths_u64=NULL;
-
- if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
- return NULL;
-
- bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
- scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
- smartlist_len(sl), NULL);
-
- {
- int idx = choose_array_element_by_weight(bandwidths_u64,
- smartlist_len(sl));
- tor_free(bandwidths_dbl);
- tor_free(bandwidths_u64);
- return idx < 0 ? NULL : smartlist_get(sl, idx);
- }
-}
-
-/** Given a list of routers and a weighting rule as in
- * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
- * values for each node and store them in a freshly allocated
- * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
- * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
- * of all the bandwidths.
- * Return 0 on success, -1 on failure. */
-static int
-compute_weighted_bandwidths(const smartlist_t *sl,
- bandwidth_weight_rule_t rule,
- double **bandwidths_out,
- double *total_bandwidth_out)
-{
- int64_t weight_scale;
- double Wg = -1, Wm = -1, We = -1, Wd = -1;
- double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
- guardfraction_bandwidth_t guardfraction_bw;
- double *bandwidths = NULL;
- double total_bandwidth = 0.0;
-
- tor_assert(sl);
- tor_assert(bandwidths_out);
-
- /* Can't choose exit and guard at same time */
- tor_assert(rule == NO_WEIGHTING ||
- rule == WEIGHT_FOR_EXIT ||
- rule == WEIGHT_FOR_GUARD ||
- rule == WEIGHT_FOR_MID ||
- rule == WEIGHT_FOR_DIR);
-
- *bandwidths_out = NULL;
-
- if (total_bandwidth_out) {
- *total_bandwidth_out = 0.0;
- }
-
- if (smartlist_len(sl) == 0) {
- log_info(LD_CIRC,
- "Empty routerlist passed in to consensus weight node "
- "selection for rule %s",
- bandwidth_weight_rule_to_string(rule));
- return -1;
- }
-
- weight_scale = networkstatus_get_weight_scale_param(NULL);
-
- if (rule == WEIGHT_FOR_GUARD) {
- Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
- Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
- We = 0;
- Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
-
- Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
- Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
- Web = networkstatus_get_bw_weight(NULL, "Web", -1);
- Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
- } else if (rule == WEIGHT_FOR_MID) {
- Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
- Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
- We = networkstatus_get_bw_weight(NULL, "Wme", -1);
- Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
-
- Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
- Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
- Web = networkstatus_get_bw_weight(NULL, "Web", -1);
- Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
- } else if (rule == WEIGHT_FOR_EXIT) {
- // Guards CAN be exits if they have weird exit policies
- // They are d then I guess...
- We = networkstatus_get_bw_weight(NULL, "Wee", -1);
- Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
- Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
- Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
-
- Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
- Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
- Web = networkstatus_get_bw_weight(NULL, "Web", -1);
- Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
- } else if (rule == WEIGHT_FOR_DIR) {
- We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
- Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
- Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
- Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
-
- Wgb = Wmb = Web = Wdb = weight_scale;
- } else if (rule == NO_WEIGHTING) {
- Wg = Wm = We = Wd = weight_scale;
- Wgb = Wmb = Web = Wdb = weight_scale;
- }
-
- if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
- || Web < 0) {
- log_debug(LD_CIRC,
- "Got negative bandwidth weights. Defaulting to naive selection"
- " algorithm.");
- Wg = Wm = We = Wd = weight_scale;
- Wgb = Wmb = Web = Wdb = weight_scale;
- }
-
- Wg /= weight_scale;
- Wm /= weight_scale;
- We /= weight_scale;
- Wd /= weight_scale;
-
- Wgb /= weight_scale;
- Wmb /= weight_scale;
- Web /= weight_scale;
- Wdb /= weight_scale;
-
- bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
-
- // Cycle through smartlist and total the bandwidth.
- static int warned_missing_bw = 0;
- SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
- int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
- double weight = 1;
- double weight_without_guard_flag = 0; /* Used for guardfraction */
- double final_weight = 0;
- is_exit = node->is_exit && ! node->is_bad_exit;
- is_guard = node->is_possible_guard;
- is_dir = node_is_dir(node);
- if (node->rs) {
- if (!node->rs->has_bandwidth) {
- /* This should never happen, unless all the authorities downgrade
- * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
- if (! warned_missing_bw) {
- log_warn(LD_BUG,
- "Consensus is missing some bandwidths. Using a naive "
- "router selection algorithm");
- warned_missing_bw = 1;
- }
- this_bw = 30000; /* Chosen arbitrarily */
- } else {
- this_bw = kb_to_bytes(node->rs->bandwidth_kb);
- }
- } else if (node->ri) {
- /* bridge or other descriptor not in our consensus */
- this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
- } else {
- /* We can't use this one. */
- continue;
- }
-
- if (is_guard && is_exit) {
- weight = (is_dir ? Wdb*Wd : Wd);
- weight_without_guard_flag = (is_dir ? Web*We : We);
- } else if (is_guard) {
- weight = (is_dir ? Wgb*Wg : Wg);
- weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
- } else if (is_exit) {
- weight = (is_dir ? Web*We : We);
- } else { // middle
- weight = (is_dir ? Wmb*Wm : Wm);
- }
- /* These should be impossible; but overflows here would be bad, so let's
- * make sure. */
- if (this_bw < 0)
- this_bw = 0;
- if (weight < 0.0)
- weight = 0.0;
- if (weight_without_guard_flag < 0.0)
- weight_without_guard_flag = 0.0;
-
- /* If guardfraction information is available in the consensus, we
- * want to calculate this router's bandwidth according to its
- * guardfraction. Quoting from proposal236:
- *
- * Let Wpf denote the weight from the 'bandwidth-weights' line a
- * client would apply to N for position p if it had the guard
- * flag, Wpn the weight if it did not have the guard flag, and B the
- * measured bandwidth of N in the consensus. Then instead of choosing
- * N for position p proportionally to Wpf*B or Wpn*B, clients should
- * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
- */
- if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
- /* XXX The assert should actually check for is_guard. However,
- * that crashes dirauths because of #13297. This should be
- * equivalent: */
- tor_assert(node->rs->is_possible_guard);
-
- guard_get_guardfraction_bandwidth(&guardfraction_bw,
- this_bw,
- node->rs->guardfraction_percentage);
-
- /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
- final_weight =
- guardfraction_bw.guard_bw * weight +
- guardfraction_bw.non_guard_bw * weight_without_guard_flag;
-
- log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
- node->rs->nickname, final_weight, weight*this_bw,
- bandwidth_weight_rule_to_string(rule));
- } else { /* no guardfraction information. calculate the weight normally. */
- final_weight = weight*this_bw;
- }
-
- bandwidths[node_sl_idx] = final_weight;
- total_bandwidth += final_weight;
- } SMARTLIST_FOREACH_END(node);
-
- log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
- "on weights "
- "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
- bandwidth_weight_rule_to_string(rule),
- Wg, Wm, We, Wd, total_bandwidth);
-
- *bandwidths_out = bandwidths;
-
- if (total_bandwidth_out) {
- *total_bandwidth_out = total_bandwidth;
- }
-
- return 0;
-}
-
-/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
- * by their weighted bandwidths with rule <b>rule</b>, for which we have
- * descriptors.
- *
- * If <b>for_direct_connect</b> is true, we intend to connect to the node
- * directly, as the first hop of a circuit; otherwise, we intend to connect
- * to it indirectly, or use it as if we were connecting to it indirectly. */
-double
-frac_nodes_with_descriptors(const smartlist_t *sl,
- bandwidth_weight_rule_t rule,
- int for_direct_conn)
-{
- double *bandwidths = NULL;
- double total, present;
-
- if (smartlist_len(sl) == 0)
- return 0.0;
-
- if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
- total <= 0.0) {
- int n_with_descs = 0;
- SMARTLIST_FOREACH(sl, const node_t *, node, {
- if (node_has_preferred_descriptor(node, for_direct_conn))
- n_with_descs++;
- });
- tor_free(bandwidths);
- return ((double)n_with_descs) / smartlist_len(sl);
- }
-
- present = 0.0;
- SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
- if (node_has_preferred_descriptor(node, for_direct_conn))
- present += bandwidths[node_sl_idx];
- } SMARTLIST_FOREACH_END(node);
-
- tor_free(bandwidths);
-
- return present / total;
-}
-
-/** Choose a random element of status list <b>sl</b>, weighted by
- * the advertised bandwidth of each node */
-const node_t *
-node_sl_choose_by_bandwidth(const smartlist_t *sl,
- bandwidth_weight_rule_t rule)
-{ /*XXXX MOVE */
- return smartlist_choose_node_by_bandwidth_weights(sl, rule);
-}
-
-/** Return a random running node from the nodelist. Never
- * pick a node that is in
- * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
- * even if they are the only nodes available.
- * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
- * a minimum uptime, return one of those.
- * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
- * advertised capacity of each router.
- * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
- * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
- * picking an exit node, otherwise we weight bandwidths for picking a relay
- * node (that is, possibly discounting exit nodes).
- * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
- * have a routerinfo or microdescriptor -- that is, enough info to be
- * used to build a circuit.
- * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
- * have an address that is preferred by the ClientPreferIPv6ORPort setting
- * (regardless of this flag, we exclude nodes that aren't allowed by the
- * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
- */
-const node_t *
-router_choose_random_node(smartlist_t *excludedsmartlist,
- routerset_t *excludedset,
- router_crn_flags_t flags)
-{ /* XXXX MOVE */
- const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
- const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
- const int need_guard = (flags & CRN_NEED_GUARD) != 0;
- const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
- const int need_desc = (flags & CRN_NEED_DESC) != 0;
- const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
- const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
- const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
-
- smartlist_t *sl=smartlist_new(),
- *excludednodes=smartlist_new();
- const node_t *choice = NULL;
- const routerinfo_t *r;
- bandwidth_weight_rule_t rule;
-
- tor_assert(!(weight_for_exit && need_guard));
- rule = weight_for_exit ? WEIGHT_FOR_EXIT :
- (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
-
- SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
- if (node_allows_single_hop_exits(node)) {
- /* Exclude relays that allow single hop exit circuits. This is an
- * obsolete option since 0.2.9.2-alpha and done by default in
- * 0.3.1.0-alpha. */
- smartlist_add(excludednodes, node);
- } else if (rendezvous_v3 &&
- !node_supports_v3_rendezvous_point(node)) {
- /* Exclude relays that do not support to rendezvous for a hidden service
- * version 3. */
- smartlist_add(excludednodes, node);
- }
- } SMARTLIST_FOREACH_END(node);
-
- /* If the node_t is not found we won't be to exclude ourself but we
- * won't be able to pick ourself in router_choose_random_node() so
- * this is fine to at least try with our routerinfo_t object. */
- if ((r = router_get_my_routerinfo()))
- routerlist_add_node_and_family(excludednodes, r);
-
- router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
- need_guard, need_desc, pref_addr,
- direct_conn);
- log_debug(LD_CIRC,
- "We found %d running nodes.",
- smartlist_len(sl));
-
- smartlist_subtract(sl,excludednodes);
- log_debug(LD_CIRC,
- "We removed %d excludednodes, leaving %d nodes.",
- smartlist_len(excludednodes),
- smartlist_len(sl));
-
- if (excludedsmartlist) {
- smartlist_subtract(sl,excludedsmartlist);
- log_debug(LD_CIRC,
- "We removed %d excludedsmartlist, leaving %d nodes.",
- smartlist_len(excludedsmartlist),
- smartlist_len(sl));
- }
- if (excludedset) {
- routerset_subtract_nodes(sl,excludedset);
- log_debug(LD_CIRC,
- "We removed excludedset, leaving %d nodes.",
- smartlist_len(sl));
- }
-
- // Always weight by bandwidth
- choice = node_sl_choose_by_bandwidth(sl, rule);
-
- smartlist_free(sl);
- if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
- /* try once more -- recurse but with fewer restrictions. */
- log_info(LD_CIRC,
- "We couldn't find any live%s%s%s routers; falling back "
- "to list of all routers.",
- need_capacity?", fast":"",
- need_uptime?", stable":"",
- need_guard?", guard":"");
- flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
- CRN_PREF_ADDR);
- choice = router_choose_random_node(
- excludedsmartlist, excludedset, flags);
- }
- smartlist_free(excludednodes);
- if (!choice) {
- log_warn(LD_CIRC,
- "No available nodes when trying to choose node. Failing.");
- }
- return choice;
-}
-
/** Helper: given an extended nickname in <b>hexdigest</b> try to decode it.
* Return 0 on success, -1 on failure. Store the result into the
* DIGEST_LEN-byte buffer at <b>digest_out</b>, the single character at
@@ -3013,23 +649,6 @@ hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
return tor_memeq(digest, identity_digest, DIGEST_LEN);
}
-/** Return true iff <b>digest</b> is the digest of the identity key of a
- * trusted directory matching at least one bit of <b>type</b>. If <b>type</b>
- * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
-int
-router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
-{
- if (!trusted_dir_servers)
- return 0;
- if (authdir_mode(get_options()) && router_digest_is_me(digest))
- return 1;
- SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
- if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
- return (!type) || ((type & ent->type) != 0);
- });
- return 0;
-}
-
/** If hexdigest is correctly formed, base16_decode it into
* digest, which must have DIGEST_LEN space in it.
* Return 0 on success, -1 on failure.
@@ -3808,14 +1427,7 @@ routerlist_free_all(void)
smartlist_free(warned_nicknames);
warned_nicknames = NULL;
}
- clear_dir_servers();
- smartlist_free(trusted_dir_servers);
- smartlist_free(fallback_dir_servers);
- trusted_dir_servers = fallback_dir_servers = NULL;
- if (trusted_dir_certs) {
- digestmap_free(trusted_dir_certs, cert_list_free_void);
- trusted_dir_certs = NULL;
- }
+ authcert_free_all();
}
/** Forget that we have issued any router-related warnings, so that we'll
@@ -4621,221 +2233,13 @@ router_exit_policy_rejects_all(const routerinfo_t *router)
return router->policy_is_reject_star;
}
-/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
- * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL,
- * add ourself. If <b>is_authority</b>, this is a directory authority. Return
- * the new directory server entry on success or NULL on failure. */
-static dir_server_t *
-dir_server_new(int is_authority,
- const char *nickname,
- const tor_addr_t *addr,
- const char *hostname,
- uint16_t dir_port, uint16_t or_port,
- const tor_addr_port_t *addrport_ipv6,
- const char *digest, const char *v3_auth_digest,
- dirinfo_type_t type,
- double weight)
-{
- dir_server_t *ent;
- uint32_t a;
- char *hostname_ = NULL;
-
- tor_assert(digest);
-
- if (weight < 0)
- return NULL;
-
- if (tor_addr_family(addr) == AF_INET)
- a = tor_addr_to_ipv4h(addr);
- else
- return NULL;
-
- if (!hostname)
- hostname_ = tor_addr_to_str_dup(addr);
- else
- hostname_ = tor_strdup(hostname);
-
- ent = tor_malloc_zero(sizeof(dir_server_t));
- ent->nickname = nickname ? tor_strdup(nickname) : NULL;
- ent->address = hostname_;
- ent->addr = a;
- ent->dir_port = dir_port;
- ent->or_port = or_port;
- ent->is_running = 1;
- ent->is_authority = is_authority;
- ent->type = type;
- ent->weight = weight;
- if (addrport_ipv6) {
- if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
- log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
- tor_addr_make_unspec(&ent->ipv6_addr);
- } else {
- tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
- ent->ipv6_orport = addrport_ipv6->port;
- }
- } else {
- tor_addr_make_unspec(&ent->ipv6_addr);
- }
-
- memcpy(ent->digest, digest, DIGEST_LEN);
- if (v3_auth_digest && (type & V3_DIRINFO))
- memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
-
- if (nickname)
- tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
- nickname, hostname_, (int)dir_port);
- else
- tor_asprintf(&ent->description, "directory server at %s:%d",
- hostname_, (int)dir_port);
-
- ent->fake_status.addr = ent->addr;
- tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
- memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
- if (nickname)
- strlcpy(ent->fake_status.nickname, nickname,
- sizeof(ent->fake_status.nickname));
- else
- ent->fake_status.nickname[0] = '\0';
- ent->fake_status.dir_port = ent->dir_port;
- ent->fake_status.or_port = ent->or_port;
- ent->fake_status.ipv6_orport = ent->ipv6_orport;
-
- return ent;
-}
-
-/** Create an authoritative directory server at
- * <b>address</b>:<b>port</b>, with identity key <b>digest</b>. If
- * <b>address</b> is NULL, add ourself. Return the new trusted directory
- * server entry on success or NULL if we couldn't add it. */
-dir_server_t *
-trusted_dir_server_new(const char *nickname, const char *address,
- uint16_t dir_port, uint16_t or_port,
- const tor_addr_port_t *ipv6_addrport,
- const char *digest, const char *v3_auth_digest,
- dirinfo_type_t type, double weight)
-{
- uint32_t a;
- tor_addr_t addr;
- char *hostname=NULL;
- dir_server_t *result;
-
- if (!address) { /* The address is us; we should guess. */
- if (resolve_my_address(LOG_WARN, get_options(),
- &a, NULL, &hostname) < 0) {
- log_warn(LD_CONFIG,
- "Couldn't find a suitable address when adding ourself as a "
- "trusted directory server.");
- return NULL;
- }
- if (!hostname)
- hostname = tor_dup_ip(a);
- } else {
- if (tor_lookup_hostname(address, &a)) {
- log_warn(LD_CONFIG,
- "Unable to lookup address for directory server at '%s'",
- address);
- return NULL;
- }
- hostname = tor_strdup(address);
- }
- tor_addr_from_ipv4h(&addr, a);
-
- result = dir_server_new(1, nickname, &addr, hostname,
- dir_port, or_port,
- ipv6_addrport,
- digest,
- v3_auth_digest, type, weight);
- tor_free(hostname);
- return result;
-}
-
-/** Return a new dir_server_t for a fallback directory server at
- * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
- * <b>id_digest</b> */
-dir_server_t *
-fallback_dir_server_new(const tor_addr_t *addr,
- uint16_t dir_port, uint16_t or_port,
- const tor_addr_port_t *addrport_ipv6,
- const char *id_digest, double weight)
-{
- return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
- addrport_ipv6,
- id_digest,
- NULL, ALL_DIRINFO, weight);
-}
-
-/** Add a directory server to the global list(s). */
-void
-dir_server_add(dir_server_t *ent)
-{
- if (!trusted_dir_servers)
- trusted_dir_servers = smartlist_new();
- if (!fallback_dir_servers)
- fallback_dir_servers = smartlist_new();
-
- if (ent->is_authority)
- smartlist_add(trusted_dir_servers, ent);
-
- smartlist_add(fallback_dir_servers, ent);
- router_dir_info_changed();
-}
-
-/** Free storage held in <b>cert</b>. */
-void
-authority_cert_free_(authority_cert_t *cert)
-{
- if (!cert)
- return;
-
- tor_free(cert->cache_info.signed_descriptor_body);
- crypto_pk_free(cert->signing_key);
- crypto_pk_free(cert->identity_key);
-
- tor_free(cert);
-}
-
-#define dir_server_free(val) \
- FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
-
-/** Free storage held in <b>ds</b>. */
-static void
-dir_server_free_(dir_server_t *ds)
-{
- if (!ds)
- return;
-
- tor_free(ds->nickname);
- tor_free(ds->description);
- tor_free(ds->address);
- tor_free(ds);
-}
-
-/** Remove all members from the list of dir servers. */
-void
-clear_dir_servers(void)
-{
- if (fallback_dir_servers) {
- SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
- dir_server_free(ent));
- smartlist_clear(fallback_dir_servers);
- } else {
- fallback_dir_servers = smartlist_new();
- }
- if (trusted_dir_servers) {
- smartlist_clear(trusted_dir_servers);
- } else {
- trusted_dir_servers = smartlist_new();
- }
- router_dir_info_changed();
-}
-
/** For every current directory connection whose purpose is <b>purpose</b>,
* and where the resource being downloaded begins with <b>prefix</b>, split
* rest of the resource into base16 fingerprints (or base64 fingerprints if
* purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the
* corresponding elements of <b>result</b> to a nonzero value.
*/
-static void
+void
list_pending_downloads(digestmap_t *result, digest256map_t *result256,
int purpose, const char *prefix)
{
@@ -4895,41 +2299,6 @@ list_pending_microdesc_downloads(digest256map_t *result)
list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/");
}
-/** For every certificate we are currently downloading by (identity digest,
- * signing key digest) pair, set result[fp_pair] to (void *1).
- */
-static void
-list_pending_fpsk_downloads(fp_pair_map_t *result)
-{
- const char *pfx = "fp-sk/";
- smartlist_t *tmp;
- smartlist_t *conns;
- const char *resource;
-
- tor_assert(result);
-
- tmp = smartlist_new();
- conns = get_connection_array();
-
- SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
- if (conn->type == CONN_TYPE_DIR &&
- conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
- !conn->marked_for_close) {
- resource = TO_DIR_CONN(conn)->requested_resource;
- if (!strcmpstart(resource, pfx))
- dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
- tmp);
- }
- } SMARTLIST_FOREACH_END(conn);
-
- SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
- fp_pair_map_set(result, fp, (void*)1);
- tor_free(fp);
- } SMARTLIST_FOREACH_END(fp);
-
- smartlist_free(tmp);
-}
-
/** Launch downloads for all the descriptors whose digests or digests256
* are listed as digests[i] for lo <= i < hi. (Lo and hi may be out of
* range.) If <b>source</b> is given, download from <b>source</b>;
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
index 4b7406364..c3e97d9dd 100644
--- a/src/feature/nodelist/routerlist.h
+++ b/src/feature/nodelist/routerlist.h
@@ -42,108 +42,13 @@ typedef enum was_router_added_t {
ROUTER_CERTS_EXPIRED = -8
} was_router_added_t;
-/** Flags to be passed to control router_choose_random_node() to indicate what
- * kind of nodes to pick according to what algorithm. */
-typedef enum router_crn_flags_t {
- CRN_NEED_UPTIME = 1<<0,
- CRN_NEED_CAPACITY = 1<<1,
- CRN_NEED_GUARD = 1<<2,
- /* XXXX not used, apparently. */
- CRN_WEIGHT_AS_EXIT = 1<<5,
- CRN_NEED_DESC = 1<<6,
- /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
- CRN_PREF_ADDR = 1<<7,
- /* On clients, only provide nodes that we can connect to directly, based on
- * our firewall rules */
- CRN_DIRECT_CONN = 1<<8,
- /* On clients, only provide nodes with HSRend >= 2 protocol version which
- * is required for hidden service version >= 3. */
- CRN_RENDEZVOUS_V3 = 1<<9,
-} router_crn_flags_t;
+/** How long do we avoid using a directory server after it's given us a 503? */
+#define DIR_503_TIMEOUT (60*60)
-/** Possible ways to weight routers when choosing one randomly. See
- * routerlist_sl_choose_by_bandwidth() for more information.*/
-typedef enum bandwidth_weight_rule_t {
- NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
- WEIGHT_FOR_DIR
-} bandwidth_weight_rule_t;
-
-/* Flags for pick_directory_server() and pick_trusteddirserver(). */
-/** Flag to indicate that we should not automatically be willing to use
- * ourself to answer a directory request.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_ALLOW_SELF (1<<0)
-/** Flag to indicate that if no servers seem to be up, we should mark all
- * directory servers as up and try again.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_RETRY_IF_NO_SERVERS (1<<1)
-/** Flag to indicate that we should not exclude directory servers that
- * our ReachableAddress settings would exclude. This usually means that
- * we're going to connect to the server over Tor, and so we don't need to
- * worry about our firewall telling us we can't.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_IGNORE_FASCISTFIREWALL (1<<2)
-/** Flag to indicate that we should not use any directory authority to which
- * we have an existing directory connection for downloading server descriptors
- * or extrainfo documents.
- *
- * Passed to router_pick_directory_server (et al)
- */
-#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
-/** Flag to indicate that we should not use any directory authority to which
- * we have an existing directory connection for downloading microdescs.
- *
- * Passed to router_pick_directory_server (et al)
- */
-#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
-
-int get_n_authorities(dirinfo_type_t type);
-int trusted_dirs_reload_certs(void);
-
-/*
- * Pass one of these as source to trusted_dirs_load_certs_from_string()
- * to indicate whence string originates; this controls error handling
- * behavior such as marking downloads as failed.
- */
-
-#define TRUSTED_DIRS_CERTS_SRC_SELF 0
-#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
-#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
-#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
-#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
-
-int trusted_dirs_load_certs_from_string(const char *contents, int source,
- int flush, const char *source_dir);
-void trusted_dirs_flush_certs_to_disk(void);
-authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
-authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
-authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
- const char *sk_digest);
-void authority_cert_get_all(smartlist_t *certs_out);
-void authority_cert_dl_failed(const char *id_digest,
- const char *signing_key_digest, int status);
-void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
- const char *dir_hint);
int router_reload_router_list(void);
-int authority_cert_dl_looks_uncertain(const char *id_digest);
-const smartlist_t *router_get_trusted_dir_servers(void);
-const smartlist_t *router_get_fallback_dir_servers(void);
-int authority_cert_is_blacklisted(const authority_cert_t *cert);
-const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
- int flags);
-dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
-dir_server_t *router_get_fallback_dirserver_by_digest(
- const char *digest);
-int router_digest_is_fallback_dir(const char *digest);
-MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
- (const char *d));
-const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
- int flags);
-const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
- int flags);
int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
-int router_get_my_share_of_directory_requests(double *v3_share_out);
+int router_skip_dir_reachability(const or_options_t *options, int try_ip_pref);
void router_reset_status_download_failures(void);
int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
@@ -155,21 +60,6 @@ const routerinfo_t *routerlist_find_my_routerinfo(void);
uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router);
-const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
- bandwidth_weight_rule_t rule);
-double frac_nodes_with_descriptors(const smartlist_t *sl,
- bandwidth_weight_rule_t rule,
- int for_direct_conn);
-
-const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
- struct routerset_t *excludedset,
- router_crn_flags_t flags);
-
-int router_digest_is_trusted_dir_type(const char *digest,
- dirinfo_type_t type);
-#define router_digest_is_trusted_dir(d) \
- router_digest_is_trusted_dir_type((d), NO_DIRINFO)
-
int hexdigest_to_digest(const char *hexdigest, char *digest);
const routerinfo_t *router_get_by_id_digest(const char *digest);
routerinfo_t *router_get_mutable_by_digest(const char *digest);
@@ -194,13 +84,10 @@ void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
void routerlist_free_all(void);
void routerlist_reset_warnings(void);
-MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
-MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
- (const char *digest));
-MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
- (const char *digest));
-MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
- (const char *id_digest, const char *sk_digest));
+/* XXXX move this */
+void list_pending_downloads(digestmap_t *result,
+ digest256map_t *result256,
+ int purpose, const char *prefix);
static int WRA_WAS_ADDED(was_router_added_t s);
static int WRA_WAS_OUTDATED(was_router_added_t s);
@@ -269,21 +156,6 @@ void routerlist_retry_directory_downloads(time_t now);
int router_exit_policy_rejects_all(const routerinfo_t *router);
-dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
- uint16_t dir_port, uint16_t or_port,
- const tor_addr_port_t *addrport_ipv6,
- const char *digest, const char *v3_auth_digest,
- dirinfo_type_t type, double weight);
-dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
- uint16_t dir_port, uint16_t or_port,
- const tor_addr_port_t *addrport_ipv6,
- const char *id_digest, double weight);
-void dir_server_add(dir_server_t *ent);
-
-void authority_cert_free_(authority_cert_t *cert);
-#define authority_cert_free(cert) \
- FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
-void clear_dir_servers(void);
void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
networkstatus_t *consensus);
void update_router_descriptor_downloads(time_t now);
@@ -321,16 +193,6 @@ int hex_digest_nickname_matches(const char *hexdigest,
const char *nickname);
#ifdef ROUTERLIST_PRIVATE
-STATIC int choose_array_element_by_weight(const uint64_t *entries,
- int n_entries);
-STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
- const double *entries_in,
- int n_entries,
- uint64_t *total_out);
-STATIC const routerstatus_t *router_pick_directory_server_impl(
- dirinfo_type_t auth, int flags,
- int *n_busy_out);
-
MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router,
int seconds));
MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
@@ -339,8 +201,6 @@ MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
MOCK_DECL(STATIC void, initiate_descriptor_downloads,
(const routerstatus_t *source, int purpose, smartlist_t *digests,
int lo, int hi, int pds_flags));
-STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
- int serverdesc, int microdesc);
#endif /* defined(ROUTERLIST_PRIVATE) */
diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c
index b76b2974f..5b82c2adf 100644
--- a/src/feature/nodelist/routerparse.c
+++ b/src/feature/nodelist/routerparse.c
@@ -75,6 +75,7 @@
#include "feature/relay/routerkeys.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
+#include "feature/nodelist/authcert.h"
#include "lib/sandbox/sandbox.h"
#include "feature/hs_common/shared_random_client.h"
#include "feature/nodelist/torcert.h"
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index 1f316ebf0..d136eaeb6 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -30,6 +30,8 @@
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "app/config/statefile.h"
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
index 37e604d07..f9508f464 100644
--- a/src/feature/rend/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -33,7 +33,7 @@
#include "core/or/relay.h"
#include "feature/stats/rephist.h"
#include "feature/hs_common/replaycache.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
#include "lib/encoding/confline.h"
diff --git a/src/test/test_config.c b/src/test/test_config.c
index bf21a8d51..f224ddde3 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -38,6 +38,7 @@
#include "feature/rend/rendclient.h"
#include "feature/rend/rendservice.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerset.h"
#include "app/config/statefile.h"
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index 0428ac6fc..4f5a9f58d 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -10,7 +10,7 @@
#include "feature/hs/hs_common.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/rend/rendservice.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/authcert.h"
#include "feature/nodelist/nodelist.h"
#include "test/test.h"
#include "test/test_helpers.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 723799ee8..363539ce4 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -15,6 +15,7 @@
#define ROUTERPARSE_PRIVATE
#define HIBERNATE_PRIVATE
#define NETWORKSTATUS_PRIVATE
+#define NODE_SELECT_PRIVATE
#define RELAY_PRIVATE
#include "core/or/or.h"
@@ -38,6 +39,9 @@
#include "feature/nodelist/networkstatus.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerset.h"
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 09799a0e5..2cfed16b5 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -19,6 +19,8 @@
#include "feature/rend/rendcommon.h"
#include "feature/rend/rendcache.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerlist.h"
#include "test/rend_test_helpers.h"
#include "feature/nodelist/microdesc.h"
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 89d1f4f90..f629596c5 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -12,6 +12,7 @@
#define HIBERNATE_PRIVATE
#define NETWORKSTATUS_PRIVATE
#define ROUTERLIST_PRIVATE
+#define NODE_SELECT_PRIVATE
#define TOR_UNIT_TESTING
#include "core/or/or.h"
#include "app/config/config.h"
@@ -27,6 +28,8 @@
#include "feature/nodelist/nodelist.h"
#include "core/or/policies.h"
#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerset.h"
#include "feature/nodelist/routerparse.h"
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index 70adf580a..725724aa5 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -17,7 +17,8 @@
#include "feature/nodelist/networkstatus.h"
#include "feature/relay/router.h"
#include "feature/relay/routerkeys.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/hs_common/shared_random_client.h"
#include "feature/dircommon/voting_schedule.h"
More information about the tor-commits
mailing list