August 2019 - tor-commits - lists.torproject.org

[tor/master] Avoid using labs() on time_t in channeltls.c
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit 0849d2a2fdaeea2871f32bed35d410f19703aae1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 6 11:11:06 2019 -0400 Avoid using labs() on time_t in channeltls.c On some windows builds, time_t is 64 bits but long is not. This is causing appveyor builds to fail. Also, one of our uses of labs() on time_t was logically incorrect: it was telling us to accept NETINFO cells up to three minutes _before_ the message they were responding to, which doesn't make sense. This patch adds a time_abs() function that we should eventually move to intmath.h or something. For now, though, it will make merges easier to have it file-local in channeltls.c. Fixes bug 31343; bugfix on 0.2.4.4-alpha. --- changes/bug31343 | 9 +++++++++ src/or/channeltls.c | 23 +++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/changes/bug31343 b/changes/bug31343 new file mode 100644 index 000000000..17a8057ea --- /dev/null +++ b/changes/bug31343 @@ -0,0 +1,9 @@ + o Minor bugfixes (compilation): + - Avoid using labs() on time_t, which can cause compilation warnings + on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (clock skew detection): + - Don't believe clock skew results from NETINFO cells that appear to + arrive before the VERSIONS cells they are responding to were sent. + Previously, we would accept them up to 3 minutes "in the past". + Fixes bug 31343; bugfix on 0.2.4.4-alpha. diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 3a352d47f..ea69792f1 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1584,6 +1584,18 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) } /** + * Helper: compute the absolute value of a time_t. + * + * (we need this because labs() doesn't always work for time_t, since + * long can be shorter than time_t.) + */ +static inline time_t +time_abs(time_t val) +{ + return (val < 0) ? -val : val; +} + +/** * Process a 'netinfo' cell * * This function is called to handle an incoming NETINFO cell; read and act @@ -1601,7 +1613,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) uint8_t n_other_addrs; time_t now = time(NULL); - long apparent_skew = 0; + time_t apparent_skew = 0; tor_addr_t my_apparent_addr = TOR_ADDR_NULL; tor_assert(cell); @@ -1659,7 +1671,11 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) /* Decode the cell. */ timestamp = ntohl(get_uint32(cell->payload)); - if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { + const time_t sent_versions_at = + chan->conn->handshake_state->sent_versions_at; + if (now > sent_versions_at && (now - sent_versions_at) < 180) { + /* If we have gotten the NETINFO cell reasonably soon after having + * sent our VERSIONS cell, maybe we can learn skew information from it. */ apparent_skew = now - timestamp; } @@ -1705,7 +1721,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) /* Act on apparent skew. */ /** Warn when we get a netinfo skew with at least this value. */ #define NETINFO_NOTICE_SKEW 3600 - if (labs(apparent_skew) > NETINFO_NOTICE_SKEW && + if (time_abs(apparent_skew) && router_get_by_id_digest(chan->conn->identity_digest)) { int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest); clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL, @@ -2182,4 +2198,3 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan) #undef ERR } -

1 0

[tor/maint-0.4.1] Fix another time_t/long warning for 31343.
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit 878f4409015f741c7075d0ccf3da794a6f313302 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 09:38:03 2019 -0400 Fix another time_t/long warning for 31343. --- src/or/routerlist.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/or/routerlist.c b/src/or/routerlist.c index f73ec9baa..f3b298006 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -5443,7 +5443,7 @@ int router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) { time_t r1pub, r2pub; - long time_difference; + time_t time_difference; tor_assert(r1 && r2); /* r1 should be the one that was published first. */ @@ -5506,7 +5506,9 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) * give or take some slop? */ r1pub = r1->cache_info.published_on; r2pub = r2->cache_info.published_on; - time_difference = labs(r2->uptime - (r1->uptime + (r2pub - r1pub))); + time_difference = r2->uptime - (r1->uptime + (r2pub - r1pub)); + if (time_difference < 0) + time_difference = - time_difference; if (time_difference > ROUTER_ALLOW_UPTIME_DRIFT && time_difference > r1->uptime * .05 && time_difference > r2->uptime * .05) @@ -5816,4 +5818,3 @@ refresh_all_country_info(void) nodelist_refresh_countries(); } -

1 0

[tor/maint-0.4.1] Merge branch 'ticket31343_040' into maint-0.4.1
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit 0bc5b7ae95a2cc11b380ba09aab0ed1f4219f90b Merge: 40d7f1e3a 7e32db478 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 11:23:59 2019 -0400 Merge branch 'ticket31343_040' into maint-0.4.1 changes/bug31343 | 9 +++++++++ src/core/or/channeltls.c | 20 ++++++++++++++++---- src/feature/nodelist/routerlist.c | 6 ++++-- 3 files changed, 29 insertions(+), 6 deletions(-)

1 0

[tor/maint-0.4.1] Merge branch 'ticket31343_029' into ticket31343_035
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit a4400a77a5db2fc60db258857a23a11ec741e426 Merge: 70d0b97ee 878f44090 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 09:39:48 2019 -0400 Merge branch 'ticket31343_029' into ticket31343_035 src/feature/nodelist/routerlist.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --cc src/feature/nodelist/routerlist.c index 4a99427cd,000000000..456f930aa mode 100644,000000..100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@@ -1,3232 -1,0 +1,3234 @@@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerlist.c + * \brief Code to + * maintain and access the global list of routerinfos for known + * servers. + * + * A "routerinfo_t" object represents a single self-signed router + * descriptor, as generated by a Tor relay in order to tell the rest of + * the world about its keys, address, and capabilities. An + * "extrainfo_t" object represents an adjunct "extra-info" object, + * certified by a corresponding router descriptor, reporting more + * information about the relay that nearly all users will not need. + * + * Most users will not use router descriptors for most relays. Instead, + * they use the information in microdescriptors and in the consensus + * networkstatus. + * + * Right now, routerinfo_t objects are used in these ways: + * <ul> + * <li>By clients, in order to learn about bridge keys and capabilities. + * (Bridges aren't listed in the consensus networkstatus, so they + * can't have microdescriptors.) + * <li>By relays, since relays want more information about other relays + * than they can learn from microdescriptors. (TODO: Is this still true?) + * <li>By authorities, which receive them and use them to generate the + * consensus and the microdescriptors. + * <li>By all directory caches, which download them in case somebody + * else wants them. + * </ul> + * + * Routerinfos are mostly created by parsing them from a string, in + * routerparse.c. We store them to disk on receiving them, and + * periodically discard the ones we don't need. On restarting, we + * re-read them from disk. (This also applies to extrainfo documents, if + * we are configured to fetch them.) + * + * In order to keep our list of routerinfos up-to-date, we periodically + * check whether there are any listed in the latest consensus (or in the + * votes from other authorities, if we are an authority) that we don't + * have. (This also applies to extrainfo documents, if we are + * configured to fetch them.) + * + * Almost nothing in Tor should use a routerinfo_t to refer directly to + * a relay; instead, almost everything should use node_t (implemented in + * nodelist.c), which provides a common interface to routerinfo_t, + * routerstatus_t, and microdescriptor_t. + * + * + * + * This module also has some of the functions used for choosing random + * nodes according to different rules and weights. Historically, they + * were all in this module. Now, they are spread across this module, + * nodelist.c, and networkstatus.c. (TODO: Fix that.) + **/ + +#define ROUTERLIST_PRIVATE +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/or/policies.h" +#include "feature/client/bridges.h" +#include "feature/control/control.h" +#include "feature/dirauth/authmode.h" +#include "feature/dirauth/process_descs.h" +#include "feature/dirauth/reachability.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/dircommon/directory.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/describe.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerinfo.h" +#include "feature/nodelist/routerlist.h" +#include "feature/dirparse/routerparse.h" +#include "feature/nodelist/routerset.h" +#include "feature/nodelist/torcert.h" +#include "feature/relay/routermode.h" +#include "feature/stats/rephist.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include "feature/dircommon/dir_connection_st.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/document_signature_st.h" +#include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/networkstatus_voter_info_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/routerlist_st.h" +#include "feature/nodelist/vote_routerstatus_st.h" + +#include "lib/crypt_ops/digestset.h" + +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif + +// #define DEBUG_ROUTERLIST + +/****************************************************************************/ + +/* Typed wrappers for different digestmap types; used to avoid type + * confusion. */ + +DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t) +DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t) +DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t) +#define SDMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \ + valvar) +#define RIMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar) +#define EIMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar) +#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn)) +#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn)) +#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn)) + +/* static function prototypes */ +static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); +static const char *signed_descriptor_get_body_impl( + const signed_descriptor_t *desc, + int with_annotations); +static void launch_dummy_descriptor_download_as_needed(time_t now, + const or_options_t *options); + +/****************************************************************************/ + +/** Global list of all of the routers that we know about. */ +static routerlist_t *routerlist = NULL; + +/** List of strings for nicknames we've already warned about and that are + * still unknown / unavailable. */ +static smartlist_t *warned_nicknames = NULL; + +/** The last time we tried to download any routerdesc, or 0 for "never". We + * use this to rate-limit download attempts when the number of routerdescs to + * download is low. */ +static time_t last_descriptor_download_attempted = 0; + +/* Router descriptor storage. + * + * Routerdescs are stored in a big file, named "cached-descriptors". As new + * routerdescs arrive, we append them to a journal file named + * "cached-descriptors.new". + * + * From time to time, we replace "cached-descriptors" with a new file + * containing only the live, non-superseded descriptors, and clear + * cached-routers.new. + * + * On startup, we read both files. + */ + +/** Helper: return 1 iff the router log is so big we want to rebuild the + * store. */ +static int +router_should_rebuild_store(desc_store_t *store) +{ + if (store->store_len > (1<<16)) + return (store->journal_len > store->store_len / 2 || + store->bytes_dropped > store->store_len / 2); + else + return store->journal_len > (1<<15); +} + +/** Return the desc_store_t in rl that should be used to store + * sd. */ +static inline desc_store_t * +desc_get_store(routerlist_t *rl, const signed_descriptor_t *sd) +{ + if (sd->is_extrainfo) + return &rl->extrainfo_store; + else + return &rl->desc_store; +} + +/** Add the signed_descriptor_t in desc to the router + * journal; change its saved_location to SAVED_IN_JOURNAL and set its + * offset appropriately. */ +static int +signed_desc_append_to_journal(signed_descriptor_t *desc, + desc_store_t *store) +{ + char *fname = get_cachedir_fname_suffix(store->fname_base, ".new"); + const char *body = signed_descriptor_get_body_impl(desc,1); + size_t len = desc->signed_descriptor_len + desc->annotations_len; + + if (append_bytes_to_file(fname, body, len, 1)) { + log_warn(LD_FS, "Unable to store router descriptor"); + tor_free(fname); + return -1; + } + desc->saved_location = SAVED_IN_JOURNAL; + tor_free(fname); + + desc->saved_offset = store->journal_len; + store->journal_len += len; + + return 0; +} + +/** Sorting helper: return <0, 0, or >0 depending on whether the + * signed_descriptor_t* in *a is older, the same age as, or newer than + * the signed_descriptor_t* in *b. */ +static int +compare_signed_descriptors_by_age_(const void **_a, const void **_b) +{ + const signed_descriptor_t *r1 = *_a, *r2 = *_b; + return (int)(r1->published_on - r2->published_on); +} + +#define RRS_FORCE 1 +#define RRS_DONT_REMOVE_OLD 2 + +/** If the journal of store is too long, or if RRS_FORCE is set in + * flags, then atomically replace the saved router store with the + * routers currently in our routerlist, and clear the journal. Unless + * RRS_DONT_REMOVE_OLD is set in flags, delete expired routers before + * rebuilding the store. Return 0 on success, -1 on failure. + */ +static int +router_rebuild_store(int flags, desc_store_t *store) +{ + smartlist_t *chunk_list = NULL; + char *fname = NULL, *fname_tmp = NULL; + int r = -1; + off_t offset = 0; + smartlist_t *signed_descriptors = NULL; + int nocache=0; + size_t total_expected_len = 0; + int had_any; + int force = flags & RRS_FORCE; + + if (!force && !router_should_rebuild_store(store)) { + r = 0; + goto done; + } + if (!routerlist) { + r = 0; + goto done; + } + + if (store->type == EXTRAINFO_STORE) + had_any = !eimap_isempty(routerlist->extra_info_map); + else + had_any = (smartlist_len(routerlist->routers)+ + smartlist_len(routerlist->old_routers))>0; + + /* Don't save deadweight. */ + if (!(flags & RRS_DONT_REMOVE_OLD)) + routerlist_remove_old_routers(); + + log_info(LD_DIR, "Rebuilding %s cache", store->description); + + fname = get_cachedir_fname(store->fname_base); + fname_tmp = get_cachedir_fname_suffix(store->fname_base, ".tmp"); + + chunk_list = smartlist_new(); + + /* We sort the routers by age to enhance locality on disk. */ + signed_descriptors = smartlist_new(); + if (store->type == EXTRAINFO_STORE) { + eimap_iter_t *iter; + for (iter = eimap_iter_init(routerlist->extra_info_map); + !eimap_iter_done(iter); + iter = eimap_iter_next(routerlist->extra_info_map, iter)) { + const char *key; + extrainfo_t *ei; + eimap_iter_get(iter, &key, &ei); + smartlist_add(signed_descriptors, &ei->cache_info); + } + } else { + SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, + smartlist_add(signed_descriptors, sd)); + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri, + smartlist_add(signed_descriptors, &ri->cache_info)); + } + + smartlist_sort(signed_descriptors, compare_signed_descriptors_by_age_); + + /* Now, add the appropriate members to chunk_list */ + SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) { + sized_chunk_t *c; + const char *body = signed_descriptor_get_body_impl(sd, 1); + if (!body) { + log_warn(LD_BUG, "No descriptor available for router."); + goto done; + } + if (sd->do_not_cache) { + ++nocache; + continue; + } + c = tor_malloc(sizeof(sized_chunk_t)); + c->bytes = body; + c->len = sd->signed_descriptor_len + sd->annotations_len; + total_expected_len += c->len; + smartlist_add(chunk_list, c); + } SMARTLIST_FOREACH_END(sd); + + if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) { + log_warn(LD_FS, "Error writing router store to disk."); + goto done; + } + + /* Our mmap is now invalid. */ + if (store->mmap) { + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Unable to munmap route store in %s", fname); + } + } + + if (replace_file(fname_tmp, fname)<0) { + log_warn(LD_FS, "Error replacing old router store: %s", strerror(errno)); + goto done; + } + + errno = 0; + store->mmap = tor_mmap_file(fname); + if (! store->mmap) { + if (errno == ERANGE) { + /* empty store.*/ + if (total_expected_len) { + log_warn(LD_FS, "We wrote some bytes to a new descriptor file at '%s'," + " but when we went to mmap it, it was empty!", fname); + } else if (had_any) { + log_info(LD_FS, "We just removed every descriptor in '%s'. This is " + "okay if we're just starting up after a long time. " + "Otherwise, it's a bug.", fname); + } + } else { + log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname); + } + } + + log_info(LD_DIR, "Reconstructing pointers into cache"); + + offset = 0; + SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) { + if (sd->do_not_cache) + continue; + sd->saved_location = SAVED_IN_CACHE; + if (store->mmap) { + tor_free(sd->signed_descriptor_body); // sets it to null + sd->saved_offset = offset; + } + offset += sd->signed_descriptor_len + sd->annotations_len; + signed_descriptor_get_body(sd); /* reconstruct and assert */ + } SMARTLIST_FOREACH_END(sd); + + tor_free(fname); + fname = get_cachedir_fname_suffix(store->fname_base, ".new"); + write_str_to_file(fname, "", 1); + + r = 0; + store->store_len = (size_t) offset; + store->journal_len = 0; + store->bytes_dropped = 0; + done: + smartlist_free(signed_descriptors); + tor_free(fname); + tor_free(fname_tmp); + if (chunk_list) { + SMARTLIST_FOREACH(chunk_list, sized_chunk_t *, c, tor_free(c)); + smartlist_free(chunk_list); + } + + return r; +} + +/** Helper: Reload a cache file and its associated journal, setting metadata + * appropriately. If extrainfo is true, reload the extrainfo store; + * else reload the router descriptor store. */ +static int +router_reload_router_list_impl(desc_store_t *store) +{ + char *fname = NULL, *contents = NULL; + struct stat st; + int extrainfo = (store->type == EXTRAINFO_STORE); + store->journal_len = store->store_len = 0; + + fname = get_cachedir_fname(store->fname_base); + + if (store->mmap) { + /* get rid of it first */ + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Failed to munmap %s", fname); + tor_free(fname); + return -1; + } + } + + store->mmap = tor_mmap_file(fname); + if (store->mmap) { + store->store_len = store->mmap->size; + if (extrainfo) + router_load_extrainfo_from_string(store->mmap->data, + store->mmap->data+store->mmap->size, + SAVED_IN_CACHE, NULL, 0); + else + router_load_routers_from_string(store->mmap->data, + store->mmap->data+store->mmap->size, + SAVED_IN_CACHE, NULL, 0, NULL); + } + + tor_free(fname); + fname = get_cachedir_fname_suffix(store->fname_base, ".new"); + /* don't load empty files - we wouldn't get any data, even if we tried */ + if (file_status(fname) == FN_FILE) + contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); + if (contents) { + if (extrainfo) + router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL, + NULL, 0); + else + router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL, + NULL, 0, NULL); + store->journal_len = (size_t) st.st_size; + tor_free(contents); + } + + tor_free(fname); + + if (store->journal_len) { + /* Always clear the journal on startup.*/ + router_rebuild_store(RRS_FORCE, store); + } else if (!extrainfo) { + /* Don't cache expired routers. (This is in an else because + * router_rebuild_store() also calls remove_old_routers().) */ + routerlist_remove_old_routers(); + } + + return 0; +} + +/** Load all cached router descriptors and extra-info documents from the + * store. Return 0 on success and -1 on failure. + */ +int +router_reload_router_list(void) +{ + routerlist_t *rl = router_get_routerlist(); + if (router_reload_router_list_impl(&rl->desc_store)) + return -1; + if (router_reload_router_list_impl(&rl->extrainfo_store)) + return -1; + return 0; +} + +/* When iterating through the routerlist, can OR address/port preference + * and reachability checks be skipped? + */ +int +router_skip_or_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or()); +} + +/* When iterating through the routerlist, can Dir address/port preference + * and reachability checks be skipped? + */ +int +router_skip_dir_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir()); +} + +/** Return true iff r1 and r2 have the same address and OR port. */ +int +routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2) +{ + return r1->addr == r2->addr && r1->or_port == r2->or_port && + tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) && + r1->ipv6_orport == r2->ipv6_orport; +} + +/** Add every suitable node from our nodelist to sl, so that + * we can pick a node for a circuit. + */ +void +router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, + int need_capacity, int need_guard, + int need_desc, int pref_addr, + int direct_conn) +{ + const int check_reach = !router_skip_or_reachability(get_options(), + pref_addr); + /* XXXX MOVE */ + SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { + if (!node->is_running || !node->is_valid) + continue; + if (need_desc && !node_has_preferred_descriptor(node, direct_conn)) + continue; + if (node->ri && node->ri->purpose != ROUTER_PURPOSE_GENERAL) + continue; + if (node_is_unreliable(node, need_uptime, need_capacity, need_guard)) + continue; + /* Don't choose nodes if we are certain they can't do EXTEND2 cells */ + if (node->rs && !routerstatus_version_supports_extend2_cells(node->rs, 1)) + continue; + /* Don't choose nodes if we are certain they can't do ntor. */ + if ((node->ri || node->md) && !node_has_curve25519_onion_key(node)) + continue; + /* Choose a node with an OR address that matches the firewall rules */ + if (direct_conn && check_reach && + !fascist_firewall_allows_node(node, + FIREWALL_OR_CONNECTION, + pref_addr)) + continue; + + smartlist_add(sl, (void *)node); + } SMARTLIST_FOREACH_END(node); +} + +/** Look through the routerlist until we find a router that has my key. + Return it. */ +const routerinfo_t * +routerlist_find_my_routerinfo(void) +{ + if (!routerlist) + return NULL; + + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, router, + { + if (router_is_me(router)) + return router; + }); + return NULL; +} + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity. */ +uint32_t +router_get_advertised_bandwidth(const routerinfo_t *router) +{ + if (router->bandwidthcapacity < router->bandwidthrate) + return router->bandwidthcapacity; + return router->bandwidthrate; +} + +/** Do not weight any declared bandwidth more than this much when picking + * routers by bandwidth. */ +#define DEFAULT_MAX_BELIEVABLE_BANDWIDTH 10000000 /* 10 MB/sec */ + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity, capped by max-believe-bw. */ +uint32_t +router_get_advertised_bandwidth_capped(const routerinfo_t *router) +{ + uint32_t result = router->bandwidthcapacity; + if (result > router->bandwidthrate) + result = router->bandwidthrate; + if (result > DEFAULT_MAX_BELIEVABLE_BANDWIDTH) + result = DEFAULT_MAX_BELIEVABLE_BANDWIDTH; + return result; +} + +/** Helper: given an extended nickname in hexdigest try to decode it. + * Return 0 on success, -1 on failure. Store the result into the + * DIGEST_LEN-byte buffer at digest_out, the single character at + * nickname_qualifier_char_out, and the MAXNICKNAME_LEN+1-byte buffer + * at nickname_out. + * + * The recognized format is: + * HexName = Dollar? HexDigest NamePart? + * Dollar = '?' + * HexDigest = HexChar*20 + * HexChar = 'a'..'f' | 'A'..'F' | '0'..'9' + * NamePart = QualChar Name + * QualChar = '=' | '~' + * Name = NameChar*(1..MAX_NICKNAME_LEN) + * NameChar = Any ASCII alphanumeric character + */ +int +hex_digest_nickname_decode(const char *hexdigest, + char *digest_out, + char *nickname_qualifier_char_out, + char *nickname_out) +{ + size_t len; + + tor_assert(hexdigest); + if (hexdigest[0] == '$') + ++hexdigest; + + len = strlen(hexdigest); + if (len < HEX_DIGEST_LEN) { + return -1; + } else if (len > HEX_DIGEST_LEN && (hexdigest[HEX_DIGEST_LEN] == '=' || + hexdigest[HEX_DIGEST_LEN] == '~') && + len <= HEX_DIGEST_LEN+1+MAX_NICKNAME_LEN) { + *nickname_qualifier_char_out = hexdigest[HEX_DIGEST_LEN]; + strlcpy(nickname_out, hexdigest+HEX_DIGEST_LEN+1 , MAX_NICKNAME_LEN+1); + } else if (len == HEX_DIGEST_LEN) { + ; + } else { + return -1; + } + + if (base16_decode(digest_out, DIGEST_LEN, + hexdigest, HEX_DIGEST_LEN) != DIGEST_LEN) + return -1; + return 0; +} + +/** Helper: Return true iff the identity_digest and nickname + * combination of a router, encoded in hexadecimal, matches hexdigest + * (which is optionally prefixed with a single dollar sign). Return false if + * hexdigest is malformed, or it doesn't match. */ +int +hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest, + const char *nickname) +{ + char digest[DIGEST_LEN]; + char nn_char='\0'; + char nn_buf[MAX_NICKNAME_LEN+1]; + + if (hex_digest_nickname_decode(hexdigest, digest, &nn_char, nn_buf) == -1) + return 0; + + if (nn_char == '=') { + return 0; + } + + if (nn_char == '~') { + if (!nickname) // XXX This seems wrong. -NM + return 0; + if (strcasecmp(nn_buf, nickname)) + return 0; + } + + return tor_memeq(digest, identity_digest, DIGEST_LEN); +} + +/** If hexdigest is correctly formed, base16_decode it into + * digest, which must have DIGEST_LEN space in it. + * Return 0 on success, -1 on failure. + */ +int +hexdigest_to_digest(const char *hexdigest, char *digest) +{ + if (hexdigest[0]=='$') + ++hexdigest; + if (strlen(hexdigest) < HEX_DIGEST_LEN || + base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN) + return -1; + return 0; +} + +/** As router_get_by_id_digest,but return a pointer that you're allowed to + * modify */ +routerinfo_t * +router_get_mutable_by_digest(const char *digest) +{ + tor_assert(digest); + + if (!routerlist) return NULL; + + // routerlist_assert_ok(routerlist); + + return rimap_get(routerlist->identity_map, digest); +} + +/** Return the router in our routerlist whose 20-byte key digest + * is digest. Return NULL if no such router is known. */ +const routerinfo_t * +router_get_by_id_digest(const char *digest) +{ + return router_get_mutable_by_digest(digest); +} + +/** Return the router in our routerlist whose 20-byte descriptor + * is digest. Return NULL if no such router is known. */ +signed_descriptor_t * +router_get_by_descriptor_digest(const char *digest) +{ + tor_assert(digest); + + if (!routerlist) return NULL; + + return sdmap_get(routerlist->desc_digest_map, digest); +} + +/** Return the signed descriptor for the router in our routerlist whose + * 20-byte extra-info digest is digest. Return NULL if no such router + * is known. */ +MOCK_IMPL(signed_descriptor_t *, +router_get_by_extrainfo_digest,(const char *digest)) +{ + tor_assert(digest); + + if (!routerlist) return NULL; + + return sdmap_get(routerlist->desc_by_eid_map, digest); +} + +/** Return the signed descriptor for the extrainfo_t in our routerlist whose + * extra-info-digest is digest. Return NULL if no such extra-info + * document is known. */ +MOCK_IMPL(signed_descriptor_t *, +extrainfo_get_by_descriptor_digest,(const char *digest)) +{ + extrainfo_t *ei; + tor_assert(digest); + if (!routerlist) return NULL; + ei = eimap_get(routerlist->extra_info_map, digest); + return ei ? &ei->cache_info : NULL; +} + +/** Return a pointer to the signed textual representation of a descriptor. + * The returned string is not guaranteed to be NUL-terminated: the string's + * length will be in desc-\>signed_descriptor_len. + * + * If with_annotations is set, the returned string will include + * the annotations + * (if any) preceding the descriptor. This will increase the length of the + * string by desc-\>annotations_len. + * + * The caller must not free the string returned. + */ +static const char * +signed_descriptor_get_body_impl(const signed_descriptor_t *desc, + int with_annotations) +{ + const char *r = NULL; + size_t len = desc->signed_descriptor_len; + off_t offset = desc->saved_offset; + if (with_annotations) + len += desc->annotations_len; + else + offset += desc->annotations_len; + + tor_assert(len > 32); + if (desc->saved_location == SAVED_IN_CACHE && routerlist) { + desc_store_t *store = desc_get_store(router_get_routerlist(), desc); + if (store && store->mmap) { + tor_assert(desc->saved_offset + len <= store->mmap->size); + r = store->mmap->data + offset; + } else if (store) { + log_err(LD_DIR, "We couldn't read a descriptor that is supposedly " + "mmaped in our cache. Is another process running in our data " + "directory? Exiting."); + exit(1); // XXXX bad exit: should recover. + } + } + if (!r) /* no mmap, or not in cache. */ + r = desc->signed_descriptor_body + + (with_annotations ? 0 : desc->annotations_len); + + tor_assert(r); + if (!with_annotations) { + if (fast_memcmp("router ", r, 7) && fast_memcmp("extra-info ", r, 11)) { + char *cp = tor_strndup(r, 64); + log_err(LD_DIR, "descriptor at %p begins with unexpected string %s. " + "Is another process running in our data directory? Exiting.", + desc, escaped(cp)); + exit(1); // XXXX bad exit: should recover. + } + } + + return r; +} + +/** Return a pointer to the signed textual representation of a descriptor. + * The returned string is not guaranteed to be NUL-terminated: the string's + * length will be in desc-\>signed_descriptor_len. + * + * The caller must not free the string returned. + */ +const char * +signed_descriptor_get_body(const signed_descriptor_t *desc) +{ + return signed_descriptor_get_body_impl(desc, 0); +} + +/** As signed_descriptor_get_body(), but points to the beginning of the + * annotations section rather than the beginning of the descriptor. */ +const char * +signed_descriptor_get_annotations(const signed_descriptor_t *desc) +{ + return signed_descriptor_get_body_impl(desc, 1); +} + +/** Return the current list of all known routers. */ +routerlist_t * +router_get_routerlist(void) +{ + if (PREDICT_UNLIKELY(!routerlist)) { + routerlist = tor_malloc_zero(sizeof(routerlist_t)); + routerlist->routers = smartlist_new(); + routerlist->old_routers = smartlist_new(); + routerlist->identity_map = rimap_new(); + routerlist->desc_digest_map = sdmap_new(); + routerlist->desc_by_eid_map = sdmap_new(); + routerlist->extra_info_map = eimap_new(); + + routerlist->desc_store.fname_base = "cached-descriptors"; + routerlist->extrainfo_store.fname_base = "cached-extrainfo"; + + routerlist->desc_store.type = ROUTER_STORE; + routerlist->extrainfo_store.type = EXTRAINFO_STORE; + + routerlist->desc_store.description = "router descriptors"; + routerlist->extrainfo_store.description = "extra-info documents"; + } + return routerlist; +} + +/** Free all storage held by router. */ +void +routerinfo_free_(routerinfo_t *router) +{ + if (!router) + return; + + tor_free(router->cache_info.signed_descriptor_body); + tor_free(router->nickname); + tor_free(router->platform); + tor_free(router->protocol_list); + tor_free(router->contact_info); + if (router->onion_pkey) + tor_free(router->onion_pkey); + tor_free(router->onion_curve25519_pkey); + if (router->identity_pkey) + crypto_pk_free(router->identity_pkey); + tor_cert_free(router->cache_info.signing_key_cert); + if (router->declared_family) { + SMARTLIST_FOREACH(router->declared_family, char *, s, tor_free(s)); + smartlist_free(router->declared_family); + } + addr_policy_list_free(router->exit_policy); + short_policy_free(router->ipv6_exit_policy); + + memset(router, 77, sizeof(routerinfo_t)); + + tor_free(router); +} + +/** Release all storage held by extrainfo */ +void +extrainfo_free_(extrainfo_t *extrainfo) +{ + if (!extrainfo) + return; + tor_cert_free(extrainfo->cache_info.signing_key_cert); + tor_free(extrainfo->cache_info.signed_descriptor_body); + tor_free(extrainfo->pending_sig); + + memset(extrainfo, 88, sizeof(extrainfo_t)); /* debug bad memory usage */ + tor_free(extrainfo); +} + +#define signed_descriptor_free(val) \ + FREE_AND_NULL(signed_descriptor_t, signed_descriptor_free_, (val)) + +/** Release storage held by sd. */ +static void +signed_descriptor_free_(signed_descriptor_t *sd) +{ + if (!sd) + return; + + tor_free(sd->signed_descriptor_body); + tor_cert_free(sd->signing_key_cert); + + memset(sd, 99, sizeof(signed_descriptor_t)); /* Debug bad mem usage */ + tor_free(sd); +} + +/** Reset the given signed descriptor sd by freeing the allocated + * memory inside the object and by zeroing its content. */ +static void +signed_descriptor_reset(signed_descriptor_t *sd) +{ + tor_assert(sd); + tor_free(sd->signed_descriptor_body); + tor_cert_free(sd->signing_key_cert); + memset(sd, 0, sizeof(*sd)); +} + +/** Copy src into dest, and steal all references inside src so that when + * we free src, we don't mess up dest. */ +static void +signed_descriptor_move(signed_descriptor_t *dest, + signed_descriptor_t *src) +{ + tor_assert(dest != src); + /* Cleanup destination object before overwriting it.*/ + signed_descriptor_reset(dest); + memcpy(dest, src, sizeof(signed_descriptor_t)); + src->signed_descriptor_body = NULL; + src->signing_key_cert = NULL; + dest->routerlist_index = -1; +} + +/** Extract a signed_descriptor_t from a general routerinfo, and free the + * routerinfo. + */ +static signed_descriptor_t * +signed_descriptor_from_routerinfo(routerinfo_t *ri) +{ + signed_descriptor_t *sd; + tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL); + sd = tor_malloc_zero(sizeof(signed_descriptor_t)); + signed_descriptor_move(sd, &ri->cache_info); + routerinfo_free(ri); + return sd; +} + +/** Helper: free the storage held by the extrainfo_t in e. */ +static void +extrainfo_free_void(void *e) +{ + extrainfo_free_(e); +} + +/** Free all storage held by a routerlist rl. */ +void +routerlist_free_(routerlist_t *rl) +{ + if (!rl) + return; + rimap_free(rl->identity_map, NULL); + sdmap_free(rl->desc_digest_map, NULL); + sdmap_free(rl->desc_by_eid_map, NULL); + eimap_free(rl->extra_info_map, extrainfo_free_void); + SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, + routerinfo_free(r)); + SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd, + signed_descriptor_free(sd)); + smartlist_free(rl->routers); + smartlist_free(rl->old_routers); + if (rl->desc_store.mmap) { + int res = tor_munmap_file(routerlist->desc_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap"); + } + } + if (rl->extrainfo_store.mmap) { + int res = tor_munmap_file(routerlist->extrainfo_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap"); + } + } + tor_free(rl); + + router_dir_info_changed(); +} + +/** Log information about how much memory is being used for routerlist, + * at log level severity. */ +void +dump_routerlist_mem_usage(int severity) +{ + uint64_t livedescs = 0; + uint64_t olddescs = 0; + if (!routerlist) + return; + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, r, + livedescs += r->cache_info.signed_descriptor_len); + SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, + olddescs += sd->signed_descriptor_len); + + tor_log(severity, LD_DIR, + "In %d live descriptors: %"PRIu64" bytes. " + "In %d old descriptors: %"PRIu64" bytes.", + smartlist_len(routerlist->routers), (livedescs), + smartlist_len(routerlist->old_routers), (olddescs)); +} + +/** Debugging helper: If idx is nonnegative, assert that ri is + * in sl at position idx. Otherwise, search sl for + * ri. Return the index of ri in sl, or -1 if ri + * is not in sl. */ +static inline int +routerlist_find_elt_(smartlist_t *sl, void *ri, int idx) +{ + if (idx < 0) { + idx = -1; + SMARTLIST_FOREACH(sl, routerinfo_t *, r, + if (r == ri) { + idx = r_sl_idx; + break; + }); + } else { + tor_assert(idx < smartlist_len(sl)); + tor_assert(smartlist_get(sl, idx) == ri); + }; + return idx; +} + +/** Insert an item ri into the routerlist rl, updating indices + * as needed. There must be no previous member of rl with the same + * identity digest as ri: If there is, call routerlist_replace + * instead. + */ +static void +routerlist_insert(routerlist_t *rl, routerinfo_t *ri) +{ + routerinfo_t *ri_old; + signed_descriptor_t *sd_old; + { + const routerinfo_t *ri_generated = router_get_my_routerinfo(); + tor_assert(ri_generated != ri); + } + tor_assert(ri->cache_info.routerlist_index == -1); + + ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri); + tor_assert(!ri_old); + + sd_old = sdmap_set(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest, + &(ri->cache_info)); + if (sd_old) { + int idx = sd_old->routerlist_index; + sd_old->routerlist_index = -1; + smartlist_del(rl->old_routers, idx); + if (idx < smartlist_len(rl->old_routers)) { + signed_descriptor_t *d = smartlist_get(rl->old_routers, idx); + d->routerlist_index = idx; + } + rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len; + sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest); + signed_descriptor_free(sd_old); + } + + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest, + &ri->cache_info); + smartlist_add(rl->routers, ri); + ri->cache_info.routerlist_index = smartlist_len(rl->routers) - 1; + nodelist_set_routerinfo(ri, NULL); + router_dir_info_changed(); +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Adds the extrainfo_t ei to the routerlist rl, if there is a + * corresponding router in rl-\>routers or rl-\>old_routers. Return the status + * of inserting ei. Free ei if it isn't inserted. */ +MOCK_IMPL(STATIC was_router_added_t, +extrainfo_insert,(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)) +{ + was_router_added_t r; + const char *compatibility_error_msg; + routerinfo_t *ri = rimap_get(rl->identity_map, + ei->cache_info.identity_digest); + signed_descriptor_t *sd = + sdmap_get(rl->desc_by_eid_map, ei->cache_info.signed_descriptor_digest); + extrainfo_t *ei_tmp; + const int severity = warn_if_incompatible ? LOG_WARN : LOG_INFO; + + { + extrainfo_t *ei_generated = router_get_my_extrainfo(); + tor_assert(ei_generated != ei); + } + + if (!ri) { + /* This router is unknown; we can't even verify the signature. Give up.*/ + r = ROUTER_NOT_IN_CONSENSUS; + goto done; + } + if (! sd) { + /* The extrainfo router doesn't have a known routerdesc to attach it to. + * This just won't work. */; + static ratelim_t no_sd_ratelim = RATELIM_INIT(1800); + r = ROUTER_BAD_EI; + log_fn_ratelim(&no_sd_ratelim, severity, LD_BUG, + "No entry found in extrainfo map."); + goto done; + } + if (tor_memneq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN)) { + static ratelim_t digest_mismatch_ratelim = RATELIM_INIT(1800); + /* The sd we got from the map doesn't match the digest we used to look + * it up. This makes no sense. */ + r = ROUTER_BAD_EI; + log_fn_ratelim(&digest_mismatch_ratelim, severity, LD_BUG, + "Mismatch in digest in extrainfo map."); + goto done; + } + if (routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, sd, + &compatibility_error_msg)) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + r = (ri->cache_info.extrainfo_is_bogus) ? + ROUTER_BAD_EI : ROUTER_NOT_IN_CONSENSUS; + + base16_encode(d1, sizeof(d1), ri->cache_info.identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), ei->cache_info.identity_digest, DIGEST_LEN); + + log_fn(severity,LD_DIR, + "router info incompatible with extra info (ri id: %s, ei id %s, " + "reason: %s)", d1, d2, compatibility_error_msg); + + goto done; + } + + /* Okay, if we make it here, we definitely have a router corresponding to + * this extrainfo. */ + + ei_tmp = eimap_set(rl->extra_info_map, + ei->cache_info.signed_descriptor_digest, + ei); + r = ROUTER_ADDED_SUCCESSFULLY; + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + + done: + if (r != ROUTER_ADDED_SUCCESSFULLY) + extrainfo_free(ei); + +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif + return r; +} + +#define should_cache_old_descriptors() \ + directory_caches_dir_info(get_options()) + +/** If we're a directory cache and routerlist rl doesn't have + * a copy of router ri yet, add it to the list of old (not + * recommended but still served) descriptors. Else free it. */ +static void +routerlist_insert_old(routerlist_t *rl, routerinfo_t *ri) +{ + { + const routerinfo_t *ri_generated = router_get_my_routerinfo(); + tor_assert(ri_generated != ri); + } + tor_assert(ri->cache_info.routerlist_index == -1); + + if (should_cache_old_descriptors() && + ri->purpose == ROUTER_PURPOSE_GENERAL && + !sdmap_get(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest)) { + signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri); + sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd); + smartlist_add(rl->old_routers, sd); + sd->routerlist_index = smartlist_len(rl->old_routers)-1; + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); + } else { + routerinfo_free(ri); + } +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Remove an item ri from the routerlist rl, updating indices + * as needed. If idx is nonnegative and smartlist_get(rl->routers, + * idx) == ri, we don't need to do a linear search over the list to decide + * which to remove. We fill the gap in rl->routers with a later element in + * the list, if any exists. ri is freed. + * + * If make_old is true, instead of deleting the router, we try adding + * it to rl->old_routers. */ +void +routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, time_t now) +{ + routerinfo_t *ri_tmp; + extrainfo_t *ei_tmp; + int idx = ri->cache_info.routerlist_index; + tor_assert(0 <= idx && idx < smartlist_len(rl->routers)); + tor_assert(smartlist_get(rl->routers, idx) == ri); + + nodelist_remove_routerinfo(ri); + + /* make sure the rephist module knows that it's not running */ + rep_hist_note_router_unreachable(ri->cache_info.identity_digest, now); + + ri->cache_info.routerlist_index = -1; + smartlist_del(rl->routers, idx); + if (idx < smartlist_len(rl->routers)) { + routerinfo_t *r = smartlist_get(rl->routers, idx); + r->cache_info.routerlist_index = idx; + } + + ri_tmp = rimap_remove(rl->identity_map, ri->cache_info.identity_digest); + router_dir_info_changed(); + tor_assert(ri_tmp == ri); + + if (make_old && should_cache_old_descriptors() && + ri->purpose == ROUTER_PURPOSE_GENERAL) { + signed_descriptor_t *sd; + sd = signed_descriptor_from_routerinfo(ri); + smartlist_add(rl->old_routers, sd); + sd->routerlist_index = smartlist_len(rl->old_routers)-1; + sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd); + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); + } else { + signed_descriptor_t *sd_tmp; + sd_tmp = sdmap_remove(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest); + tor_assert(sd_tmp == &(ri->cache_info)); + rl->desc_store.bytes_dropped += ri->cache_info.signed_descriptor_len; + ei_tmp = eimap_remove(rl->extra_info_map, + ri->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) + sdmap_remove(rl->desc_by_eid_map, ri->cache_info.extra_info_digest); + routerinfo_free(ri); + } +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Remove a signed_descriptor_t sd from rl-\>old_routers, and + * adjust rl as appropriate. idx is -1, or the index of + * sd. */ +static void +routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx) +{ + signed_descriptor_t *sd_tmp; + extrainfo_t *ei_tmp; + desc_store_t *store; + if (idx == -1) { + idx = sd->routerlist_index; + } + tor_assert(0 <= idx && idx < smartlist_len(rl->old_routers)); + /* XXXX edmanm's bridge relay triggered the following assert while + * running 0.2.0.12-alpha. If anybody triggers this again, see if we + * can get a backtrace. */ + tor_assert(smartlist_get(rl->old_routers, idx) == sd); + tor_assert(idx == sd->routerlist_index); + + sd->routerlist_index = -1; + smartlist_del(rl->old_routers, idx); + if (idx < smartlist_len(rl->old_routers)) { + signed_descriptor_t *d = smartlist_get(rl->old_routers, idx); + d->routerlist_index = idx; + } + sd_tmp = sdmap_remove(rl->desc_digest_map, + sd->signed_descriptor_digest); + tor_assert(sd_tmp == sd); + store = desc_get_store(rl, sd); + if (store) + store->bytes_dropped += sd->signed_descriptor_len; + + ei_tmp = eimap_remove(rl->extra_info_map, + sd->extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_remove(rl->desc_by_eid_map, sd->extra_info_digest); + + signed_descriptor_free(sd); +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Remove ri_old from the routerlist rl, and replace it with + * ri_new, updating all index info. If idx is nonnegative and + * smartlist_get(rl->routers, idx) == ri, we don't need to do a linear + * search over the list to decide which to remove. We put ri_new in the same + * index as ri_old, if possible. ri is freed as appropriate. + * + * If should_cache_descriptors() is true, instead of deleting the router, + * we add it to rl->old_routers. */ +static void +routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, + routerinfo_t *ri_new) +{ + int idx; + int same_descriptors; + + routerinfo_t *ri_tmp; + extrainfo_t *ei_tmp; + { + const routerinfo_t *ri_generated = router_get_my_routerinfo(); + tor_assert(ri_generated != ri_new); + } + tor_assert(ri_old != ri_new); + tor_assert(ri_new->cache_info.routerlist_index == -1); + + idx = ri_old->cache_info.routerlist_index; + tor_assert(0 <= idx && idx < smartlist_len(rl->routers)); + tor_assert(smartlist_get(rl->routers, idx) == ri_old); + + { + routerinfo_t *ri_old_tmp=NULL; + nodelist_set_routerinfo(ri_new, &ri_old_tmp); + tor_assert(ri_old == ri_old_tmp); + } + + router_dir_info_changed(); + if (idx >= 0) { + smartlist_set(rl->routers, idx, ri_new); + ri_old->cache_info.routerlist_index = -1; + ri_new->cache_info.routerlist_index = idx; + /* Check that ri_old is not in rl->routers anymore: */ + tor_assert( routerlist_find_elt_(rl->routers, ri_old, -1) == -1 ); + } else { + log_warn(LD_BUG, "Appending entry from routerlist_replace."); + routerlist_insert(rl, ri_new); + return; + } + if (tor_memneq(ri_old->cache_info.identity_digest, + ri_new->cache_info.identity_digest, DIGEST_LEN)) { + /* digests don't match; digestmap_set won't replace */ + rimap_remove(rl->identity_map, ri_old->cache_info.identity_digest); + } + ri_tmp = rimap_set(rl->identity_map, + ri_new->cache_info.identity_digest, ri_new); + tor_assert(!ri_tmp || ri_tmp == ri_old); + sdmap_set(rl->desc_digest_map, + ri_new->cache_info.signed_descriptor_digest, + &(ri_new->cache_info)); + + if (!tor_digest_is_zero(ri_new->cache_info.extra_info_digest)) { + sdmap_set(rl->desc_by_eid_map, ri_new->cache_info.extra_info_digest, + &ri_new->cache_info); + } + + same_descriptors = tor_memeq(ri_old->cache_info.signed_descriptor_digest, + ri_new->cache_info.signed_descriptor_digest, + DIGEST_LEN); + + if (should_cache_old_descriptors() && + ri_old->purpose == ROUTER_PURPOSE_GENERAL && + !same_descriptors) { + /* ri_old is going to become a signed_descriptor_t and go into + * old_routers */ + signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old); + smartlist_add(rl->old_routers, sd); + sd->routerlist_index = smartlist_len(rl->old_routers)-1; + sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd); + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); + } else { + /* We're dropping ri_old. */ + if (!same_descriptors) { + /* digests don't match; The sdmap_set above didn't replace */ + sdmap_remove(rl->desc_digest_map, + ri_old->cache_info.signed_descriptor_digest); + + if (tor_memneq(ri_old->cache_info.extra_info_digest, + ri_new->cache_info.extra_info_digest, DIGEST_LEN)) { + ei_tmp = eimap_remove(rl->extra_info_map, + ri_old->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + } + + if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { + sdmap_remove(rl->desc_by_eid_map, + ri_old->cache_info.extra_info_digest); + } + } + rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len; + routerinfo_free(ri_old); + } +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Extract the descriptor sd from old_routerlist, and re-parse + * it as a fresh routerinfo_t. */ +static routerinfo_t * +routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) +{ + routerinfo_t *ri; + const char *body; + + body = signed_descriptor_get_annotations(sd); + + ri = router_parse_entry_from_string(body, + body+sd->signed_descriptor_len+sd->annotations_len, + 0, 1, NULL, NULL); + if (!ri) + return NULL; + signed_descriptor_move(&ri->cache_info, sd); + + routerlist_remove_old(rl, sd, -1); + + return ri; +} + +/** Free all memory held by the routerlist module. + * Note: Calling routerlist_free_all() should always be paired with + * a call to nodelist_free_all(). These should only be called during + * cleanup. + */ +void +routerlist_free_all(void) +{ + routerlist_free(routerlist); + routerlist = NULL; + dirlist_free_all(); + if (warned_nicknames) { + SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); + smartlist_free(warned_nicknames); + warned_nicknames = NULL; + } + authcert_free_all(); +} + +/** Forget that we have issued any router-related warnings, so that we'll + * warn again if we see the same errors. */ +void +routerlist_reset_warnings(void) +{ + if (!warned_nicknames) + warned_nicknames = smartlist_new(); + SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); + smartlist_clear(warned_nicknames); /* now the list is empty. */ + + networkstatus_reset_warnings(); +} + +/** Return 1 if the signed descriptor of this router is older than + * seconds seconds. Otherwise return 0. */ +MOCK_IMPL(int, +router_descriptor_is_older_than,(const routerinfo_t *router, int seconds)) +{ + return router->cache_info.published_on < approx_time() - seconds; +} + +/** Add router to the routerlist, if we don't already have it. Replace + * older entries (if any) with the same key. Note: Callers should not hold + * their pointers to router if this function fails; router + * will either be inserted into the routerlist or freed. Similarly, even + * if this call succeeds, they should not hold their pointers to + * router after subsequent calls with other routerinfo's -- they + * might cause the original routerinfo to get freed. + * + * Returns the status for the operation. Might set *msg if it wants + * the poster of the router to know something. + * + * If from_cache, this descriptor came from our disk cache. If + * from_fetch, we received it in response to a request we made. + * (If both are false, that means it was uploaded to us as an auth dir + * server or via the controller.) + * + * This function should be called *after* + * routers_update_status_from_consensus_networkstatus; subsequently, you + * should call router_rebuild_store and routerlist_descriptors_added. + */ +was_router_added_t +router_add_to_routerlist(routerinfo_t *router, const char **msg, + int from_cache, int from_fetch) +{ + const char *id_digest; + const or_options_t *options = get_options(); + int authdir = authdir_mode_handles_descs(options, router->purpose); + int authdir_believes_valid = 0; + routerinfo_t *old_router; + networkstatus_t *consensus = + networkstatus_get_latest_consensus_by_flavor(FLAV_NS); + int in_consensus = 0; + + tor_assert(msg); + + if (!routerlist) + router_get_routerlist(); + + id_digest = router->cache_info.identity_digest; + + old_router = router_get_mutable_by_digest(id_digest); + + /* Make sure that it isn't expired. */ + if (router->cert_expiration_time < approx_time()) { + routerinfo_free(router); + *msg = "Some certs on this router are expired."; + return ROUTER_CERTS_EXPIRED; + } + + /* Make sure that we haven't already got this exact descriptor. */ + if (sdmap_get(routerlist->desc_digest_map, + router->cache_info.signed_descriptor_digest)) { + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + + /* Only members of routerlist->identity_map can be bridges; we don't + * put bridges in old_routers. */ + const int was_bridge = old_router && + old_router->purpose == ROUTER_PURPOSE_BRIDGE; + + if (routerinfo_is_a_configured_bridge(router) && + router->purpose == ROUTER_PURPOSE_BRIDGE && + !was_bridge) { + log_info(LD_DIR, "Replacing non-bridge descriptor with bridge " + "descriptor for router %s", + router_describe(router)); + } else { + log_info(LD_DIR, + "Dropping descriptor that we already have for router %s", + router_describe(router)); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_IS_ALREADY_KNOWN; + } + } + + if (authdir) { + if (authdir_wants_to_reject_router(router, msg, + !from_cache && !from_fetch, + &authdir_believes_valid)) { + tor_assert(*msg); + routerinfo_free(router); + return ROUTER_AUTHDIR_REJECTS; + } + } else if (from_fetch) { + /* Only check the descriptor digest against the network statuses when + * we are receiving in response to a fetch. */ + + if (!signed_desc_digest_is_recognized(&router->cache_info) && + !routerinfo_is_a_configured_bridge(router)) { + /* We asked for it, so some networkstatus must have listed it when we + * did. Save it if we're a cache in case somebody else asks for it. */ + log_info(LD_DIR, + "Received a no-longer-recognized descriptor for router %s", + router_describe(router)); + *msg = "Router descriptor is not referenced by any network-status."; + + /* Only journal this desc if we want to keep old descriptors */ + if (!from_cache && should_cache_old_descriptors()) + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + routerlist_insert_old(routerlist, router); + return ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS; + } + } + + /* We no longer need a router with this descriptor digest. */ + if (consensus) { + routerstatus_t *rs = networkstatus_vote_find_mutable_entry( + consensus, id_digest); + if (rs && tor_memeq(rs->descriptor_digest, + router->cache_info.signed_descriptor_digest, + DIGEST_LEN)) { + in_consensus = 1; + } + } + + if (router->purpose == ROUTER_PURPOSE_GENERAL && + consensus && !in_consensus && !authdir) { + /* If it's a general router not listed in the consensus, then don't + * consider replacing the latest router with it. */ + if (!from_cache && should_cache_old_descriptors()) + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + routerlist_insert_old(routerlist, router); + *msg = "Skipping router descriptor: not in consensus."; + return ROUTER_NOT_IN_CONSENSUS; + } + + /* If we're reading a bridge descriptor from our cache, and we don't + * recognize it as one of our currently configured bridges, drop the + * descriptor. Otherwise we could end up using it as one of our entry + * guards even if it isn't in our Bridge config lines. */ + if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache && + !authdir_mode_bridge(options) && + !routerinfo_is_a_configured_bridge(router)) { + log_info(LD_DIR, "Dropping bridge descriptor for %s because we have " + "no bridge configured at that address.", + safe_str_client(router_describe(router))); + *msg = "Router descriptor was not a configured bridge."; + routerinfo_free(router); + return ROUTER_WAS_NOT_WANTED; + } + + /* If we have a router with the same identity key, choose the newer one. */ + if (old_router) { + if (!in_consensus && (router->cache_info.published_on <= + old_router->cache_info.published_on)) { + /* Same key, but old. This one is not listed in the consensus. */ + log_debug(LD_DIR, "Not-new descriptor for router %s", + router_describe(router)); + /* Only journal this desc if we'll be serving it. */ + if (!from_cache && should_cache_old_descriptors()) + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + routerlist_insert_old(routerlist, router); + *msg = "Router descriptor was not new."; + return ROUTER_IS_ALREADY_KNOWN; + } else { + /* Same key, and either new, or listed in the consensus. */ + log_debug(LD_DIR, "Replacing entry for router %s", + router_describe(router)); + routerlist_replace(routerlist, old_router, router); + if (!from_cache) { + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + } + *msg = authdir_believes_valid ? "Valid server updated" : + ("Invalid server updated. (This dirserver is marking your " + "server as unapproved.)"); + return ROUTER_ADDED_SUCCESSFULLY; + } + } + + if (!in_consensus && from_cache && + router_descriptor_is_older_than(router, OLD_ROUTER_DESC_MAX_AGE)) { + *msg = "Router descriptor was really old."; + routerinfo_free(router); + return ROUTER_WAS_TOO_OLD; + } + + /* We haven't seen a router with this identity before. Add it to the end of + * the list. */ + routerlist_insert(routerlist, router); + if (!from_cache) { + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + } + return ROUTER_ADDED_SUCCESSFULLY; +} + +/** Insert ei into the routerlist, or free it. Other arguments are + * as for router_add_to_routerlist(). Return ROUTER_ADDED_SUCCESSFULLY iff + * we actually inserted it, ROUTER_BAD_EI otherwise. + */ +was_router_added_t +router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch) +{ + was_router_added_t inserted; + (void)from_fetch; + if (msg) *msg = NULL; + /*XXXX Do something with msg */ + + inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache); + + if (WRA_WAS_ADDED(inserted) && !from_cache) + signed_desc_append_to_journal(&ei->cache_info, + &routerlist->extrainfo_store); + + return inserted; +} + +/** Sorting helper: return <0, 0, or >0 depending on whether the + * signed_descriptor_t* in *a has an identity digest preceding, equal + * to, or later than that of *b. */ +static int +compare_old_routers_by_identity_(const void **_a, const void **_b) +{ + int i; + const signed_descriptor_t *r1 = *_a, *r2 = *_b; + if ((i = fast_memcmp(r1->identity_digest, r2->identity_digest, DIGEST_LEN))) + return i; + return (int)(r1->published_on - r2->published_on); +} + +/** Internal type used to represent how long an old descriptor was valid, + * where it appeared in the list of old descriptors, and whether it's extra + * old. Used only by routerlist_remove_old_cached_routers_with_id(). */ +struct duration_idx_t { + int duration; + int idx; + int old; +}; + +/** Sorting helper: compare two duration_idx_t by their duration. */ +static int +compare_duration_idx_(const void *_d1, const void *_d2) +{ + const struct duration_idx_t *d1 = _d1; + const struct duration_idx_t *d2 = _d2; + return d1->duration - d2->duration; +} + +/** The range lo through hi inclusive of routerlist->old_routers + * must contain routerinfo_t with the same identity and with publication time + * in ascending order. Remove members from this range until there are no more + * than max_descriptors_per_router() remaining. Start by removing the oldest + * members from before cutoff, then remove members which were current + * for the lowest amount of time. The order of members of old_routers at + * indices lo or higher may be changed. + */ +static void +routerlist_remove_old_cached_routers_with_id(time_t now, + time_t cutoff, int lo, int hi, + digestset_t *retain) +{ + int i, n = hi-lo+1; + unsigned n_extra, n_rmv = 0; + struct duration_idx_t *lifespans; + uint8_t *rmv, *must_keep; + smartlist_t *lst = routerlist->old_routers; +#if 1 + const char *ident; + tor_assert(hi < smartlist_len(lst)); + tor_assert(lo <= hi); + ident = ((signed_descriptor_t*)smartlist_get(lst, lo))->identity_digest; + for (i = lo+1; i <= hi; ++i) { + signed_descriptor_t *r = smartlist_get(lst, i); + tor_assert(tor_memeq(ident, r->identity_digest, DIGEST_LEN)); + } +#endif /* 1 */ + /* Check whether we need to do anything at all. */ + { + int mdpr = directory_caches_dir_info(get_options()) ? 2 : 1; + if (n <= mdpr) + return; + n_extra = n - mdpr; + } + + lifespans = tor_calloc(n, sizeof(struct duration_idx_t)); + rmv = tor_calloc(n, sizeof(uint8_t)); + must_keep = tor_calloc(n, sizeof(uint8_t)); + /* Set lifespans to contain the lifespan and index of each server. */ + /* Set rmv[i-lo]=1 if we're going to remove a server for being too old. */ + for (i = lo; i <= hi; ++i) { + signed_descriptor_t *r = smartlist_get(lst, i); + signed_descriptor_t *r_next; + lifespans[i-lo].idx = i; + if (r->last_listed_as_valid_until >= now || + (retain && digestset_probably_contains(retain, + r->signed_descriptor_digest))) { + must_keep[i-lo] = 1; + } + if (i < hi) { + r_next = smartlist_get(lst, i+1); + tor_assert(r->published_on <= r_next->published_on); + lifespans[i-lo].duration = (int)(r_next->published_on - r->published_on); + } else { + r_next = NULL; + lifespans[i-lo].duration = INT_MAX; + } + if (!must_keep[i-lo] && r->published_on < cutoff && n_rmv < n_extra) { + ++n_rmv; + lifespans[i-lo].old = 1; + rmv[i-lo] = 1; + } + } + + if (n_rmv < n_extra) { + /** + * We aren't removing enough servers for being old. Sort lifespans by + * the duration of liveness, and remove the ones we're not already going to + * remove based on how long they were alive. + **/ + qsort(lifespans, n, sizeof(struct duration_idx_t), compare_duration_idx_); + for (i = 0; i < n && n_rmv < n_extra; ++i) { + if (!must_keep[lifespans[i].idx-lo] && !lifespans[i].old) { + rmv[lifespans[i].idx-lo] = 1; + ++n_rmv; + } + } + } + + i = hi; + do { + if (rmv[i-lo]) + routerlist_remove_old(routerlist, smartlist_get(lst, i), i); + } while (--i >= lo); + tor_free(must_keep); + tor_free(rmv); + tor_free(lifespans); +} + +/** Deactivate any routers from the routerlist that are more than + * ROUTER_MAX_AGE seconds old and not recommended by any networkstatuses; + * remove old routers from the list of cached routers if we have too many. + */ +void +routerlist_remove_old_routers(void) +{ + int i, hi=-1; + const char *cur_id = NULL; + time_t now = time(NULL); + time_t cutoff; + routerinfo_t *router; + signed_descriptor_t *sd; + digestset_t *retain; + const networkstatus_t *consensus = networkstatus_get_latest_consensus(); + + trusted_dirs_remove_old_certs(); + + if (!routerlist || !consensus) + return; + + // routerlist_assert_ok(routerlist); + + /* We need to guess how many router descriptors we will wind up wanting to + retain, so that we can be sure to allocate a large enough Bloom filter + to hold the digest set. Overestimating is fine; underestimating is bad. + */ + { + /* We'll probably retain everything in the consensus. */ + int n_max_retain = smartlist_len(consensus->routerstatus_list); + retain = digestset_new(n_max_retain); + } + + cutoff = now - OLD_ROUTER_DESC_MAX_AGE; + /* Retain anything listed in the consensus. */ + if (consensus) { + SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, + if (rs->published_on >= cutoff) + digestset_add(retain, rs->descriptor_digest)); + } + + /* If we have a consensus, we should consider pruning current routers that + * are too old and that nobody recommends. (If we don't have a consensus, + * then we should get one before we decide to kill routers.) */ + + if (consensus) { + cutoff = now - ROUTER_MAX_AGE; + /* Remove too-old unrecommended members of routerlist->routers. */ + for (i = 0; i < smartlist_len(routerlist->routers); ++i) { + router = smartlist_get(routerlist->routers, i); + if (router->cache_info.published_on <= cutoff && + router->cache_info.last_listed_as_valid_until < now && + !digestset_probably_contains(retain, + router->cache_info.signed_descriptor_digest)) { + /* Too old: remove it. (If we're a cache, just move it into + * old_routers.) */ + log_info(LD_DIR, + "Forgetting obsolete (too old) routerinfo for router %s", + router_describe(router)); + routerlist_remove(routerlist, router, 1, now); + i--; + } + } + } + + //routerlist_assert_ok(routerlist); + + /* Remove far-too-old members of routerlist->old_routers. */ + cutoff = now - OLD_ROUTER_DESC_MAX_AGE; + for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) { + sd = smartlist_get(routerlist->old_routers, i); + if (sd->published_on <= cutoff && + sd->last_listed_as_valid_until < now && + !digestset_probably_contains(retain, sd->signed_descriptor_digest)) { + /* Too old. Remove it. */ + routerlist_remove_old(routerlist, sd, i--); + } + } + + //routerlist_assert_ok(routerlist); + + log_info(LD_DIR, "We have %d live routers and %d old router descriptors.", + smartlist_len(routerlist->routers), + smartlist_len(routerlist->old_routers)); + + /* Now we might have to look at routerlist->old_routers for extraneous + * members. (We'd keep all the members if we could, but we need to save + * space.) First, check whether we have too many router descriptors, total. + * We're okay with having too many for some given router, so long as the + * total number doesn't approach max_descriptors_per_router()*len(router). + */ + if (smartlist_len(routerlist->old_routers) < + smartlist_len(routerlist->routers)) + goto done; + + /* Sort by identity, then fix indices. */ + smartlist_sort(routerlist->old_routers, compare_old_routers_by_identity_); + /* Fix indices. */ + for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) { + signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i); + r->routerlist_index = i; + } + + /* Iterate through the list from back to front, so when we remove descriptors + * we don't mess up groups we haven't gotten to. */ + for (i = smartlist_len(routerlist->old_routers)-1; i >= 0; --i) { + signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i); + if (!cur_id) { + cur_id = r->identity_digest; + hi = i; + } + if (tor_memneq(cur_id, r->identity_digest, DIGEST_LEN)) { + routerlist_remove_old_cached_routers_with_id(now, + cutoff, i+1, hi, retain); + cur_id = r->identity_digest; + hi = i; + } + } + if (hi>=0) + routerlist_remove_old_cached_routers_with_id(now, cutoff, 0, hi, retain); + //routerlist_assert_ok(routerlist); + + done: + digestset_free(retain); + router_rebuild_store(RRS_DONT_REMOVE_OLD, &routerlist->desc_store); + router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store); +} + +/** We just added a new set of descriptors. Take whatever extra steps + * we need. */ +void +routerlist_descriptors_added(smartlist_t *sl, int from_cache) +{ + tor_assert(sl); + control_event_descriptors_changed(sl); + SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { + if (ri->purpose == ROUTER_PURPOSE_BRIDGE) + learned_bridge_descriptor(ri, from_cache); + if (ri->needs_retest_if_added) { + ri->needs_retest_if_added = 0; + dirserv_single_reachability_test(approx_time(), ri); + } + } SMARTLIST_FOREACH_END(ri); +} + +/** + * Code to parse a single router descriptor and insert it into the + * routerlist. Return -1 if the descriptor was ill-formed; 0 if the + * descriptor was well-formed but could not be added; and 1 if the + * descriptor was added. + * + * If we don't add it and msg is not NULL, then assign to + * *msg a static string describing the reason for refusing the + * descriptor. + * + * This is used only by the controller. + */ +int +router_load_single_router(const char *s, uint8_t purpose, int cache, + const char **msg) +{ + routerinfo_t *ri; + was_router_added_t r; + smartlist_t *lst; + char annotation_buf[ROUTER_ANNOTATION_BUF_LEN]; + tor_assert(msg); + *msg = NULL; + + tor_snprintf(annotation_buf, sizeof(annotation_buf), + "@source controller\n" + "@purpose %s\n", router_purpose_to_string(purpose)); + + if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, + annotation_buf, NULL))) { + log_warn(LD_DIR, "Error parsing router descriptor; dropping."); + *msg = "Couldn't parse router descriptor."; + return -1; + } + tor_assert(ri->purpose == purpose); + if (router_is_me(ri)) { + log_warn(LD_DIR, "Router's identity key matches mine; dropping."); + *msg = "Router's identity key matches mine."; + routerinfo_free(ri); + return 0; + } + + if (!cache) /* obey the preference of the controller */ + ri->cache_info.do_not_cache = 1; + + lst = smartlist_new(); + smartlist_add(lst, ri); + routers_update_status_from_consensus_networkstatus(lst, 0); + + r = router_add_to_routerlist(ri, msg, 0, 0); + if (!WRA_WAS_ADDED(r)) { + /* we've already assigned to *msg now, and ri is already freed */ + tor_assert(*msg); + if (r == ROUTER_AUTHDIR_REJECTS) + log_warn(LD_DIR, "Couldn't add router to list: %s Dropping.", *msg); + smartlist_free(lst); + return 0; + } else { + routerlist_descriptors_added(lst, 0); + smartlist_free(lst); + log_debug(LD_DIR, "Added router to list"); + return 1; + } +} + +/** Given a string s containing some routerdescs, parse it and put the + * routers into our directory. If saved_location is SAVED_NOWHERE, the routers + * are in response to a query to the network: cache them by adding them to + * the journal. + * + * Return the number of routers actually added. + * + * If requested_fingerprints is provided, it must contain a list of + * uppercased fingerprints. Do not update any router whose + * fingerprint is not on the list; after updating a router, remove its + * fingerprint from the list. + * + * If descriptor_digests is non-zero, then the requested_fingerprints + * are descriptor digests. Otherwise they are identity digests. + */ +int +router_load_routers_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests, + const char *prepend_annotations) +{ + smartlist_t *routers = smartlist_new(), *changed = smartlist_new(); + char fp[HEX_DIGEST_LEN+1]; + const char *msg; + int from_cache = (saved_location != SAVED_NOWHERE); + int allow_annotations = (saved_location != SAVED_NOWHERE); + int any_changed = 0; + smartlist_t *invalid_digests = smartlist_new(); + + router_parse_list_from_string(&s, eos, routers, saved_location, 0, + allow_annotations, prepend_annotations, + invalid_digests); + + routers_update_status_from_consensus_networkstatus(routers, !from_cache); + + log_info(LD_DIR, "%d elements to add", smartlist_len(routers)); + + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + was_router_added_t r; + char d[DIGEST_LEN]; + if (requested_fingerprints) { + base16_encode(fp, sizeof(fp), descriptor_digests ? + ri->cache_info.signed_descriptor_digest : + ri->cache_info.identity_digest, + DIGEST_LEN); + if (smartlist_contains_string(requested_fingerprints, fp)) { + smartlist_string_remove(requested_fingerprints, fp); + } else { + char *requested = + smartlist_join_strings(requested_fingerprints," ",0,NULL); + log_warn(LD_DIR, + "We received a router descriptor with a fingerprint (%s) " + "that we never requested. (We asked for: %s.) Dropping.", + fp, requested); + tor_free(requested); + routerinfo_free(ri); + continue; + } + } + + memcpy(d, ri->cache_info.signed_descriptor_digest, DIGEST_LEN); + r = router_add_to_routerlist(ri, &msg, from_cache, !from_cache); + if (WRA_WAS_ADDED(r)) { + any_changed++; + smartlist_add(changed, ri); + routerlist_descriptors_added(changed, from_cache); + smartlist_clear(changed); + } else if (WRA_NEVER_DOWNLOADABLE(r)) { + download_status_t *dl_status; + dl_status = router_get_dl_status_by_descriptor_digest(d); + if (dl_status) { + log_info(LD_GENERAL, "Marking router %s as never downloadable", + hex_str(d, DIGEST_LEN)); + download_status_mark_impossible(dl_status); + } + } + } SMARTLIST_FOREACH_END(ri); + + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints && descriptor_digests) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + download_status_t *dls; + dls = router_get_dl_status_by_descriptor_digest((char*)bad_digest); + if (dls) { + log_info(LD_GENERAL, "Marking router with descriptor %s as unparseable, " + "and therefore undownloadable", fp); + download_status_mark_impossible(dls); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + + routerlist_assert_ok(routerlist); + + if (any_changed) + router_rebuild_store(0, &routerlist->desc_store); + + smartlist_free(routers); + smartlist_free(changed); + + return any_changed; +} + +/** Parse one or more extrainfos from s (ending immediately before + * eos if eos is present). Other arguments are as for + * router_load_routers_from_string(). */ +void +router_load_extrainfo_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests) +{ + smartlist_t *extrainfo_list = smartlist_new(); + const char *msg; + int from_cache = (saved_location != SAVED_NOWHERE); + smartlist_t *invalid_digests = smartlist_new(); + + router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0, + NULL, invalid_digests); + + log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list)); + + SMARTLIST_FOREACH_BEGIN(extrainfo_list, extrainfo_t *, ei) { + uint8_t d[DIGEST_LEN]; + memcpy(d, ei->cache_info.signed_descriptor_digest, DIGEST_LEN); + was_router_added_t added = + router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache); + if (WRA_WAS_ADDED(added) && requested_fingerprints) { + char fp[HEX_DIGEST_LEN+1]; + base16_encode(fp, sizeof(fp), descriptor_digests ? + ei->cache_info.signed_descriptor_digest : + ei->cache_info.identity_digest, + DIGEST_LEN); + smartlist_string_remove(requested_fingerprints, fp); + /* We silently let relays stuff us with extrainfos we didn't ask for, + * so long as we would have wanted them anyway. Since we always fetch + * all the extrainfos we want, and we never actually act on them + * inside Tor, this should be harmless. */ + } else if (WRA_NEVER_DOWNLOADABLE(added)) { + signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", + hex_str((char*)d,DIGEST_LEN)); + download_status_mark_impossible(&sd->ei_dl_status); + } + } + } SMARTLIST_FOREACH_END(ei); + + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + char fp[HEX_DIGEST_LEN+1]; + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + signed_descriptor_t *sd = + router_get_by_extrainfo_digest((char*)bad_digest); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", fp); + download_status_mark_impossible(&sd->ei_dl_status); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + + routerlist_assert_ok(routerlist); + router_rebuild_store(0, &router_get_routerlist()->extrainfo_store); + + smartlist_free(extrainfo_list); +} + +/** Return true iff the latest ns-flavored consensus includes a descriptor + * whose digest is that of desc. */ +static int +signed_desc_digest_is_recognized(signed_descriptor_t *desc) +{ + const routerstatus_t *rs; + networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor( + FLAV_NS); + + if (consensus) { + rs = networkstatus_vote_find_entry(consensus, desc->identity_digest); + if (rs && tor_memeq(rs->descriptor_digest, + desc->signed_descriptor_digest, DIGEST_LEN)) + return 1; + } + return 0; +} + +/** Update downloads for router descriptors and/or microdescriptors as + * appropriate. */ +void +update_all_descriptor_downloads(time_t now) +{ + if (should_delay_dir_fetches(get_options(), NULL)) + return; + update_router_descriptor_downloads(now); + update_microdesc_downloads(now); + launch_dummy_descriptor_download_as_needed(now, get_options()); +} + +/** Clear all our timeouts for fetching v3 directory stuff, and then + * give it all a try again. */ +void +routerlist_retry_directory_downloads(time_t now) +{ + (void)now; + + log_debug(LD_GENERAL, + "In routerlist_retry_directory_downloads()"); + + router_reset_status_download_failures(); + router_reset_descriptor_download_failures(); + reschedule_directory_downloads(); +} + +/** Return true iff router does not permit exit streams. + */ +int +router_exit_policy_rejects_all(const routerinfo_t *router) +{ + return router->policy_is_reject_star; +} + +/** For every current directory connection whose purpose is purpose, + * and where the resource being downloaded begins with prefix, split + * rest of the resource into base16 fingerprints (or base64 fingerprints if + * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the + * corresponding elements of result to a nonzero value. + */ +void +list_pending_downloads(digestmap_t *result, digest256map_t *result256, + int purpose, const char *prefix) +{ + const size_t p_len = strlen(prefix); + smartlist_t *tmp = smartlist_new(); + smartlist_t *conns = get_connection_array(); + int flags = DSR_HEX; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) + flags = DSR_DIGEST256|DSR_BASE64; + + tor_assert(result || result256); + + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + if (conn->type == CONN_TYPE_DIR && + conn->purpose == purpose && + !conn->marked_for_close) { + const char *resource = TO_DIR_CONN(conn)->requested_resource; + if (!strcmpstart(resource, prefix)) + dir_split_resource_into_fingerprints(resource + p_len, + tmp, NULL, flags); + } + } SMARTLIST_FOREACH_END(conn); + + if (result) { + SMARTLIST_FOREACH(tmp, char *, d, + { + digestmap_set(result, d, (void*)1); + tor_free(d); + }); + } else if (result256) { + SMARTLIST_FOREACH(tmp, uint8_t *, d, + { + digest256map_set(result256, d, (void*)1); + tor_free(d); + }); + } + smartlist_free(tmp); +} + +/** For every router descriptor (or extra-info document if extrainfo is + * true) we are currently downloading by descriptor digest, set result[d] to + * (void*)1. */ +static void +list_pending_descriptor_downloads(digestmap_t *result, int extrainfo) +{ + int purpose = + extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC; + list_pending_downloads(result, NULL, purpose, "d/"); +} + +/** For every microdescriptor we are currently downloading by descriptor + * digest, set result[d] to (void*)1. + */ +void +list_pending_microdesc_downloads(digest256map_t *result) +{ + list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); +} + +/** Launch downloads for all the descriptors whose digests or digests256 + * are listed as digests[i] for lo <= i < hi. (Lo and hi may be out of + * range.) If source is given, download from source; + * otherwise, download from an appropriate random directory server. + */ +MOCK_IMPL(STATIC void, +initiate_descriptor_downloads,(const routerstatus_t *source, + int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)) +{ + char *resource, *cp; + int digest_len, enc_digest_len; + const char *sep; + int b64_256; + smartlist_t *tmp; + + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + /* Microdescriptors are downloaded by "-"-separated base64-encoded + * 256-bit digests. */ + digest_len = DIGEST256_LEN; + enc_digest_len = BASE64_DIGEST256_LEN + 1; + sep = "-"; + b64_256 = 1; + } else { + digest_len = DIGEST_LEN; + enc_digest_len = HEX_DIGEST_LEN + 1; + sep = "+"; + b64_256 = 0; + } + + if (lo < 0) + lo = 0; + if (hi > smartlist_len(digests)) + hi = smartlist_len(digests); + + if (hi-lo <= 0) + return; + + tmp = smartlist_new(); + + for (; lo < hi; ++lo) { + cp = tor_malloc(enc_digest_len); + if (b64_256) { + digest256_to_base64(cp, smartlist_get(digests, lo)); + } else { + base16_encode(cp, enc_digest_len, smartlist_get(digests, lo), + digest_len); + } + smartlist_add(tmp, cp); + } + + cp = smartlist_join_strings(tmp, sep, 0, NULL); + tor_asprintf(&resource, "d/%s.z", cp); + + SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1)); + smartlist_free(tmp); + tor_free(cp); + + if (source) { + /* We know which authority or directory mirror we want. */ + directory_request_t *req = directory_request_new(purpose); + directory_request_set_routerstatus(req, source); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); + } else { + directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource, + pds_flags, DL_WANT_ANY_DIRSERVER); + } + tor_free(resource); +} + +/** Return the max number of hashes to put in a URL for a given request. + */ +static int +max_dl_per_request(const or_options_t *options, int purpose) +{ + /* Since squid does not like URLs >= 4096 bytes we limit it to 96. + * 4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535 + * /tor/server/d/.z) == 4026 + * 4026/41 (40 for the hash and 1 for the + that separates them) => 98 + * So use 96 because it's a nice number. + * + * For microdescriptors, the calculation is + * 4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535 + * /tor/micro/d/.z) == 4027 + * 4027/44 (43 for the hash and 1 for the - that separates them) => 91 + * So use 90 because it's a nice number. + */ + int max = 96; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + max = 90; + } + /* If we're going to tunnel our connections, we can ask for a lot more + * in a request. */ + if (directory_must_use_begindir(options)) { + max = 500; + } + return max; +} + +/** Don't split our requests so finely that we are requesting fewer than + * this number per server. (Grouping more than this at once leads to + * diminishing returns.) */ +#define MIN_DL_PER_REQUEST 32 +/** To prevent a single screwy cache from confusing us by selective reply, + * try to split our requests into at least this many requests. */ +#define MIN_REQUESTS 3 +/** If we want fewer than this many descriptors, wait until we + * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */ +#define MAX_DL_TO_DELAY 16 + +/** Given a purpose (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of + * router descriptor digests or microdescriptor digest256s in + * downloadable, decide whether to delay fetching until we have more. + * If we don't want to delay, launch one or more requests to the appropriate + * directory authorities. + */ +void +launch_descriptor_downloads(int purpose, + smartlist_t *downloadable, + const routerstatus_t *source, time_t now) +{ + const or_options_t *options = get_options(); + const char *descname; + const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC); + int n_downloadable = smartlist_len(downloadable); + + int i, n_per_request, max_dl_per_req; + const char *req_plural = "", *rtr_plural = ""; + int pds_flags = PDS_RETRY_IF_NO_SERVERS; + + tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC); + descname = fetch_microdesc ? "microdesc" : "routerdesc"; + + if (!n_downloadable) + return; + + if (!directory_fetches_dir_info_early(options)) { + if (n_downloadable >= MAX_DL_TO_DELAY) { + log_debug(LD_DIR, + "There are enough downloadable %ss to launch requests.", + descname); + } else if (! router_have_minimum_dir_info()) { + log_debug(LD_DIR, + "We are only missing %d %ss, but we'll fetch anyway, since " + "we don't yet have enough directory info.", + n_downloadable, descname); + } else { + + /* should delay */ + if ((last_descriptor_download_attempted + + options->TestingClientMaxIntervalWithoutRequest) > now) + return; + + if (last_descriptor_download_attempted) { + log_info(LD_DIR, + "There are not many downloadable %ss, but we've " + "been waiting long enough (%d seconds). Downloading.", + descname, + (int)(now-last_descriptor_download_attempted)); + } else { + log_info(LD_DIR, + "There are not many downloadable %ss, but we haven't " + "tried downloading descriptors recently. Downloading.", + descname); + } + } + } + + if (!authdir_mode(options)) { + /* If we wind up going to the authorities, we want to only open one + * connection to each authority at a time, so that we don't overload + * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH + * regardless of whether we're a cache or not. + * + * Setting this flag can make initiate_descriptor_downloads() ignore + * requests. We need to make sure that we do in fact call + * update_router_descriptor_downloads() later on, once the connections + * have succeeded or failed. + */ + pds_flags |= fetch_microdesc ? + PDS_NO_EXISTING_MICRODESC_FETCH : + PDS_NO_EXISTING_SERVERDESC_FETCH; + } + + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); + max_dl_per_req = max_dl_per_request(options, purpose); + + if (n_per_request > max_dl_per_req) + n_per_request = max_dl_per_req; + + if (n_per_request < MIN_DL_PER_REQUEST) { + n_per_request = MIN(MIN_DL_PER_REQUEST, n_downloadable); + } + + if (n_downloadable > n_per_request) + req_plural = rtr_plural = "s"; + else if (n_downloadable > 1) + rtr_plural = "s"; + + log_info(LD_DIR, + "Launching %d request%s for %d %s%s, %d at a time", + CEIL_DIV(n_downloadable, n_per_request), req_plural, + n_downloadable, descname, rtr_plural, n_per_request); + smartlist_sort_digests(downloadable); + for (i=0; i < n_downloadable; i += n_per_request) { + initiate_descriptor_downloads(source, purpose, + downloadable, i, i+n_per_request, + pds_flags); + } + last_descriptor_download_attempted = now; +} + +/** For any descriptor that we want that's currently listed in + * consensus, download it as appropriate. */ +void +update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus) +{ + const or_options_t *options = get_options(); + digestmap_t *map = NULL; + smartlist_t *no_longer_old = smartlist_new(); + smartlist_t *downloadable = smartlist_new(); + routerstatus_t *source = NULL; + int authdir = authdir_mode(options); + int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0, + n_inprogress=0, n_in_oldrouters=0; + + if (directory_too_idle_to_fetch_descriptors(options, now)) + goto done; + if (!consensus) + goto done; + + if (is_vote) { + /* where's it from, so we know whom to ask for descriptors */ + dir_server_t *ds; + networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0); + tor_assert(voter); + ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest); + if (ds) + source = &(ds->fake_status); + else + log_warn(LD_DIR, "couldn't lookup source from vote?"); + } + + map = digestmap_new(); + list_pending_descriptor_downloads(map, 0); + SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, void *, rsp) { + routerstatus_t *rs = + is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp; + signed_descriptor_t *sd; + if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) { + const routerinfo_t *ri; + ++n_have; + if (!(ri = router_get_by_id_digest(rs->identity_digest)) || + tor_memneq(ri->cache_info.signed_descriptor_digest, + sd->signed_descriptor_digest, DIGEST_LEN)) { + /* We have a descriptor with this digest, but either there is no + * entry in routerlist with the same ID (!ri), or there is one, + * but the identity digest differs (memneq). + */ + smartlist_add(no_longer_old, sd); + ++n_in_oldrouters; /* We have it in old_routers. */ + } + continue; /* We have it already. */ + } + if (digestmap_get(map, rs->descriptor_digest)) { + ++n_inprogress; + continue; /* We have an in-progress download. */ + } + if (!download_status_is_ready(&rs->dl_status, now)) { + ++n_delayed; /* Not ready for retry. */ + continue; + } + if (authdir && dirserv_would_reject_router(rs)) { + ++n_would_reject; + continue; /* We would throw it out immediately. */ + } + if (!we_want_to_fetch_flavor(options, consensus->flavor) && + !client_would_use_router(rs, now)) { + ++n_wouldnt_use; + continue; /* We would never use it ourself. */ + } + if (is_vote && source) { + char time_bufnew[ISO_TIME_LEN+1]; + char time_bufold[ISO_TIME_LEN+1]; + const routerinfo_t *oldrouter; + oldrouter = router_get_by_id_digest(rs->identity_digest); + format_iso_time(time_bufnew, rs->published_on); + if (oldrouter) + format_iso_time(time_bufold, oldrouter->cache_info.published_on); + log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)", + routerstatus_describe(rs), + time_bufnew, + oldrouter ? time_bufold : "none", + source->nickname, oldrouter ? "known" : "unknown"); + } + smartlist_add(downloadable, rs->descriptor_digest); + } SMARTLIST_FOREACH_END(rsp); + + if (!authdir_mode_v3(options) + && smartlist_len(no_longer_old)) { + routerlist_t *rl = router_get_routerlist(); + log_info(LD_DIR, "%d router descriptors listed in consensus are " + "currently in old_routers; making them current.", + smartlist_len(no_longer_old)); + SMARTLIST_FOREACH_BEGIN(no_longer_old, signed_descriptor_t *, sd) { + const char *msg; + was_router_added_t r; + time_t tmp_cert_expiration_time; + routerinfo_t *ri = routerlist_reparse_old(rl, sd); + if (!ri) { + log_warn(LD_BUG, "Failed to re-parse a router."); + continue; + } + /* need to remember for below, since add_to_routerlist may free. */ + tmp_cert_expiration_time = ri->cert_expiration_time; + + r = router_add_to_routerlist(ri, &msg, 1, 0); + if (WRA_WAS_OUTDATED(r)) { + log_warn(LD_DIR, "Couldn't add re-parsed router: %s. This isn't " + "usually a big deal, but you should make sure that your " + "clock and timezone are set correctly.", + msg?msg:"???"); + if (r == ROUTER_CERTS_EXPIRED) { + char time_cons[ISO_TIME_LEN+1]; + char time_cert_expires[ISO_TIME_LEN+1]; + format_iso_time(time_cons, consensus->valid_after); + format_iso_time(time_cert_expires, tmp_cert_expiration_time); + log_warn(LD_DIR, " (I'm looking at a consensus from %s; This " + "router's certificates began expiring at %s.)", + time_cons, time_cert_expires); + } + } + } SMARTLIST_FOREACH_END(sd); + routerlist_assert_ok(rl); + } + + log_info(LD_DIR, + "%d router descriptors downloadable. %d delayed; %d present " + "(%d of those were in old_routers); %d would_reject; " + "%d wouldnt_use; %d in progress.", + smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, + n_would_reject, n_wouldnt_use, n_inprogress); + + launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC, + downloadable, source, now); + + digestmap_free(map, NULL); + done: + smartlist_free(downloadable); + smartlist_free(no_longer_old); +} + +/** How often should we launch a server/authority request to be sure of getting + * a guess for our IP? */ +/*XXXX+ this info should come from netinfo cells or something, or we should + * do this only when we aren't seeing incoming data. see bug 652. */ +#define DUMMY_DOWNLOAD_INTERVAL (20*60) + +/** As needed, launch a dummy router descriptor fetch to see if our + * address has changed. */ +static void +launch_dummy_descriptor_download_as_needed(time_t now, + const or_options_t *options) +{ + static time_t last_dummy_download = 0; + /* XXXX+ we could be smarter here; see notes on bug 652. */ + /* If we're a server that doesn't have a configured address, we rely on + * directory fetches to learn when our address changes. So if we haven't + * tried to get any routerdescs in a long time, try a dummy fetch now. */ + if (!options->Address && + server_mode(options) && + last_descriptor_download_attempted + DUMMY_DOWNLOAD_INTERVAL < now && + last_dummy_download + DUMMY_DOWNLOAD_INTERVAL < now) { + last_dummy_download = now; + /* XX/teor - do we want an authority here, because they are less likely + * to give us the wrong address? (See #17782) + * I'm leaving the previous behaviour intact, because I don't like + * the idea of some relays contacting an authority every 20 minutes. */ + directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC, + ROUTER_PURPOSE_GENERAL, "authority.z", + PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); + } +} + +/** Launch downloads for router status as needed. */ +void +update_router_descriptor_downloads(time_t now) +{ + const or_options_t *options = get_options(); + if (should_delay_dir_fetches(options, NULL)) + return; + if (!we_fetch_router_descriptors(options)) + return; + + update_consensus_router_descriptor_downloads(now, 0, + networkstatus_get_reasonably_live_consensus(now, FLAV_NS)); +} + +/** Launch extrainfo downloads as needed. */ +void +update_extrainfo_downloads(time_t now) +{ + const or_options_t *options = get_options(); + routerlist_t *rl; + smartlist_t *wanted; + digestmap_t *pending; + int old_routers, i, max_dl_per_req; + int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0, n_bogus[2] = {0,0}; + if (! options->DownloadExtraInfo) + return; + if (should_delay_dir_fetches(options, NULL)) + return; + if (!router_have_minimum_dir_info()) + return; + + pending = digestmap_new(); + list_pending_descriptor_downloads(pending, 1); + rl = router_get_routerlist(); + wanted = smartlist_new(); + for (old_routers = 0; old_routers < 2; ++old_routers) { + smartlist_t *lst = old_routers ? rl->old_routers : rl->routers; + for (i = 0; i < smartlist_len(lst); ++i) { + signed_descriptor_t *sd; + char *d; + if (old_routers) + sd = smartlist_get(lst, i); + else + sd = &((routerinfo_t*)smartlist_get(lst, i))->cache_info; + if (sd->is_extrainfo) + continue; /* This should never happen. */ + if (old_routers && !router_get_by_id_digest(sd->identity_digest)) + continue; /* Couldn't check the signature if we got it. */ + if (sd->extrainfo_is_bogus) + continue; + d = sd->extra_info_digest; + if (tor_digest_is_zero(d)) { + ++n_no_ei; + continue; + } + if (eimap_get(rl->extra_info_map, d)) { + ++n_have; + continue; + } + if (!download_status_is_ready(&sd->ei_dl_status, now)) { + ++n_delay; + continue; + } + if (digestmap_get(pending, d)) { + ++n_pending; + continue; + } + + const signed_descriptor_t *sd2 = router_get_by_extrainfo_digest(d); + if (sd2 != sd) { + if (sd2 != NULL) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + char d3[HEX_DIGEST_LEN+1], d4[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), sd2->identity_digest, DIGEST_LEN); + base16_encode(d3, sizeof(d3), d, DIGEST_LEN); + base16_encode(d4, sizeof(d3), sd2->extra_info_digest, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with mismatched " + "router_get_by_extrainfo_digest() value. This has ID %s " + "but the entry in the map has ID %s. This has EI digest " + "%s and the entry in the map has EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2, d3, d4); + } else { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), d, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with NULL " + "router_get_by_extrainfo_digest() value. This has ID %s " + "and EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2); + } + ++n_bogus[old_routers]; + continue; + } + smartlist_add(wanted, d); + } + } + digestmap_free(pending, NULL); + + log_info(LD_DIR, "Extrainfo download status: %d router with no ei, %d " + "with present ei, %d delaying, %d pending, %d downloadable, %d " + "bogus in routers, %d bogus in old_routers", + n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted), + n_bogus[0], n_bogus[1]); + + smartlist_shuffle(wanted); + + max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO); + for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) { + initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO, + wanted, i, i+max_dl_per_req, + PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH); + } + + smartlist_free(wanted); +} + +/** Reset the consensus and extra-info download failure count on all routers. + * When we get a new consensus, + * routers_update_status_from_consensus_networkstatus() will reset the + * download statuses on the descriptors in that consensus. + */ +void +router_reset_descriptor_download_failures(void) +{ + log_debug(LD_GENERAL, + "In router_reset_descriptor_download_failures()"); + + networkstatus_reset_download_failures(); + last_descriptor_download_attempted = 0; + if (!routerlist) + return; + /* We want to download *all* extra-info descriptors, not just those in + * the consensus we currently have (or are about to have) */ + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri, + { + download_status_reset(&ri->cache_info.ei_dl_status); + }); + SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, + { + download_status_reset(&sd->ei_dl_status); + }); +} + +/** Any changes in a router descriptor's publication time larger than this are + * automatically non-cosmetic. */ +#define ROUTER_MAX_COSMETIC_TIME_DIFFERENCE (2*60*60) + +/** We allow uptime to vary from how much it ought to be by this much. */ +#define ROUTER_ALLOW_UPTIME_DRIFT (6*60*60) + +/** Return true iff the only differences between r1 and r2 are such that + * would not cause a recent (post 0.1.1.6) dirserver to republish. + */ +int +router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) +{ + time_t r1pub, r2pub; - long time_difference; ++ time_t time_difference; + tor_assert(r1 && r2); + + /* r1 should be the one that was published first. */ + if (r1->cache_info.published_on > r2->cache_info.published_on) { + const routerinfo_t *ri_tmp = r2; + r2 = r1; + r1 = ri_tmp; + } + + /* If any key fields differ, they're different. */ + if (r1->addr != r2->addr || + strcasecmp(r1->nickname, r2->nickname) || + r1->or_port != r2->or_port || + !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || + r1->ipv6_orport != r2->ipv6_orport || + r1->dir_port != r2->dir_port || + r1->purpose != r2->purpose || + r1->onion_pkey_len != r2->onion_pkey_len || + !tor_memeq(r1->onion_pkey, r2->onion_pkey, r1->onion_pkey_len) || + !crypto_pk_eq_keys(r1->identity_pkey, r2->identity_pkey) || + strcasecmp(r1->platform, r2->platform) || + (r1->contact_info && !r2->contact_info) || /* contact_info is optional */ + (!r1->contact_info && r2->contact_info) || + (r1->contact_info && r2->contact_info && + strcasecmp(r1->contact_info, r2->contact_info)) || + r1->is_hibernating != r2->is_hibernating || + ! addr_policies_eq(r1->exit_policy, r2->exit_policy) || + (r1->supports_tunnelled_dir_requests != + r2->supports_tunnelled_dir_requests)) + return 0; + if ((r1->declared_family == NULL) != (r2->declared_family == NULL)) + return 0; + if (r1->declared_family && r2->declared_family) { + int i, n; + if (smartlist_len(r1->declared_family)!=smartlist_len(r2->declared_family)) + return 0; + n = smartlist_len(r1->declared_family); + for (i=0; i < n; ++i) { + if (strcasecmp(smartlist_get(r1->declared_family, i), + smartlist_get(r2->declared_family, i))) + return 0; + } + } + + /* Did bandwidth change a lot? */ + if ((r1->bandwidthcapacity < r2->bandwidthcapacity/2) || + (r2->bandwidthcapacity < r1->bandwidthcapacity/2)) + return 0; + + /* Did the bandwidthrate or bandwidthburst change? */ + if ((r1->bandwidthrate != r2->bandwidthrate) || + (r1->bandwidthburst != r2->bandwidthburst)) + return 0; + + /* Did more than 12 hours pass? */ + if (r1->cache_info.published_on + ROUTER_MAX_COSMETIC_TIME_DIFFERENCE + < r2->cache_info.published_on) + return 0; + + /* Did uptime fail to increase by approximately the amount we would think, + * give or take some slop? */ + r1pub = r1->cache_info.published_on; + r2pub = r2->cache_info.published_on; - time_difference = labs(r2->uptime - (r1->uptime + (r2pub - r1pub))); ++ time_difference = r2->uptime - (r1->uptime + (r2pub - r1pub)); ++ if (time_difference < 0) ++ time_difference = - time_difference; + if (time_difference > ROUTER_ALLOW_UPTIME_DRIFT && + time_difference > r1->uptime * .05 && + time_difference > r2->uptime * .05) + return 0; + + /* Otherwise, the difference is cosmetic. */ + return 1; +} + +/** Check whether sd describes a router descriptor compatible with the + * extrainfo document ei. + * + * identity_pkey (which must also be provided) is RSA1024 identity key + * for the router. We use it to check the signature of the extrainfo document, + * if it has not already been checked. + * + * If no router is compatible with ei, ei should be + * dropped. Return 0 for "compatible", return 1 for "reject, and inform + * whoever uploaded ei, and return -1 for "reject silently.". If + * msg is present, set *msg to a description of the + * incompatibility (if any). + * + * Set the extrainfo_is_bogus field in sd if the digests matched + * but the extrainfo was nonetheless incompatible. + **/ +int +routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey, + extrainfo_t *ei, + signed_descriptor_t *sd, + const char **msg) +{ + int digest_matches, digest256_matches, r=1; + tor_assert(identity_pkey); + tor_assert(sd); + tor_assert(ei); + + if (ei->bad_sig) { + if (msg) *msg = "Extrainfo signature was bad, or signed with wrong key."; + return 1; + } + + digest_matches = tor_memeq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN); + /* Set digest256_matches to 1 if the digest is correct, or if no + * digest256 was in the ri. */ + digest256_matches = tor_memeq(ei->digest256, + sd->extra_info_digest256, DIGEST256_LEN); + digest256_matches |= + tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN); + + /* The identity must match exactly to have been generated at the same time + * by the same router. */ + if (tor_memneq(sd->identity_digest, + ei->cache_info.identity_digest, + DIGEST_LEN)) { + if (msg) *msg = "Extrainfo nickname or identity did not match routerinfo"; + goto err; /* different servers */ + } + + if (! tor_cert_opt_eq(sd->signing_key_cert, + ei->cache_info.signing_key_cert)) { + if (msg) *msg = "Extrainfo signing key cert didn't match routerinfo"; + goto err; /* different servers */ + } + + if (ei->pending_sig) { + char signed_digest[128]; + if (crypto_pk_public_checksig(identity_pkey, + signed_digest, sizeof(signed_digest), + ei->pending_sig, ei->pending_sig_len) != DIGEST_LEN || + tor_memneq(signed_digest, ei->cache_info.signed_descriptor_digest, + DIGEST_LEN)) { + ei->bad_sig = 1; + tor_free(ei->pending_sig); + if (msg) *msg = "Extrainfo signature bad, or signed with wrong key"; + goto err; /* Bad signature, or no match. */ + } + + ei->cache_info.send_unencrypted = sd->send_unencrypted; + tor_free(ei->pending_sig); + } + + if (ei->cache_info.published_on < sd->published_on) { + if (msg) *msg = "Extrainfo published time did not match routerdesc"; + goto err; + } else if (ei->cache_info.published_on > sd->published_on) { + if (msg) *msg = "Extrainfo published time did not match routerdesc"; + r = -1; + goto err; + } + + if (!digest256_matches && !digest_matches) { + if (msg) *msg = "Neither digest256 or digest matched " + "digest from routerdesc"; + goto err; + } + + if (!digest256_matches) { + if (msg) *msg = "Extrainfo digest did not match digest256 from routerdesc"; + goto err; /* Digest doesn't match declared value. */ + } + + if (!digest_matches) { + if (msg) *msg = "Extrainfo digest did not match value from routerdesc"; + goto err; /* Digest doesn't match declared value. */ + } + + return 0; + err: + if (digest_matches) { + /* This signature was okay, and the digest was right: This is indeed the + * corresponding extrainfo. But insanely, it doesn't match the routerinfo + * that lists it. Don't try to fetch this one again. */ + sd->extrainfo_is_bogus = 1; + } + + return r; +} + +/* Does ri have a valid ntor onion key? + * Valid ntor onion keys exist and have at least one non-zero byte. */ +int +routerinfo_has_curve25519_onion_key(const routerinfo_t *ri) +{ + if (!ri) { + return 0; + } + + if (!ri->onion_curve25519_pkey) { + return 0; + } + + if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key, + CURVE25519_PUBKEY_LEN)) { + return 0; + } + + return 1; +} + +/* Is rs running a tor version known to support EXTEND2 cells? + * If allow_unknown_versions is true, return true if we can't tell + * (from a versions line or a protocols line) whether it supports extend2 + * cells. + * Otherwise, return false if the version is unknown. */ +int +routerstatus_version_supports_extend2_cells(const routerstatus_t *rs, + int allow_unknown_versions) +{ + if (!rs) { + return allow_unknown_versions; + } + + if (!rs->pv.protocols_known) { + return allow_unknown_versions; + } + + return rs->pv.supports_extend2_cells; +} + +/** Assert that the internal representation of rl is + * self-consistent. */ +void +routerlist_assert_ok(const routerlist_t *rl) +{ + routerinfo_t *r2; + signed_descriptor_t *sd2; + if (!rl) + return; + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, r) { + r2 = rimap_get(rl->identity_map, r->cache_info.identity_digest); + tor_assert(r == r2); + sd2 = sdmap_get(rl->desc_digest_map, + r->cache_info.signed_descriptor_digest); + tor_assert(&(r->cache_info) == sd2); + tor_assert(r->cache_info.routerlist_index == r_sl_idx); + /* XXXX + * + * Hoo boy. We need to fix this one, and the fix is a bit tricky, so + * commenting this out is just a band-aid. + * + * The problem is that, although well-behaved router descriptors + * should never have the same value for their extra_info_digest, it's + * possible for ill-behaved routers to claim whatever they like there. + * + * The real answer is to trash desc_by_eid_map and instead have + * something that indicates for a given extra-info digest we want, + * what its download status is. We'll do that as a part of routerlist + * refactoring once consensus directories are in. For now, + * this rep violation is probably harmless: an adversary can make us + * reset our retry count for an extrainfo, but that's not the end + * of the world. Changing the representation in 0.2.0.x would just + * destabilize the codebase. + if (!tor_digest_is_zero(r->cache_info.extra_info_digest)) { + signed_descriptor_t *sd3 = + sdmap_get(rl->desc_by_eid_map, r->cache_info.extra_info_digest); + tor_assert(sd3 == &(r->cache_info)); + } + */ + } SMARTLIST_FOREACH_END(r); + SMARTLIST_FOREACH_BEGIN(rl->old_routers, signed_descriptor_t *, sd) { + r2 = rimap_get(rl->identity_map, sd->identity_digest); + tor_assert(!r2 || sd != &(r2->cache_info)); + sd2 = sdmap_get(rl->desc_digest_map, sd->signed_descriptor_digest); + tor_assert(sd == sd2); + tor_assert(sd->routerlist_index == sd_sl_idx); + /* XXXX see above. + if (!tor_digest_is_zero(sd->extra_info_digest)) { + signed_descriptor_t *sd3 = + sdmap_get(rl->desc_by_eid_map, sd->extra_info_digest); + tor_assert(sd3 == sd); + } + */ + } SMARTLIST_FOREACH_END(sd); + + RIMAP_FOREACH(rl->identity_map, d, r) { + tor_assert(tor_memeq(r->cache_info.identity_digest, d, DIGEST_LEN)); + } DIGESTMAP_FOREACH_END; + SDMAP_FOREACH(rl->desc_digest_map, d, sd) { + tor_assert(tor_memeq(sd->signed_descriptor_digest, d, DIGEST_LEN)); + } DIGESTMAP_FOREACH_END; + SDMAP_FOREACH(rl->desc_by_eid_map, d, sd) { + tor_assert(!tor_digest_is_zero(d)); + tor_assert(sd); + tor_assert(tor_memeq(sd->extra_info_digest, d, DIGEST_LEN)); + } DIGESTMAP_FOREACH_END; + EIMAP_FOREACH(rl->extra_info_map, d, ei) { + signed_descriptor_t *sd; + tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest, + d, DIGEST_LEN)); + sd = sdmap_get(rl->desc_by_eid_map, + ei->cache_info.signed_descriptor_digest); + // tor_assert(sd); // XXXX see above + if (sd) { + tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN)); + } + } DIGESTMAP_FOREACH_END; +} + +/** Allocate and return a new string representing the contact info + * and platform string for router, + * surrounded by quotes and using standard C escapes. + * + * THIS FUNCTION IS NOT REENTRANT. Don't call it from outside the main + * thread. Also, each call invalidates the last-returned value, so don't + * try log_warn(LD_GENERAL, "%s %s", esc_router_info(a), esc_router_info(b)); + * + * If router is NULL, it just frees its internal memory and returns. + */ +const char * +esc_router_info(const routerinfo_t *router) +{ + static char *info=NULL; + char *esc_contact, *esc_platform; + tor_free(info); + + if (!router) + return NULL; /* we're exiting; just free the memory we use */ + + esc_contact = esc_for_log(router->contact_info); + esc_platform = esc_for_log(router->platform); + + tor_asprintf(&info, "Contact %s, Platform %s", esc_contact, esc_platform); + tor_free(esc_contact); + tor_free(esc_platform); + + return info; +} + +/** Helper for sorting: compare two routerinfos by their identity + * digest. */ +static int +compare_routerinfo_by_id_digest_(const void **a, const void **b) +{ + routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; + return fast_memcmp(first->cache_info.identity_digest, + second->cache_info.identity_digest, + DIGEST_LEN); +} + +/** Sort a list of routerinfo_t in ascending order of identity digest. */ +void +routers_sort_by_identity(smartlist_t *routers) +{ + smartlist_sort(routers, compare_routerinfo_by_id_digest_); +} + +/** Called when we change a node set, or when we reload the geoip IPv4 list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ +void +refresh_all_country_info(void) +{ + const or_options_t *options = get_options(); + + if (options->EntryNodes) + routerset_refresh_countries(options->EntryNodes); + if (options->ExitNodes) + routerset_refresh_countries(options->ExitNodes); + if (options->ExcludeNodes) + routerset_refresh_countries(options->ExcludeNodes); + if (options->ExcludeExitNodes) + routerset_refresh_countries(options->ExcludeExitNodes); + if (options->ExcludeExitNodesUnion_) + routerset_refresh_countries(options->ExcludeExitNodesUnion_); + + nodelist_refresh_countries(); +}

1 0

[tor/master] Merge branch 'ticket31343_035' into ticket31343_040
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit 79569d86b36ae1a89b5656bf8356247574b32e95 Merge: 2a42d6be2 bc9492a93 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 6 11:18:40 2019 -0400 Merge branch 'ticket31343_035' into ticket31343_040 changes/bug31343 | 9 +++++++++ src/core/or/channeltls.c | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --cc src/core/or/channeltls.c index f552b2077,91a424728..5a00a9e00 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@@ -1635,36 -1637,19 +1635,48 @@@ channel_tls_process_padding_negotiate_c } /** + * Convert netinfo_addr into corresponding tor_addr. + * Return 0 on success; on failure, return -1 and log a warning. + */ +static int +tor_addr_from_netinfo_addr(tor_addr_t *tor_addr, + const netinfo_addr_t *netinfo_addr) { + tor_assert(tor_addr); + tor_assert(netinfo_addr); + + uint8_t type = netinfo_addr_get_addr_type(netinfo_addr); + uint8_t len = netinfo_addr_get_len(netinfo_addr); + + if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4) { + uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr); + tor_addr_from_ipv4h(tor_addr, ipv4); + } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) { + const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6( + netinfo_addr); + tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO " + "- wrong type/length."); + return -1; + } + + return 0; +} + +/** - * Process a 'netinfo' cell. + * Helper: compute the absolute value of a time_t. + * + * (we need this because labs() doesn't always work for time_t, since + * long can be shorter than time_t.) + */ + static inline time_t + time_abs(time_t val) + { + return (val < 0) ? -val : val; + } + + /** + * Process a 'netinfo' cell * * This function is called to handle an incoming NETINFO cell; read and act * on its contents, and set the connection state to "open". @@@ -1745,29 -1732,21 +1757,29 @@@ channel_tls_process_netinfo_cell(cell_ } /* Decode the cell. */ - timestamp = ntohl(get_uint32(cell->payload)); - const time_t sent_versions_at = - chan->conn->handshake_state->sent_versions_at; - if (now > sent_versions_at && (now - sent_versions_at) < 180) { - /* If we have gotten the NETINFO cell reasonably soon after having - * sent our VERSIONS cell, maybe we can learn skew information from it. */ - apparent_skew = now - timestamp; + netinfo_cell_t *netinfo_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload, + CELL_PAYLOAD_SIZE); + + if (parsed < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Failed to parse NETINFO cell - closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; } - my_addr_type = (uint8_t) cell->payload[4]; - my_addr_len = (uint8_t) cell->payload[5]; - my_addr_ptr = (uint8_t*) cell->payload + 6; - end = cell->payload + CELL_PAYLOAD_SIZE; - cp = cell->payload + 6 + my_addr_len; + timestamp = netinfo_cell_get_timestamp(netinfo_cell); + const netinfo_addr_t *my_addr = + netinfo_cell_getconst_other_addr(netinfo_cell); + + my_addr_type = netinfo_addr_get_addr_type(my_addr); + my_addr_len = netinfo_addr_get_len(my_addr); + - if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { ++ if ((now - chan->conn->handshake_state->sent_versions_at) < 180) { + apparent_skew = now - timestamp; + } /* We used to check: * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { *

1 0

[tor/master] Merge branch 'ticket31343_029' into ticket31343_035
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit bc9492a938f0bc2ee8dfb6f94f0f8d81f16b9575 Merge: 1dd952789 0849d2a2f Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 6 11:15:20 2019 -0400 Merge branch 'ticket31343_029' into ticket31343_035 changes/bug31343 | 9 +++++++++ src/core/or/channeltls.c | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --cc src/core/or/channeltls.c index a83d54ed3,000000000..91a424728 mode 100644,000000..100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@@ -1,2461 -1,0 +1,2477 @@@ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file channeltls.c + * + * \brief A concrete subclass of channel_t using or_connection_t to transfer + * cells between Tor instances. + * + * This module fills in the various function pointers in channel_t, to + * implement the channel_tls_t channels as used in Tor today. These channels + * are created from channel_tls_connect() and + * channel_tls_handle_incoming(). Each corresponds 1:1 to or_connection_t + * object, as implemented in connection_or.c. These channels transmit cells + * to the underlying or_connection_t by calling + * connection_or_write_*_cell_to_buf(), and receive cells from the underlying + * or_connection_t when connection_or_process_cells_from_inbuf() calls + * channel_tls_handle_*_cell(). + * + * Here we also implement the server (responder) side of the v3+ Tor link + * handshake, which uses CERTS and AUTHENTICATE cell to negotiate versions, + * exchange expected and observed IP and time information, and bootstrap a + * level of authentication higher than we have gotten on the raw TLS + * handshake. + * + * NOTE: Since there is currently only one type of channel, there are probably + * more than a few cases where functionality that is currently in + * channeltls.c, connection_or.c, and channel.c ought to be divided up + * differently. The right time to do this is probably whenever we introduce + * our next channel type. + **/ + +/* + * Define this so channel.h gives us things only channel_t subclasses + * should touch. + */ +#define TOR_CHANNEL_INTERNAL_ + +#define CHANNELTLS_PRIVATE + +#include "core/or/or.h" +#include "core/or/channel.h" +#include "core/or/channeltls.h" +#include "core/or/circuitmux.h" +#include "core/or/circuitmux_ewma.h" +#include "core/or/command.h" +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_or.h" +#include "feature/control/control.h" +#include "feature/client/entrynodes.h" +#include "trunnel/link_handshake.h" +#include "core/or/relay.h" +#include "feature/stats/rephist.h" +#include "feature/relay/router.h" +#include "feature/relay/routermode.h" +#include "feature/nodelist/dirlist.h" +#include "core/or/scheduler.h" +#include "feature/nodelist/torcert.h" +#include "feature/nodelist/networkstatus.h" +#include "trunnel/channelpadding_negotiation.h" +#include "core/or/channelpadding.h" + +#include "core/or/cell_st.h" +#include "core/or/cell_queue_st.h" +#include "core/or/extend_info_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/or_handshake_certs_st.h" +#include "core/or/or_handshake_state_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "core/or/var_cell_st.h" + +#include "lib/tls/tortls.h" +#include "lib/tls/x509.h" + +/** How many CELL_PADDING cells have we received, ever? */ +uint64_t stats_n_padding_cells_processed = 0; +/** How many CELL_VERSIONS cells have we received, ever? */ +uint64_t stats_n_versions_cells_processed = 0; +/** How many CELL_NETINFO cells have we received, ever? */ +uint64_t stats_n_netinfo_cells_processed = 0; +/** How many CELL_VPADDING cells have we received, ever? */ +uint64_t stats_n_vpadding_cells_processed = 0; +/** How many CELL_CERTS cells have we received, ever? */ +uint64_t stats_n_certs_cells_processed = 0; +/** How many CELL_AUTH_CHALLENGE cells have we received, ever? */ +uint64_t stats_n_auth_challenge_cells_processed = 0; +/** How many CELL_AUTHENTICATE cells have we received, ever? */ +uint64_t stats_n_authenticate_cells_processed = 0; +/** How many CELL_AUTHORIZE cells have we received, ever? */ +uint64_t stats_n_authorize_cells_processed = 0; + +/** Active listener, if any */ +static channel_listener_t *channel_tls_listener = NULL; + +/* channel_tls_t method declarations */ + +static void channel_tls_close_method(channel_t *chan); +static const char * channel_tls_describe_transport_method(channel_t *chan); +static void channel_tls_free_method(channel_t *chan); +static double channel_tls_get_overhead_estimate_method(channel_t *chan); +static int +channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out); +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out); +static const char * +channel_tls_get_remote_descr_method(channel_t *chan, int flags); +static int channel_tls_has_queued_writes_method(channel_t *chan); +static int channel_tls_is_canonical_method(channel_t *chan, int req); +static int +channel_tls_matches_extend_info_method(channel_t *chan, + extend_info_t *extend_info); +static int channel_tls_matches_target_method(channel_t *chan, + const tor_addr_t *target); +static int channel_tls_num_cells_writeable_method(channel_t *chan); +static size_t channel_tls_num_bytes_queued_method(channel_t *chan); +static int channel_tls_write_cell_method(channel_t *chan, + cell_t *cell); +static int channel_tls_write_packed_cell_method(channel_t *chan, + packed_cell_t *packed_cell); +static int channel_tls_write_var_cell_method(channel_t *chan, + var_cell_t *var_cell); + +/* channel_listener_tls_t method declarations */ + +static void channel_tls_listener_close_method(channel_listener_t *chan_l); +static const char * +channel_tls_listener_describe_transport_method(channel_listener_t *chan_l); + +/** Handle incoming cells for the handshake stuff here rather than + * passing them on up. */ + +static void channel_tls_process_versions_cell(var_cell_t *cell, + channel_tls_t *tlschan); +static void channel_tls_process_netinfo_cell(cell_t *cell, + channel_tls_t *tlschan); +static int command_allowed_before_handshake(uint8_t command); +static int enter_v3_handshake_with_cell(var_cell_t *cell, + channel_tls_t *tlschan); +static void channel_tls_process_padding_negotiate_cell(cell_t *cell, + channel_tls_t *chan); + +/** + * Do parts of channel_tls_t initialization common to channel_tls_connect() + * and channel_tls_handle_incoming(). + */ +STATIC void +channel_tls_common_init(channel_tls_t *tlschan) +{ + channel_t *chan; + + tor_assert(tlschan); + + chan = &(tlschan->base_); + channel_init(chan); + chan->magic = TLS_CHAN_MAGIC; + chan->state = CHANNEL_STATE_OPENING; + chan->close = channel_tls_close_method; + chan->describe_transport = channel_tls_describe_transport_method; + chan->free_fn = channel_tls_free_method; + chan->get_overhead_estimate = channel_tls_get_overhead_estimate_method; + chan->get_remote_addr = channel_tls_get_remote_addr_method; + chan->get_remote_descr = channel_tls_get_remote_descr_method; + chan->get_transport_name = channel_tls_get_transport_name_method; + chan->has_queued_writes = channel_tls_has_queued_writes_method; + chan->is_canonical = channel_tls_is_canonical_method; + chan->matches_extend_info = channel_tls_matches_extend_info_method; + chan->matches_target = channel_tls_matches_target_method; + chan->num_bytes_queued = channel_tls_num_bytes_queued_method; + chan->num_cells_writeable = channel_tls_num_cells_writeable_method; + chan->write_cell = channel_tls_write_cell_method; + chan->write_packed_cell = channel_tls_write_packed_cell_method; + chan->write_var_cell = channel_tls_write_var_cell_method; + + chan->cmux = circuitmux_alloc(); + /* We only have one policy for now so always set it to EWMA. */ + circuitmux_set_policy(chan->cmux, &ewma_policy); +} + +/** + * Start a new TLS channel. + * + * Launch a new OR connection to addr:port and expect to + * handshake with an OR with identity digest id_digest, and wrap + * it in a channel_tls_t. + */ +channel_t * +channel_tls_connect(const tor_addr_t *addr, uint16_t port, + const char *id_digest, + const ed25519_public_key_t *ed_id) +{ + channel_tls_t *tlschan = tor_malloc_zero(sizeof(*tlschan)); + channel_t *chan = &(tlschan->base_); + + channel_tls_common_init(tlschan); + + log_debug(LD_CHANNEL, + "In channel_tls_connect() for channel %p " + "(global id %"PRIu64 ")", + tlschan, + (chan->global_identifier)); + + if (is_local_addr(addr)) { + log_debug(LD_CHANNEL, + "Marking new outgoing channel %"PRIu64 " at %p as local", + (chan->global_identifier), chan); + channel_mark_local(chan); + } else { + log_debug(LD_CHANNEL, + "Marking new outgoing channel %"PRIu64 " at %p as remote", + (chan->global_identifier), chan); + channel_mark_remote(chan); + } + + channel_mark_outgoing(chan); + + /* Set up or_connection stuff */ + tlschan->conn = connection_or_connect(addr, port, id_digest, ed_id, tlschan); + /* connection_or_connect() will fill in tlschan->conn */ + if (!(tlschan->conn)) { + chan->reason_for_closing = CHANNEL_CLOSE_FOR_ERROR; + channel_change_state(chan, CHANNEL_STATE_ERROR); + goto err; + } + + log_debug(LD_CHANNEL, + "Got orconn %p for channel with global id %"PRIu64, + tlschan->conn, (chan->global_identifier)); + + goto done; + + err: + circuitmux_free(chan->cmux); + tor_free(tlschan); + chan = NULL; + + done: + /* If we got one, we should register it */ + if (chan) channel_register(chan); + + return chan; +} + +/** + * Return the current channel_tls_t listener. + * + * Returns the current channel listener for incoming TLS connections, or + * NULL if none has been established + */ +channel_listener_t * +channel_tls_get_listener(void) +{ + return channel_tls_listener; +} + +/** + * Start a channel_tls_t listener if necessary. + * + * Return the current channel_tls_t listener, or start one if we haven't yet, + * and return that. + */ +channel_listener_t * +channel_tls_start_listener(void) +{ + channel_listener_t *listener; + + if (!channel_tls_listener) { + listener = tor_malloc_zero(sizeof(*listener)); + channel_init_listener(listener); + listener->state = CHANNEL_LISTENER_STATE_LISTENING; + listener->close = channel_tls_listener_close_method; + listener->describe_transport = + channel_tls_listener_describe_transport_method; + + channel_tls_listener = listener; + + log_debug(LD_CHANNEL, + "Starting TLS channel listener %p with global id %"PRIu64, + listener, (listener->global_identifier)); + + channel_listener_register(listener); + } else listener = channel_tls_listener; + + return listener; +} + +/** + * Free everything on shutdown. + * + * Not much to do here, since channel_free_all() takes care of a lot, but let's + * get rid of the listener. + */ +void +channel_tls_free_all(void) +{ + channel_listener_t *old_listener = NULL; + + log_debug(LD_CHANNEL, + "Shutting down TLS channels..."); + + if (channel_tls_listener) { + /* + * When we close it, channel_tls_listener will get nulled out, so save + * a pointer so we can free it. + */ + old_listener = channel_tls_listener; + log_debug(LD_CHANNEL, + "Closing channel_tls_listener with ID %"PRIu64 + " at %p.", + (old_listener->global_identifier), + old_listener); + channel_listener_unregister(old_listener); + channel_listener_mark_for_close(old_listener); + channel_listener_free(old_listener); + tor_assert(channel_tls_listener == NULL); + } + + log_debug(LD_CHANNEL, + "Done shutting down TLS channels"); +} + +/** + * Create a new channel around an incoming or_connection_t. + */ +channel_t * +channel_tls_handle_incoming(or_connection_t *orconn) +{ + channel_tls_t *tlschan = tor_malloc_zero(sizeof(*tlschan)); + channel_t *chan = &(tlschan->base_); + + tor_assert(orconn); + tor_assert(!(orconn->chan)); + + channel_tls_common_init(tlschan); + + /* Link the channel and orconn to each other */ + tlschan->conn = orconn; + orconn->chan = tlschan; + + if (is_local_addr(&(TO_CONN(orconn)->addr))) { + log_debug(LD_CHANNEL, + "Marking new incoming channel %"PRIu64 " at %p as local", + (chan->global_identifier), chan); + channel_mark_local(chan); + } else { + log_debug(LD_CHANNEL, + "Marking new incoming channel %"PRIu64 " at %p as remote", + (chan->global_identifier), chan); + channel_mark_remote(chan); + } + + channel_mark_incoming(chan); + + /* Register it */ + channel_register(chan); + + return chan; +} + +/********* + * Casts * + ********/ + +/** + * Cast a channel_tls_t to a channel_t. + */ +channel_t * +channel_tls_to_base(channel_tls_t *tlschan) +{ + if (!tlschan) return NULL; + + return &(tlschan->base_); +} + +/** + * Cast a channel_t to a channel_tls_t, with appropriate type-checking + * asserts. + */ +channel_tls_t * +channel_tls_from_base(channel_t *chan) +{ + if (!chan) return NULL; + + tor_assert(chan->magic == TLS_CHAN_MAGIC); + + return (channel_tls_t *)(chan); +} + +/******************************************** + * Method implementations for channel_tls_t * + *******************************************/ + +/** + * Close a channel_tls_t. + * + * This implements the close method for channel_tls_t. + */ +static void +channel_tls_close_method(channel_t *chan) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + + if (tlschan->conn) connection_or_close_normally(tlschan->conn, 1); + else { + /* Weird - we'll have to change the state ourselves, I guess */ + log_info(LD_CHANNEL, + "Tried to close channel_tls_t %p with NULL conn", + tlschan); + channel_change_state(chan, CHANNEL_STATE_ERROR); + } +} + +/** + * Describe the transport for a channel_tls_t. + * + * This returns the string "TLS channel on connection <id>" to the upper + * layer. + */ +static const char * +channel_tls_describe_transport_method(channel_t *chan) +{ + static char *buf = NULL; + uint64_t id; + channel_tls_t *tlschan; + const char *rv = NULL; + + tor_assert(chan); + + tlschan = BASE_CHAN_TO_TLS(chan); + + if (tlschan->conn) { + id = TO_CONN(tlschan->conn)->global_identifier; + + if (buf) tor_free(buf); + tor_asprintf(&buf, + "TLS channel (connection %"PRIu64 ")", + (id)); + + rv = buf; + } else { + rv = "TLS channel (no connection)"; + } + + return rv; +} + +/** + * Free a channel_tls_t. + * + * This is called by the generic channel layer when freeing a channel_tls_t; + * this happens either on a channel which has already reached + * CHANNEL_STATE_CLOSED or CHANNEL_STATE_ERROR from channel_run_cleanup() or + * on shutdown from channel_free_all(). In the latter case we might still + * have an orconn active (which connection_free_all() will get to later), + * so we should null out its channel pointer now. + */ +static void +channel_tls_free_method(channel_t *chan) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + + if (tlschan->conn) { + tlschan->conn->chan = NULL; + tlschan->conn = NULL; + } +} + +/** + * Get an estimate of the average TLS overhead for the upper layer. + */ +static double +channel_tls_get_overhead_estimate_method(channel_t *chan) +{ + double overhead = 1.0; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + /* Just return 1.0f if we don't have sensible data */ + if (tlschan->conn->bytes_xmitted > 0 && + tlschan->conn->bytes_xmitted_by_tls >= + tlschan->conn->bytes_xmitted) { + overhead = ((double)(tlschan->conn->bytes_xmitted_by_tls)) / + ((double)(tlschan->conn->bytes_xmitted)); + + /* + * Never estimate more than 2.0; otherwise we get silly large estimates + * at the very start of a new TLS connection. + */ + if (overhead > 2.0) + overhead = 2.0; + } + + log_debug(LD_CHANNEL, + "Estimated overhead ratio for TLS chan %"PRIu64 " is %f", + (chan->global_identifier), overhead); + + return overhead; +} + +/** + * Get the remote address of a channel_tls_t. + * + * This implements the get_remote_addr method for channel_tls_t; copy the + * remote endpoint of the channel to addr_out and return 1 (always + * succeeds for this transport). + */ +static int +channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out) +{ + int rv = 0; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(addr_out); + + if (tlschan->conn) { + tor_addr_copy(addr_out, &(tlschan->conn->real_addr)); + rv = 1; + } else tor_addr_make_unspec(addr_out); + + return rv; +} + +/** + * Get the name of the pluggable transport used by a channel_tls_t. + * + * This implements the get_transport_name for channel_tls_t. If the + * channel uses a pluggable transport, copy its name to + * transport_out and return 0. If the channel did not use a + * pluggable transport, return -1. + */ +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(transport_out); + tor_assert(tlschan->conn); + + if (!tlschan->conn->ext_or_transport) + return -1; + + *transport_out = tor_strdup(tlschan->conn->ext_or_transport); + return 0; +} + +/** + * Get endpoint description of a channel_tls_t. + * + * This implements the get_remote_descr method for channel_tls_t; it returns + * a text description of the remote endpoint of the channel suitable for use + * in log messages. The req parameter is 0 for the canonical address or 1 for + * the actual address seen. + */ +static const char * +channel_tls_get_remote_descr_method(channel_t *chan, int flags) +{ +#define MAX_DESCR_LEN 32 + + static char buf[MAX_DESCR_LEN + 1]; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + connection_t *conn; + const char *answer = NULL; + char *addr_str; + + tor_assert(tlschan); + + if (tlschan->conn) { + conn = TO_CONN(tlschan->conn); + switch (flags) { + case 0: + /* Canonical address with port*/ + tor_snprintf(buf, MAX_DESCR_LEN + 1, + "%s:%u", conn->address, conn->port); + answer = buf; + break; + case GRD_FLAG_ORIGINAL: + /* Actual address with port */ + addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr)); + tor_snprintf(buf, MAX_DESCR_LEN + 1, + "%s:%u", addr_str, conn->port); + tor_free(addr_str); + answer = buf; + break; + case GRD_FLAG_ADDR_ONLY: + /* Canonical address, no port */ + strlcpy(buf, conn->address, sizeof(buf)); + answer = buf; + break; + case GRD_FLAG_ORIGINAL|GRD_FLAG_ADDR_ONLY: + /* Actual address, no port */ + addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr)); + strlcpy(buf, addr_str, sizeof(buf)); + tor_free(addr_str); + answer = buf; + break; + default: + /* Something's broken in channel.c */ + tor_assert_nonfatal_unreached_once(); + } + } else { + strlcpy(buf, "(No connection)", sizeof(buf)); + answer = buf; + } + + return answer; +} + +/** + * Tell the upper layer if we have queued writes. + * + * This implements the has_queued_writes method for channel_tls t_; it returns + * 1 iff we have queued writes on the outbuf of the underlying or_connection_t. + */ +static int +channel_tls_has_queued_writes_method(channel_t *chan) +{ + size_t outbuf_len; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + if (!(tlschan->conn)) { + log_info(LD_CHANNEL, + "something called has_queued_writes on a tlschan " + "(%p with ID %"PRIu64 " but no conn", + chan, (chan->global_identifier)); + } + + outbuf_len = (tlschan->conn != NULL) ? + connection_get_outbuf_len(TO_CONN(tlschan->conn)) : + 0; + + return (outbuf_len > 0); +} + +/** + * Tell the upper layer if we're canonical. + * + * This implements the is_canonical method for channel_tls_t; if req is zero, + * it returns whether this is a canonical channel, and if it is one it returns + * whether that can be relied upon. + */ +static int +channel_tls_is_canonical_method(channel_t *chan, int req) +{ + int answer = 0; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + + if (tlschan->conn) { + switch (req) { + case 0: + answer = tlschan->conn->is_canonical; + break; + case 1: + /* + * Is the is_canonical bit reliable? In protocols version 2 and up + * we get the canonical address from a NETINFO cell, but in older + * versions it might be based on an obsolete descriptor. + */ + answer = (tlschan->conn->link_proto >= 2); + break; + default: + /* This shouldn't happen; channel.c is broken if it does */ + tor_assert_nonfatal_unreached_once(); + } + } + /* else return 0 for tlschan->conn == NULL */ + + return answer; +} + +/** + * Check if we match an extend_info_t. + * + * This implements the matches_extend_info method for channel_tls_t; the upper + * layer wants to know if this channel matches an extend_info_t. + */ +static int +channel_tls_matches_extend_info_method(channel_t *chan, + extend_info_t *extend_info) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(extend_info); + + /* Never match if we have no conn */ + if (!(tlschan->conn)) { + log_info(LD_CHANNEL, + "something called matches_extend_info on a tlschan " + "(%p with ID %"PRIu64 " but no conn", + chan, (chan->global_identifier)); + return 0; + } + + return (tor_addr_eq(&(extend_info->addr), + &(TO_CONN(tlschan->conn)->addr)) && + (extend_info->port == TO_CONN(tlschan->conn)->port)); +} + +/** + * Check if we match a target address; return true iff we do. + * + * This implements the matches_target method for channel_tls t_; the upper + * layer wants to know if this channel matches a target address when extending + * a circuit. + */ +static int +channel_tls_matches_target_method(channel_t *chan, + const tor_addr_t *target) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(target); + + /* Never match if we have no conn */ + if (!(tlschan->conn)) { + log_info(LD_CHANNEL, + "something called matches_target on a tlschan " + "(%p with ID %"PRIu64 " but no conn", + chan, (chan->global_identifier)); + return 0; + } + + /* real_addr is the address this connection came from. + * base_.addr is updated by connection_or_init_conn_from_address() + * to be the address in the descriptor. It may be tempting to + * allow either address to be allowed, but if we did so, it would + * enable someone who steals a relay's keys to impersonate/MITM it + * from anywhere on the Internet! (Because they could make long-lived + * TLS connections from anywhere to all relays, and wait for them to + * be used for extends). + */ + return tor_addr_eq(&(tlschan->conn->real_addr), target); +} + +/** + * Tell the upper layer how many bytes we have queued and not yet + * sent. + */ +static size_t +channel_tls_num_bytes_queued_method(channel_t *chan) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + return connection_get_outbuf_len(TO_CONN(tlschan->conn)); +} + +/** + * Tell the upper layer how many cells we can accept to write. + * + * This implements the num_cells_writeable method for channel_tls_t; it + * returns an estimate of the number of cells we can accept with + * channel_tls_write_*_cell(). + */ +static int +channel_tls_num_cells_writeable_method(channel_t *chan) +{ + size_t outbuf_len; + ssize_t n; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + size_t cell_network_size; + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + cell_network_size = get_cell_network_size(tlschan->conn->wide_circ_ids); + outbuf_len = connection_get_outbuf_len(TO_CONN(tlschan->conn)); + /* Get the number of cells */ + n = CEIL_DIV(OR_CONN_HIGHWATER - outbuf_len, cell_network_size); + if (n < 0) n = 0; +#if SIZEOF_SIZE_T > SIZEOF_INT + if (n > INT_MAX) n = INT_MAX; +#endif + + return (int)n; +} + +/** + * Write a cell to a channel_tls_t. + * + * This implements the write_cell method for channel_tls_t; given a + * channel_tls_t and a cell_t, transmit the cell_t. + */ +static int +channel_tls_write_cell_method(channel_t *chan, cell_t *cell) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + int written = 0; + + tor_assert(tlschan); + tor_assert(cell); + + if (tlschan->conn) { + connection_or_write_cell_to_buf(cell, tlschan->conn); + ++written; + } else { + log_info(LD_CHANNEL, + "something called write_cell on a tlschan " + "(%p with ID %"PRIu64 " but no conn", + chan, (chan->global_identifier)); + } + + return written; +} + +/** + * Write a packed cell to a channel_tls_t. + * + * This implements the write_packed_cell method for channel_tls_t; given a + * channel_tls_t and a packed_cell_t, transmit the packed_cell_t. + * + * Return 0 on success or negative value on error. The caller must free the + * packed cell. + */ +static int +channel_tls_write_packed_cell_method(channel_t *chan, + packed_cell_t *packed_cell) +{ + tor_assert(chan); + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + size_t cell_network_size = get_cell_network_size(chan->wide_circ_ids); + + tor_assert(tlschan); + tor_assert(packed_cell); + + if (tlschan->conn) { + connection_buf_add(packed_cell->body, cell_network_size, + TO_CONN(tlschan->conn)); + } else { + log_info(LD_CHANNEL, + "something called write_packed_cell on a tlschan " + "(%p with ID %"PRIu64 " but no conn", + chan, (chan->global_identifier)); + return -1; + } + + return 0; +} + +/** + * Write a variable-length cell to a channel_tls_t. + * + * This implements the write_var_cell method for channel_tls_t; given a + * channel_tls_t and a var_cell_t, transmit the var_cell_t. + */ +static int +channel_tls_write_var_cell_method(channel_t *chan, var_cell_t *var_cell) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + int written = 0; + + tor_assert(tlschan); + tor_assert(var_cell); + + if (tlschan->conn) { + connection_or_write_var_cell_to_buf(var_cell, tlschan->conn); + ++written; + } else { + log_info(LD_CHANNEL, + "something called write_var_cell on a tlschan " + "(%p with ID %"PRIu64 " but no conn", + chan, (chan->global_identifier)); + } + + return written; +} + +/************************************************* + * Method implementations for channel_listener_t * + ************************************************/ + +/** + * Close a channel_listener_t. + * + * This implements the close method for channel_listener_t. + */ +static void +channel_tls_listener_close_method(channel_listener_t *chan_l) +{ + tor_assert(chan_l); + + /* + * Listeners we just go ahead and change state through to CLOSED, but + * make sure to check if they're channel_tls_listener to NULL it out. + */ + if (chan_l == channel_tls_listener) + channel_tls_listener = NULL; + + if (!(chan_l->state == CHANNEL_LISTENER_STATE_CLOSING || + chan_l->state == CHANNEL_LISTENER_STATE_CLOSED || + chan_l->state == CHANNEL_LISTENER_STATE_ERROR)) { + channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSING); + } + + if (chan_l->incoming_list) { + SMARTLIST_FOREACH_BEGIN(chan_l->incoming_list, + channel_t *, ichan) { + channel_mark_for_close(ichan); + } SMARTLIST_FOREACH_END(ichan); + + smartlist_free(chan_l->incoming_list); + chan_l->incoming_list = NULL; + } + + if (!(chan_l->state == CHANNEL_LISTENER_STATE_CLOSED || + chan_l->state == CHANNEL_LISTENER_STATE_ERROR)) { + channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSED); + } +} + +/** + * Describe the transport for a channel_listener_t. + * + * This returns the string "TLS channel (listening)" to the upper + * layer. + */ +static const char * +channel_tls_listener_describe_transport_method(channel_listener_t *chan_l) +{ + tor_assert(chan_l); + + return "TLS channel (listening)"; +} + +/******************************************************* + * Functions for handling events on an or_connection_t * + ******************************************************/ + +/** + * Handle an orconn state change. + * + * This function will be called by connection_or.c when the or_connection_t + * associated with this channel_tls_t changes state. + */ +void +channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, + or_connection_t *conn, + uint8_t old_state, + uint8_t state) +{ + channel_t *base_chan; + + tor_assert(chan); + tor_assert(conn); + tor_assert(conn->chan == chan); + tor_assert(chan->conn == conn); + /* Shut the compiler up without triggering -Wtautological-compare */ + (void)old_state; + + base_chan = TLS_CHAN_TO_BASE(chan); + + /* Make sure the base connection state makes sense - shouldn't be error + * or closed. */ + + tor_assert(CHANNEL_IS_OPENING(base_chan) || + CHANNEL_IS_OPEN(base_chan) || + CHANNEL_IS_MAINT(base_chan) || + CHANNEL_IS_CLOSING(base_chan)); + + /* Did we just go to state open? */ + if (state == OR_CONN_STATE_OPEN) { + /* + * We can go to CHANNEL_STATE_OPEN from CHANNEL_STATE_OPENING or + * CHANNEL_STATE_MAINT on this. + */ + channel_change_state_open(base_chan); + /* We might have just become writeable; check and tell the scheduler */ + if (connection_or_num_cells_writeable(conn) > 0) { + scheduler_channel_wants_writes(base_chan); + } + } else { + /* + * Not open, so from CHANNEL_STATE_OPEN we go to CHANNEL_STATE_MAINT, + * otherwise no change. + */ + if (CHANNEL_IS_OPEN(base_chan)) { + channel_change_state(base_chan, CHANNEL_STATE_MAINT); + } + } +} + +#ifdef KEEP_TIMING_STATS + +/** + * Timing states wrapper. + * + * This is a wrapper function around the actual function that processes the + * cell that just arrived on chan. Increment *time + * by the number of microseconds used by the call to *func(cell, chan). + */ +static void +channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time, + void (*func)(cell_t *, channel_tls_t *)) +{ + struct timeval start, end; + long time_passed; + + tor_gettimeofday(&start); + + (*func)(cell, chan); + + tor_gettimeofday(&end); + time_passed = tv_udiff(&start, &end) ; + + if (time_passed > 10000) { /* more than 10ms */ + log_debug(LD_OR,"That call just took %ld ms.",time_passed/1000); + } + + if (time_passed < 0) { + log_info(LD_GENERAL,"That call took us back in time!"); + time_passed = 0; + } + + *time += time_passed; +} +#endif /* defined(KEEP_TIMING_STATS) */ + +/** + * Handle an incoming cell on a channel_tls_t. + * + * This is called from connection_or.c to handle an arriving cell; it checks + * for cell types specific to the handshake for this transport protocol and + * handles them, and queues all other cells to the channel_t layer, which + * eventually will hand them off to command.c. + * + * The channel layer itself decides whether the cell should be queued or + * can be handed off immediately to the upper-layer code. It is responsible + * for copying in the case that it queues; we merely pass pointers through + * which we get from connection_or_process_cells_from_inbuf(). + */ +void +channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) +{ + channel_tls_t *chan; + int handshaking; + +#ifdef KEEP_TIMING_STATS +#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN { \ + ++num ## tp; \ + channel_tls_time_process_cell(cl, cn, & tp ## time , \ + channel_tls_process_ ## tp ## _cell); \ + } STMT_END +#else /* !(defined(KEEP_TIMING_STATS)) */ +#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn) +#endif /* defined(KEEP_TIMING_STATS) */ + + tor_assert(cell); + tor_assert(conn); + + chan = conn->chan; + + if (!chan) { + log_warn(LD_CHANNEL, + "Got a cell_t on an OR connection with no channel"); + return; + } + + handshaking = (TO_CONN(conn)->state != OR_CONN_STATE_OPEN); + + if (conn->base_.marked_for_close) + return; + + /* Reject all but VERSIONS and NETINFO when handshaking. */ + /* (VERSIONS should actually be impossible; it's variable-length.) */ + if (handshaking && cell->command != CELL_VERSIONS && + cell->command != CELL_NETINFO) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received unexpected cell command %d in chan state %s / " + "conn state %s; closing the connection.", + (int)cell->command, + channel_state_to_string(TLS_CHAN_TO_BASE(chan)->state), + conn_state_to_string(CONN_TYPE_OR, TO_CONN(conn)->state)); + connection_or_close_for_error(conn, 0); + return; + } + + if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) + or_handshake_state_record_cell(conn, conn->handshake_state, cell, 1); + + /* We note that we're on the internet whenever we read a cell. This is + * a fast operation. */ + entry_guards_note_internet_connectivity(get_guard_selection_info()); + rep_hist_padding_count_read(PADDING_TYPE_TOTAL); + + if (TLS_CHAN_TO_BASE(chan)->currently_padding) + rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL); + + switch (cell->command) { + case CELL_PADDING: + rep_hist_padding_count_read(PADDING_TYPE_CELL); + if (TLS_CHAN_TO_BASE(chan)->currently_padding) + rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL); + ++stats_n_padding_cells_processed; + /* do nothing */ + break; + case CELL_VERSIONS: + tor_fragile_assert(); + break; + case CELL_NETINFO: + ++stats_n_netinfo_cells_processed; + PROCESS_CELL(netinfo, cell, chan); + break; + case CELL_PADDING_NEGOTIATE: + ++stats_n_netinfo_cells_processed; + PROCESS_CELL(padding_negotiate, cell, chan); + break; + case CELL_CREATE: + case CELL_CREATE_FAST: + case CELL_CREATED: + case CELL_CREATED_FAST: + case CELL_RELAY: + case CELL_RELAY_EARLY: + case CELL_DESTROY: + case CELL_CREATE2: + case CELL_CREATED2: + /* + * These are all transport independent and we pass them up through the + * channel_t mechanism. They are ultimately handled in command.c. + */ + channel_process_cell(TLS_CHAN_TO_BASE(chan), cell); + break; + default: + log_fn(LOG_INFO, LD_PROTOCOL, + "Cell of unknown type (%d) received in channeltls.c. " + "Dropping.", + cell->command); + break; + } +} + +/** + * Handle an incoming variable-length cell on a channel_tls_t. + * + * Process a var_cell that was just received on conn. Keep + * internal statistics about how many of each cell we've processed so far + * this second, and the total number of microseconds it took to + * process each type of cell. All the var_cell commands are handshake- + * related and live below the channel_t layer, so no variable-length + * cells ever get delivered in the current implementation, but I've left + * the mechanism in place for future use. + * + * If we were handing them off to the upper layer, the channel_t queueing + * code would be responsible for memory management, and we'd just be passing + * pointers through from connection_or_process_cells_from_inbuf(). That + * caller always frees them after this function returns, so this function + * should never free var_cell. + */ +void +channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn) +{ + channel_tls_t *chan; + +#ifdef KEEP_TIMING_STATS + /* how many of each cell have we seen so far this second? needs better + * name. */ + static int num_versions = 0, num_certs = 0; + static time_t current_second = 0; /* from previous calls to time */ + time_t now = time(NULL); + + if (current_second == 0) current_second = now; + if (now > current_second) { /* the second has rolled over */ + /* print stats */ + log_info(LD_OR, + "At end of second: %d versions (%d ms), %d certs (%d ms)", + num_versions, versions_time / ((now - current_second) * 1000), + num_certs, certs_time / ((now - current_second) * 1000)); + + num_versions = num_certs = 0; + versions_time = certs_time = 0; + + /* remember which second it is, for next time */ + current_second = now; + } +#endif /* defined(KEEP_TIMING_STATS) */ + + tor_assert(var_cell); + tor_assert(conn); + + chan = conn->chan; + + if (!chan) { + log_warn(LD_CHANNEL, + "Got a var_cell_t on an OR connection with no channel"); + return; + } + + if (TO_CONN(conn)->marked_for_close) + return; + + switch (TO_CONN(conn)->state) { + case OR_CONN_STATE_OR_HANDSHAKING_V2: + if (var_cell->command != CELL_VERSIONS) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received a cell with command %d in unexpected " + "orconn state \"%s\" [%d], channel state \"%s\" [%d]; " + "closing the connection.", + (int)(var_cell->command), + conn_state_to_string(CONN_TYPE_OR, TO_CONN(conn)->state), + TO_CONN(conn)->state, + channel_state_to_string(TLS_CHAN_TO_BASE(chan)->state), + (int)(TLS_CHAN_TO_BASE(chan)->state)); + /* + * The code in connection_or.c will tell channel_t to close for + * error; it will go to CHANNEL_STATE_CLOSING, and then to + * CHANNEL_STATE_ERROR when conn is closed. + */ + connection_or_close_for_error(conn, 0); + return; + } + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + /* If we're using bufferevents, it's entirely possible for us to + * notice "hey, data arrived!" before we notice "hey, the handshake + * finished!" And we need to be accepting both at once to handle both + * the v2 and v3 handshakes. */ + /* But that should be happening any longer've disabled bufferevents. */ + tor_assert_nonfatal_unreached_once(); + + /* fall through */ + case OR_CONN_STATE_TLS_SERVER_RENEGOTIATING: + if (!(command_allowed_before_handshake(var_cell->command))) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received a cell with command %d in unexpected " + "orconn state \"%s\" [%d], channel state \"%s\" [%d]; " + "closing the connection.", + (int)(var_cell->command), + conn_state_to_string(CONN_TYPE_OR, TO_CONN(conn)->state), + (int)(TO_CONN(conn)->state), + channel_state_to_string(TLS_CHAN_TO_BASE(chan)->state), + (int)(TLS_CHAN_TO_BASE(chan)->state)); + /* see above comment about CHANNEL_STATE_ERROR */ + connection_or_close_for_error(conn, 0); + return; + } else { + if (enter_v3_handshake_with_cell(var_cell, chan) < 0) + return; + } + break; + case OR_CONN_STATE_OR_HANDSHAKING_V3: + if (var_cell->command != CELL_AUTHENTICATE) + or_handshake_state_record_var_cell(conn, conn->handshake_state, + var_cell, 1); + break; /* Everything is allowed */ + case OR_CONN_STATE_OPEN: + if (conn->link_proto < 3) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received a variable-length cell with command %d in orconn " + "state %s [%d], channel state %s [%d] with link protocol %d; " + "ignoring it.", + (int)(var_cell->command), + conn_state_to_string(CONN_TYPE_OR, TO_CONN(conn)->state), + (int)(TO_CONN(conn)->state), + channel_state_to_string(TLS_CHAN_TO_BASE(chan)->state), + (int)(TLS_CHAN_TO_BASE(chan)->state), + (int)(conn->link_proto)); + return; + } + break; + default: + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received var-length cell with command %d in unexpected " + "orconn state \"%s\" [%d], channel state \"%s\" [%d]; " + "ignoring it.", + (int)(var_cell->command), + conn_state_to_string(CONN_TYPE_OR, TO_CONN(conn)->state), + (int)(TO_CONN(conn)->state), + channel_state_to_string(TLS_CHAN_TO_BASE(chan)->state), + (int)(TLS_CHAN_TO_BASE(chan)->state)); + return; + } + + /* We note that we're on the internet whenever we read a cell. This is + * a fast operation. */ + entry_guards_note_internet_connectivity(get_guard_selection_info()); + + /* Now handle the cell */ + + switch (var_cell->command) { + case CELL_VERSIONS: + ++stats_n_versions_cells_processed; + PROCESS_CELL(versions, var_cell, chan); + break; + case CELL_VPADDING: + ++stats_n_vpadding_cells_processed; + /* Do nothing */ + break; + case CELL_CERTS: + ++stats_n_certs_cells_processed; + PROCESS_CELL(certs, var_cell, chan); + break; + case CELL_AUTH_CHALLENGE: + ++stats_n_auth_challenge_cells_processed; + PROCESS_CELL(auth_challenge, var_cell, chan); + break; + case CELL_AUTHENTICATE: + ++stats_n_authenticate_cells_processed; + PROCESS_CELL(authenticate, var_cell, chan); + break; + case CELL_AUTHORIZE: + ++stats_n_authorize_cells_processed; + /* Ignored so far. */ + break; + default: + log_fn(LOG_INFO, LD_PROTOCOL, + "Variable-length cell of unknown type (%d) received.", + (int)(var_cell->command)); + break; + } +} + +/** + * Update channel marks after connection_or.c has changed an address. + * + * This is called from connection_or_init_conn_from_address() after the + * connection's _base.addr or real_addr fields have potentially been changed + * so we can recalculate the local mark. Notably, this happens when incoming + * connections are reverse-proxied and we only learn the real address of the + * remote router by looking it up in the consensus after we finish the + * handshake and know an authenticated identity digest. + */ +void +channel_tls_update_marks(or_connection_t *conn) +{ + channel_t *chan = NULL; + + tor_assert(conn); + tor_assert(conn->chan); + + chan = TLS_CHAN_TO_BASE(conn->chan); + + if (is_local_addr(&(TO_CONN(conn)->addr))) { + if (!channel_is_local(chan)) { + log_debug(LD_CHANNEL, + "Marking channel %"PRIu64 " at %p as local", + (chan->global_identifier), chan); + channel_mark_local(chan); + } + } else { + if (channel_is_local(chan)) { + log_debug(LD_CHANNEL, + "Marking channel %"PRIu64 " at %p as remote", + (chan->global_identifier), chan); + channel_mark_remote(chan); + } + } +} + +/** + * Check if this cell type is allowed before the handshake is finished. + * + * Return true if command is a cell command that's allowed to start a + * V3 handshake. + */ +static int +command_allowed_before_handshake(uint8_t command) +{ + switch (command) { + case CELL_VERSIONS: + case CELL_VPADDING: + case CELL_AUTHORIZE: + return 1; + default: + return 0; + } +} + +/** + * Start a V3 handshake on an incoming connection. + * + * Called when we as a server receive an appropriate cell while waiting + * either for a cell or a TLS handshake. Set the connection's state to + * "handshaking_v3', initializes the or_handshake_state field as needed, + * and add the cell to the hash of incoming cells.) + */ +static int +enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *chan) +{ + int started_here = 0; + + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + + started_here = connection_or_nonopen_was_started_here(chan->conn); + + tor_assert(TO_CONN(chan->conn)->state == OR_CONN_STATE_TLS_HANDSHAKING || + TO_CONN(chan->conn)->state == + OR_CONN_STATE_TLS_SERVER_RENEGOTIATING); + + if (started_here) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a cell while TLS-handshaking, not in " + "OR_HANDSHAKING_V3, on a connection we originated."); + } + connection_or_block_renegotiation(chan->conn); + chan->conn->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3; + if (connection_init_or_handshake_state(chan->conn, started_here) < 0) { + connection_or_close_for_error(chan->conn, 0); + return -1; + } + or_handshake_state_record_var_cell(chan->conn, + chan->conn->handshake_state, cell, 1); + return 0; +} + +/** + * Process a 'versions' cell. + * + * This function is called to handle an incoming VERSIONS cell; the current + * link protocol version must be 0 to indicate that no version has yet been + * negotiated. We compare the versions in the cell to the list of versions + * we support, pick the highest version we have in common, and continue the + * negotiation from there. + */ +static void +channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) +{ + int highest_supported_version = 0; + int started_here = 0; + + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + + if ((cell->payload_len % 2) == 1) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a VERSION cell with odd payload length %d; " + "closing connection.",cell->payload_len); + connection_or_close_for_error(chan->conn, 0); + return; + } + + started_here = connection_or_nonopen_was_started_here(chan->conn); + + if (chan->conn->link_proto != 0 || + (chan->conn->handshake_state && + chan->conn->handshake_state->received_versions)) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a VERSIONS cell on a connection with its version " + "already set to %d; dropping", + (int)(chan->conn->link_proto)); + return; + } + switch (chan->conn->base_.state) + { + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + case OR_CONN_STATE_TLS_SERVER_RENEGOTIATING: + default: + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "VERSIONS cell while in unexpected state"); + return; + } + + tor_assert(chan->conn->handshake_state); + + { + int i; + const uint8_t *cp = cell->payload; + for (i = 0; i < cell->payload_len / 2; ++i, cp += 2) { + uint16_t v = ntohs(get_uint16(cp)); + if (is_or_protocol_version_known(v) && v > highest_supported_version) + highest_supported_version = v; + } + } + if (!highest_supported_version) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Couldn't find a version in common between my version list and the " + "list in the VERSIONS cell; closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; + } else if (highest_supported_version == 1) { + /* Negotiating version 1 makes no sense, since version 1 has no VERSIONS + * cells. */ + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Used version negotiation protocol to negotiate a v1 connection. " + "That's crazily non-compliant. Closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; + } else if (highest_supported_version < 3 && + chan->conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Negotiated link protocol 2 or lower after doing a v3 TLS " + "handshake. Closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; + } else if (highest_supported_version != 2 && + chan->conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V2) { + /* XXXX This should eventually be a log_protocol_warn */ + log_fn(LOG_WARN, LD_OR, + "Negotiated link with non-2 protocol after doing a v2 TLS " + "handshake with %s. Closing connection.", + fmt_addr(&chan->conn->base_.addr)); + connection_or_close_for_error(chan->conn, 0); + return; + } + + rep_hist_note_negotiated_link_proto(highest_supported_version, started_here); + + chan->conn->link_proto = highest_supported_version; + chan->conn->handshake_state->received_versions = 1; + + if (chan->conn->link_proto == 2) { + log_info(LD_OR, + "Negotiated version %d with %s:%d; sending NETINFO.", + highest_supported_version, + safe_str_client(chan->conn->base_.address), + chan->conn->base_.port); + + if (connection_or_send_netinfo(chan->conn) < 0) { + connection_or_close_for_error(chan->conn, 0); + return; + } + } else { + const int send_versions = !started_here; + /* If we want to authenticate, send a CERTS cell */ + const int send_certs = !started_here || public_server_mode(get_options()); + /* If we're a host that got a connection, ask for authentication. */ + const int send_chall = !started_here; + /* If our certs cell will authenticate us, we can send a netinfo cell + * right now. */ + const int send_netinfo = !started_here; + const int send_any = + send_versions || send_certs || send_chall || send_netinfo; + tor_assert(chan->conn->link_proto >= 3); + + log_info(LD_OR, + "Negotiated version %d with %s:%d; %s%s%s%s%s", + highest_supported_version, + safe_str_client(chan->conn->base_.address), + chan->conn->base_.port, + send_any ? "Sending cells:" : "Waiting for CERTS cell", + send_versions ? " VERSIONS" : "", + send_certs ? " CERTS" : "", + send_chall ? " AUTH_CHALLENGE" : "", + send_netinfo ? " NETINFO" : ""); + +#ifdef DISABLE_V3_LINKPROTO_SERVERSIDE + if (1) { + connection_or_close_normally(chan->conn, 1); + return; + } +#endif /* defined(DISABLE_V3_LINKPROTO_SERVERSIDE) */ + + if (send_versions) { + if (connection_or_send_versions(chan->conn, 1) < 0) { + log_warn(LD_OR, "Couldn't send versions cell"); + connection_or_close_for_error(chan->conn, 0); + return; + } + } + + /* We set this after sending the versions cell. */ + /*XXXXX symbolic const.*/ + TLS_CHAN_TO_BASE(chan)->wide_circ_ids = + chan->conn->link_proto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS; + chan->conn->wide_circ_ids = TLS_CHAN_TO_BASE(chan)->wide_circ_ids; + + TLS_CHAN_TO_BASE(chan)->padding_enabled = + chan->conn->link_proto >= MIN_LINK_PROTO_FOR_CHANNEL_PADDING; + + if (send_certs) { + if (connection_or_send_certs_cell(chan->conn) < 0) { + log_warn(LD_OR, "Couldn't send certs cell"); + connection_or_close_for_error(chan->conn, 0); + return; + } + } + if (send_chall) { + if (connection_or_send_auth_challenge_cell(chan->conn) < 0) { + log_warn(LD_OR, "Couldn't send auth_challenge cell"); + connection_or_close_for_error(chan->conn, 0); + return; + } + } + if (send_netinfo) { + if (connection_or_send_netinfo(chan->conn) < 0) { + log_warn(LD_OR, "Couldn't send netinfo cell"); + connection_or_close_for_error(chan->conn, 0); + return; + } + } + } +} + +/** + * Process a 'padding_negotiate' cell. + * + * This function is called to handle an incoming PADDING_NEGOTIATE cell; + * enable or disable padding accordingly, and read and act on its timeout + * value contents. + */ +static void +channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) +{ + channelpadding_negotiate_t *negotiation; + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + + if (chan->conn->link_proto < MIN_LINK_PROTO_FOR_CHANNEL_PADDING) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a PADDING_NEGOTIATE cell on v%d connection; dropping.", + chan->conn->link_proto); + return; + } + + if (channelpadding_negotiate_parse(&negotiation, cell->payload, + CELL_PAYLOAD_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received malformed PADDING_NEGOTIATE cell on v%d connection; " + "dropping.", chan->conn->link_proto); + + return; + } + + channelpadding_update_padding_for_channel(TLS_CHAN_TO_BASE(chan), + negotiation); + + channelpadding_negotiate_free(negotiation); +} + +/** - * Process a 'netinfo' cell. ++ * Helper: compute the absolute value of a time_t. ++ * ++ * (we need this because labs() doesn't always work for time_t, since ++ * long can be shorter than time_t.) ++ */ ++static inline time_t ++time_abs(time_t val) ++{ ++ return (val < 0) ? -val : val; ++} ++ ++/** ++ * Process a 'netinfo' cell + * + * This function is called to handle an incoming NETINFO cell; read and act + * on its contents, and set the connection state to "open". + */ +static void +channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) +{ + time_t timestamp; + uint8_t my_addr_type; + uint8_t my_addr_len; + const uint8_t *my_addr_ptr; + const uint8_t *cp, *end; + uint8_t n_other_addrs; + time_t now = time(NULL); + const routerinfo_t *me = router_get_my_routerinfo(); + - long apparent_skew = 0; ++ time_t apparent_skew = 0; + tor_addr_t my_apparent_addr = TOR_ADDR_NULL; + int started_here = 0; + const char *identity_digest = NULL; + + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + + if (chan->conn->link_proto < 2) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a NETINFO cell on %s connection; dropping.", + chan->conn->link_proto == 0 ? "non-versioned" : "a v1"); + return; + } + if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V2 && + chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a NETINFO cell on non-handshaking connection; dropping."); + return; + } + tor_assert(chan->conn->handshake_state && + chan->conn->handshake_state->received_versions); + started_here = connection_or_nonopen_was_started_here(chan->conn); + identity_digest = chan->conn->identity_digest; + + if (chan->conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) { + tor_assert(chan->conn->link_proto >= 3); + if (started_here) { + if (!(chan->conn->handshake_state->authenticated)) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Got a NETINFO cell from server, " + "but no authentication. Closing the connection."); + connection_or_close_for_error(chan->conn, 0); + return; + } + } else { + /* we're the server. If the client never authenticated, we have + some housekeeping to do.*/ + if (!(chan->conn->handshake_state->authenticated)) { + tor_assert(tor_digest_is_zero( + (const char*)(chan->conn->handshake_state-> + authenticated_rsa_peer_id))); + tor_assert(tor_mem_is_zero( + (const char*)(chan->conn->handshake_state-> + authenticated_ed25519_peer_id.pubkey), 32)); + /* If the client never authenticated, it's a tor client or bridge + * relay, and we must not use it for EXTEND requests (nor could we, as + * there are no authenticated peer IDs) */ + channel_mark_client(TLS_CHAN_TO_BASE(chan)); + channel_set_circid_type(TLS_CHAN_TO_BASE(chan), NULL, + chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS); + + connection_or_init_conn_from_address(chan->conn, + &(chan->conn->base_.addr), + chan->conn->base_.port, + /* zero, checked above */ + (const char*)(chan->conn->handshake_state-> + authenticated_rsa_peer_id), + NULL, /* Ed25519 ID: Also checked as zero */ + 0); + } + } + } + + /* Decode the cell. */ + timestamp = ntohl(get_uint32(cell->payload)); - if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { ++ const time_t sent_versions_at = ++ chan->conn->handshake_state->sent_versions_at; ++ if (now > sent_versions_at && (now - sent_versions_at) < 180) { ++ /* If we have gotten the NETINFO cell reasonably soon after having ++ * sent our VERSIONS cell, maybe we can learn skew information from it. */ + apparent_skew = now - timestamp; + } + + my_addr_type = (uint8_t) cell->payload[4]; + my_addr_len = (uint8_t) cell->payload[5]; + my_addr_ptr = (uint8_t*) cell->payload + 6; + end = cell->payload + CELL_PAYLOAD_SIZE; + cp = cell->payload + 6 + my_addr_len; + + /* We used to check: + * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { + * + * This is actually never going to happen, since my_addr_len is at most 255, + * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ + + if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { + tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + + if (!get_options()->BridgeRelay && me && + get_uint32(my_addr_ptr) == htonl(me->addr)) { + TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; + } + + } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { + tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); + + if (!get_options()->BridgeRelay && me && + !tor_addr_is_null(&me->ipv6_addr) && + tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { + TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; + } + } + + n_other_addrs = (uint8_t) *cp++; + while (n_other_addrs && cp < end-2) { + /* Consider all the other addresses; if any matches, this connection is + * "canonical." */ + tor_addr_t addr; + const uint8_t *next = + decode_address_from_payload(&addr, cp, (int)(end-cp)); + if (next == NULL) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Bad address in netinfo cell; closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; + } + /* A relay can connect from anywhere and be canonical, so + * long as it tells you from where it came. This may sound a bit + * concerning... but that's what "canonical" means: that the + * address is one that the relay itself has claimed. The relay + * might be doing something funny, but nobody else is doing a MITM + * on the relay's TCP. + */ + if (tor_addr_eq(&addr, &(chan->conn->real_addr))) { + connection_or_set_canonical(chan->conn, 1); + break; + } + cp = next; + --n_other_addrs; + } + + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && + channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { + const char *descr = + TLS_CHAN_TO_BASE(chan)->get_remote_descr(TLS_CHAN_TO_BASE(chan), 0); + log_info(LD_OR, + "We made a connection to a relay at %s (fp=%s) but we think " + "they will not consider this connection canonical. They " + "think we are at %s, but we think its %s.", + safe_str(descr), + safe_str(hex_str(identity_digest, DIGEST_LEN)), + safe_str(tor_addr_is_null(&my_apparent_addr) ? + "<none>" : fmt_and_decorate_addr(&my_apparent_addr)), + safe_str(fmt_addr32(me->addr))); + } + + /* Act on apparent skew. */ + /** Warn when we get a netinfo skew with at least this value. */ +#define NETINFO_NOTICE_SKEW 3600 - if (labs(apparent_skew) > NETINFO_NOTICE_SKEW && ++ if (time_abs(apparent_skew) > NETINFO_NOTICE_SKEW && + (started_here || + connection_or_digest_is_known_relay(chan->conn->identity_digest))) { + int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest); + clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL, + "NETINFO cell", "OR"); + } + + /* XXX maybe act on my_apparent_addr, if the source is sufficiently + * trustworthy. */ + + if (! chan->conn->handshake_state->sent_netinfo) { + /* If we were prepared to authenticate, but we never got an AUTH_CHALLENGE + * cell, then we would not previously have sent a NETINFO cell. Do so + * now. */ + if (connection_or_send_netinfo(chan->conn) < 0) { + connection_or_close_for_error(chan->conn, 0); + return; + } + } + + if (connection_or_set_state_open(chan->conn) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Got good NETINFO cell from %s:%d; but " + "was unable to make the OR connection become open.", + safe_str_client(chan->conn->base_.address), + chan->conn->base_.port); + connection_or_close_for_error(chan->conn, 0); + } else { + log_info(LD_OR, + "Got good NETINFO cell from %s:%d; OR connection is now " + "open, using protocol version %d. Its ID digest is %s. " + "Our address is apparently %s.", + safe_str_client(chan->conn->base_.address), + chan->conn->base_.port, + (int)(chan->conn->link_proto), + hex_str(identity_digest, DIGEST_LEN), + tor_addr_is_null(&my_apparent_addr) ? + "<none>" : + safe_str_client(fmt_and_decorate_addr(&my_apparent_addr))); + } + assert_connection_ok(TO_CONN(chan->conn),time(NULL)); +} + +/** Types of certificates that we know how to parse from CERTS cells. Each + * type corresponds to a different encoding format. */ +typedef enum cert_encoding_t { + CERT_ENCODING_UNKNOWN, /**< We don't recognize this. */ + CERT_ENCODING_X509, /**< It's an RSA key, signed with RSA, encoded in x509. + * (Actually, it might not be RSA. We test that later.) */ + CERT_ENCODING_ED25519, /**< It's something signed with an Ed25519 key, + * encoded asa a tor_cert_t.*/ + CERT_ENCODING_RSA_CROSSCERT, /**< It's an Ed key signed with an RSA key. */ +} cert_encoding_t; + +/** + * Given one of the certificate type codes used in a CERTS cell, + * return the corresponding cert_encoding_t that we should use to parse + * the certificate. + */ +static cert_encoding_t +certs_cell_typenum_to_cert_type(int typenum) +{ + switch (typenum) { + case CERTTYPE_RSA1024_ID_LINK: + case CERTTYPE_RSA1024_ID_ID: + case CERTTYPE_RSA1024_ID_AUTH: + return CERT_ENCODING_X509; + case CERTTYPE_ED_ID_SIGN: + case CERTTYPE_ED_SIGN_LINK: + case CERTTYPE_ED_SIGN_AUTH: + return CERT_ENCODING_ED25519; + case CERTTYPE_RSA1024_ID_EDID: + return CERT_ENCODING_RSA_CROSSCERT; + default: + return CERT_ENCODING_UNKNOWN; + } +} + +/** + * Process a CERTS cell from a channel. + * + * This function is called to process an incoming CERTS cell on a + * channel_tls_t: + * + * If the other side should not have sent us a CERTS cell, or the cell is + * malformed, or it is supposed to authenticate the TLS key but it doesn't, + * then mark the connection. + * + * If the cell has a good cert chain and we're doing a v3 handshake, then + * store the certificates in or_handshake_state. If this is the client side + * of the connection, we then authenticate the server or mark the connection. + * If it's the server side, wait for an AUTHENTICATE cell. + */ +STATIC void +channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) +{ +#define MAX_CERT_TYPE_WANTED CERTTYPE_RSA1024_ID_EDID + /* These arrays will be sparse, since a cert type can be at most one + * of ed/x509 */ + tor_x509_cert_t *x509_certs[MAX_CERT_TYPE_WANTED + 1]; + tor_cert_t *ed_certs[MAX_CERT_TYPE_WANTED + 1]; + uint8_t *rsa_ed_cc_cert = NULL; + size_t rsa_ed_cc_cert_len = 0; + + int n_certs, i; + certs_cell_t *cc = NULL; + + int send_netinfo = 0, started_here = 0; + + memset(x509_certs, 0, sizeof(x509_certs)); + memset(ed_certs, 0, sizeof(ed_certs)); + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + +#define ERR(s) \ + do { \ + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, \ + "Received a bad CERTS cell from %s:%d: %s", \ + safe_str(chan->conn->base_.address), \ + chan->conn->base_.port, (s)); \ + connection_or_close_for_error(chan->conn, 0); \ + goto err; \ + } while (0) + + /* Can't use connection_or_nonopen_was_started_here(); its conn->tls + * check looks like it breaks + * test_link_handshake_recv_certs_ok_server(). */ + started_here = chan->conn->handshake_state->started_here; + + if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3) + ERR("We're not doing a v3 handshake!"); + if (chan->conn->link_proto < 3) + ERR("We're not using link protocol >= 3"); + if (chan->conn->handshake_state->received_certs_cell) + ERR("We already got one"); + if (chan->conn->handshake_state->authenticated) { + /* Should be unreachable, but let's make sure. */ + ERR("We're already authenticated!"); + } + if (cell->payload_len < 1) + ERR("It had no body"); + if (cell->circ_id) + ERR("It had a nonzero circuit ID"); + + if (certs_cell_parse(&cc, cell->payload, cell->payload_len) < 0) + ERR("It couldn't be parsed."); + + n_certs = cc->n_certs; + + for (i = 0; i < n_certs; ++i) { + certs_cell_cert_t *c = certs_cell_get_certs(cc, i); + + uint16_t cert_type = c->cert_type; + uint16_t cert_len = c->cert_len; + uint8_t *cert_body = certs_cell_cert_getarray_body(c); + + if (cert_type > MAX_CERT_TYPE_WANTED) + continue; + const cert_encoding_t ct = certs_cell_typenum_to_cert_type(cert_type); + switch (ct) { + default: + case CERT_ENCODING_UNKNOWN: + break; + case CERT_ENCODING_X509: { + tor_x509_cert_t *x509_cert = tor_x509_cert_decode(cert_body, cert_len); + if (!x509_cert) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received undecodable certificate in CERTS cell from %s:%d", + safe_str(chan->conn->base_.address), + chan->conn->base_.port); + } else { + if (x509_certs[cert_type]) { + tor_x509_cert_free(x509_cert); + ERR("Duplicate x509 certificate"); + } else { + x509_certs[cert_type] = x509_cert; + } + } + break; + } + case CERT_ENCODING_ED25519: { + tor_cert_t *ed_cert = tor_cert_parse(cert_body, cert_len); + if (!ed_cert) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received undecodable Ed certificate " + "in CERTS cell from %s:%d", + safe_str(chan->conn->base_.address), + chan->conn->base_.port); + } else { + if (ed_certs[cert_type]) { + tor_cert_free(ed_cert); + ERR("Duplicate Ed25519 certificate"); + } else { + ed_certs[cert_type] = ed_cert; + } + } + break; + } + + case CERT_ENCODING_RSA_CROSSCERT: { + if (rsa_ed_cc_cert) { + ERR("Duplicate RSA->Ed25519 crosscert"); + } else { + rsa_ed_cc_cert = tor_memdup(cert_body, cert_len); + rsa_ed_cc_cert_len = cert_len; + } + break; + } + } + } + + /* Move the certificates we (might) want into the handshake_state->certs + * structure. */ + tor_x509_cert_t *id_cert = x509_certs[CERTTYPE_RSA1024_ID_ID]; + tor_x509_cert_t *auth_cert = x509_certs[CERTTYPE_RSA1024_ID_AUTH]; + tor_x509_cert_t *link_cert = x509_certs[CERTTYPE_RSA1024_ID_LINK]; + chan->conn->handshake_state->certs->auth_cert = auth_cert; + chan->conn->handshake_state->certs->link_cert = link_cert; + chan->conn->handshake_state->certs->id_cert = id_cert; + x509_certs[CERTTYPE_RSA1024_ID_ID] = + x509_certs[CERTTYPE_RSA1024_ID_AUTH] = + x509_certs[CERTTYPE_RSA1024_ID_LINK] = NULL; + + tor_cert_t *ed_id_sign = ed_certs[CERTTYPE_ED_ID_SIGN]; + tor_cert_t *ed_sign_link = ed_certs[CERTTYPE_ED_SIGN_LINK]; + tor_cert_t *ed_sign_auth = ed_certs[CERTTYPE_ED_SIGN_AUTH]; + chan->conn->handshake_state->certs->ed_id_sign = ed_id_sign; + chan->conn->handshake_state->certs->ed_sign_link = ed_sign_link; + chan->conn->handshake_state->certs->ed_sign_auth = ed_sign_auth; + ed_certs[CERTTYPE_ED_ID_SIGN] = + ed_certs[CERTTYPE_ED_SIGN_LINK] = + ed_certs[CERTTYPE_ED_SIGN_AUTH] = NULL; + + chan->conn->handshake_state->certs->ed_rsa_crosscert = rsa_ed_cc_cert; + chan->conn->handshake_state->certs->ed_rsa_crosscert_len = + rsa_ed_cc_cert_len; + rsa_ed_cc_cert = NULL; + + int severity; + /* Note that this warns more loudly about time and validity if we were + * _trying_ to connect to an authority, not necessarily if we _did_ connect + * to one. */ + if (started_here && + router_digest_is_trusted_dir(TLS_CHAN_TO_BASE(chan)->identity_digest)) + severity = LOG_WARN; + else + severity = LOG_PROTOCOL_WARN; + + const ed25519_public_key_t *checked_ed_id = NULL; + const common_digests_t *checked_rsa_id = NULL; + or_handshake_certs_check_both(severity, + chan->conn->handshake_state->certs, + chan->conn->tls, + time(NULL), + &checked_ed_id, + &checked_rsa_id); + + if (!checked_rsa_id) + ERR("Invalid certificate chain!"); + + if (started_here) { + /* No more information is needed. */ + + chan->conn->handshake_state->authenticated = 1; + chan->conn->handshake_state->authenticated_rsa = 1; + { + const common_digests_t *id_digests = checked_rsa_id; + crypto_pk_t *identity_rcvd; + if (!id_digests) + ERR("Couldn't compute digests for key in ID cert"); + + identity_rcvd = tor_tls_cert_get_key(id_cert); + if (!identity_rcvd) { + ERR("Couldn't get RSA key from ID cert."); + } + memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id, + id_digests->d[DIGEST_SHA1], DIGEST_LEN); + channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd, + chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS); + crypto_pk_free(identity_rcvd); + } + + if (checked_ed_id) { + chan->conn->handshake_state->authenticated_ed25519 = 1; + memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id, + checked_ed_id, sizeof(ed25519_public_key_t)); + } + + log_debug(LD_HANDSHAKE, "calling client_learned_peer_id from " + "process_certs_cell"); + + if (connection_or_client_learned_peer_id(chan->conn, + chan->conn->handshake_state->authenticated_rsa_peer_id, + checked_ed_id) < 0) + ERR("Problem setting or checking peer id"); + + log_info(LD_HANDSHAKE, + "Got some good certificates from %s:%d: Authenticated it with " + "RSA%s", + safe_str(chan->conn->base_.address), chan->conn->base_.port, + checked_ed_id ? " and Ed25519" : ""); + + if (!public_server_mode(get_options())) { + /* If we initiated the connection and we are not a public server, we + * aren't planning to authenticate at all. At this point we know who we + * are talking to, so we can just send a netinfo now. */ + send_netinfo = 1; + } + } else { + /* We can't call it authenticated till we see an AUTHENTICATE cell. */ + log_info(LD_OR, + "Got some good RSA%s certificates from %s:%d. " + "Waiting for AUTHENTICATE.", + checked_ed_id ? " and Ed25519" : "", + safe_str(chan->conn->base_.address), + chan->conn->base_.port); + /* XXXX check more stuff? */ + } + + chan->conn->handshake_state->received_certs_cell = 1; + + if (send_netinfo) { + if (connection_or_send_netinfo(chan->conn) < 0) { + log_warn(LD_OR, "Couldn't send netinfo cell"); + connection_or_close_for_error(chan->conn, 0); + goto err; + } + } + + err: + for (unsigned u = 0; u < ARRAY_LENGTH(x509_certs); ++u) { + tor_x509_cert_free(x509_certs[u]); + } + for (unsigned u = 0; u < ARRAY_LENGTH(ed_certs); ++u) { + tor_cert_free(ed_certs[u]); + } + tor_free(rsa_ed_cc_cert); + certs_cell_free(cc); +#undef ERR +} + +/** + * Process an AUTH_CHALLENGE cell from a channel_tls_t. + * + * This function is called to handle an incoming AUTH_CHALLENGE cell on a + * channel_tls_t; if we weren't supposed to get one (for example, because we're + * not the originator of the channel), or it's ill-formed, or we aren't doing + * a v3 handshake, mark the channel. If the cell is well-formed but we don't + * want to authenticate, just drop it. If the cell is well-formed *and* we + * want to authenticate, send an AUTHENTICATE cell and then a NETINFO cell. + */ +STATIC void +channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) +{ + int n_types, i, use_type = -1; + auth_challenge_cell_t *ac = NULL; + + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + +#define ERR(s) \ + do { \ + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, \ + "Received a bad AUTH_CHALLENGE cell from %s:%d: %s", \ + safe_str(chan->conn->base_.address), \ + chan->conn->base_.port, (s)); \ + connection_or_close_for_error(chan->conn, 0); \ + goto done; \ + } while (0) + + if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3) + ERR("We're not currently doing a v3 handshake"); + if (chan->conn->link_proto < 3) + ERR("We're not using link protocol >= 3"); + if (!(chan->conn->handshake_state->started_here)) + ERR("We didn't originate this connection"); + if (chan->conn->handshake_state->received_auth_challenge) + ERR("We already received one"); + if (!(chan->conn->handshake_state->received_certs_cell)) + ERR("We haven't gotten a CERTS cell yet"); + if (cell->circ_id) + ERR("It had a nonzero circuit ID"); + + if (auth_challenge_cell_parse(&ac, cell->payload, cell->payload_len) < 0) + ERR("It was not well-formed."); + + n_types = ac->n_methods; + + /* Now see if there is an authentication type we can use */ + for (i = 0; i < n_types; ++i) { + uint16_t authtype = auth_challenge_cell_get_methods(ac, i); + if (authchallenge_type_is_supported(authtype)) { + if (use_type == -1 || + authchallenge_type_is_better(authtype, use_type)) { + use_type = authtype; + } + } + } + + chan->conn->handshake_state->received_auth_challenge = 1; + + if (! public_server_mode(get_options())) { + /* If we're not a public server then we don't want to authenticate on a + connection we originated, and we already sent a NETINFO cell when we + got the CERTS cell. We have nothing more to do. */ + goto done; + } + + if (use_type >= 0) { + log_info(LD_OR, + "Got an AUTH_CHALLENGE cell from %s:%d: Sending " + "authentication type %d", + safe_str(chan->conn->base_.address), + chan->conn->base_.port, + use_type); + + if (connection_or_send_authenticate_cell(chan->conn, use_type) < 0) { + log_warn(LD_OR, + "Couldn't send authenticate cell"); + connection_or_close_for_error(chan->conn, 0); + goto done; + } + } else { + log_info(LD_OR, + "Got an AUTH_CHALLENGE cell from %s:%d, but we don't " + "know any of its authentication types. Not authenticating.", + safe_str(chan->conn->base_.address), + chan->conn->base_.port); + } + + if (connection_or_send_netinfo(chan->conn) < 0) { + log_warn(LD_OR, "Couldn't send netinfo cell"); + connection_or_close_for_error(chan->conn, 0); + goto done; + } + + done: + auth_challenge_cell_free(ac); + +#undef ERR +} + +/** + * Process an AUTHENTICATE cell from a channel_tls_t. + * + * If it's ill-formed or we weren't supposed to get one or we're not doing a + * v3 handshake, then mark the connection. If it does not authenticate the + * other side of the connection successfully (because it isn't signed right, + * we didn't get a CERTS cell, etc) mark the connection. Otherwise, accept + * the identity of the router on the other side of the connection. + */ +STATIC void +channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan) +{ + var_cell_t *expected_cell = NULL; + const uint8_t *auth; + int authlen; + int authtype; + int bodylen; + + tor_assert(cell); + tor_assert(chan); + tor_assert(chan->conn); + +#define ERR(s) \ + do { \ + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, \ + "Received a bad AUTHENTICATE cell from %s:%d: %s", \ + safe_str(chan->conn->base_.address), \ + chan->conn->base_.port, (s)); \ + connection_or_close_for_error(chan->conn, 0); \ + var_cell_free(expected_cell); \ + return; \ + } while (0) + + if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3) + ERR("We're not doing a v3 handshake"); + if (chan->conn->link_proto < 3) + ERR("We're not using link protocol >= 3"); + if (chan->conn->handshake_state->started_here) + ERR("We originated this connection"); + if (chan->conn->handshake_state->received_authenticate) + ERR("We already got one!"); + if (chan->conn->handshake_state->authenticated) { + /* Should be impossible given other checks */ + ERR("The peer is already authenticated"); + } + if (!(chan->conn->handshake_state->received_certs_cell)) + ERR("We never got a certs cell"); + if (chan->conn->handshake_state->certs->id_cert == NULL) + ERR("We never got an identity certificate"); + if (cell->payload_len < 4) + ERR("Cell was way too short"); + + auth = cell->payload; + { + uint16_t type = ntohs(get_uint16(auth)); + uint16_t len = ntohs(get_uint16(auth+2)); + if (4 + len > cell->payload_len) + ERR("Authenticator was truncated"); + + if (! authchallenge_type_is_supported(type)) + ERR("Authenticator type was not recognized"); + authtype = type; + + auth += 4; + authlen = len; + } + + if (authlen < V3_AUTH_BODY_LEN + 1) + ERR("Authenticator was too short"); + + expected_cell = connection_or_compute_authenticate_cell_body( + chan->conn, authtype, NULL, NULL, 1); + if (! expected_cell) + ERR("Couldn't compute expected AUTHENTICATE cell body"); + + int sig_is_rsa; + if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET || + authtype == AUTHTYPE_RSA_SHA256_RFC5705) { + bodylen = V3_AUTH_BODY_LEN; + sig_is_rsa = 1; + } else { + tor_assert(authtype == AUTHTYPE_ED25519_SHA256_RFC5705); + /* Our earlier check had better have made sure we had room + * for an ed25519 sig (inadvertently) */ + tor_assert(V3_AUTH_BODY_LEN > ED25519_SIG_LEN); + bodylen = authlen - ED25519_SIG_LEN; + sig_is_rsa = 0; + } + if (expected_cell->payload_len != bodylen+4) { + ERR("Expected AUTHENTICATE cell body len not as expected."); + } + + /* Length of random part. */ + if (BUG(bodylen < 24)) { + // LCOV_EXCL_START + ERR("Bodylen is somehow less than 24, which should really be impossible"); + // LCOV_EXCL_STOP + } + + if (tor_memneq(expected_cell->payload+4, auth, bodylen-24)) + ERR("Some field in the AUTHENTICATE cell body was not as expected"); + + if (sig_is_rsa) { + if (chan->conn->handshake_state->certs->ed_id_sign != NULL) + ERR("RSA-signed AUTHENTICATE response provided with an ED25519 cert"); + + if (chan->conn->handshake_state->certs->auth_cert == NULL) + ERR("We never got an RSA authentication certificate"); + + crypto_pk_t *pk = tor_tls_cert_get_key( + chan->conn->handshake_state->certs->auth_cert); + char d[DIGEST256_LEN]; + char *signed_data; + size_t keysize; + int signed_len; + + if (! pk) { + ERR("Couldn't get RSA key from AUTH cert."); + } + crypto_digest256(d, (char*)auth, V3_AUTH_BODY_LEN, DIGEST_SHA256); + + keysize = crypto_pk_keysize(pk); + signed_data = tor_malloc(keysize); + signed_len = crypto_pk_public_checksig(pk, signed_data, keysize, + (char*)auth + V3_AUTH_BODY_LEN, + authlen - V3_AUTH_BODY_LEN); + crypto_pk_free(pk); + if (signed_len < 0) { + tor_free(signed_data); + ERR("RSA signature wasn't valid"); + } + if (signed_len < DIGEST256_LEN) { + tor_free(signed_data); + ERR("Not enough data was signed"); + } + /* Note that we deliberately allow *more* than DIGEST256_LEN bytes here, + * in case they're later used to hold a SHA3 digest or something. */ + if (tor_memneq(signed_data, d, DIGEST256_LEN)) { + tor_free(signed_data); + ERR("Signature did not match data to be signed."); + } + tor_free(signed_data); + } else { + if (chan->conn->handshake_state->certs->ed_id_sign == NULL) + ERR("We never got an Ed25519 identity certificate."); + if (chan->conn->handshake_state->certs->ed_sign_auth == NULL) + ERR("We never got an Ed25519 authentication certificate."); + + const ed25519_public_key_t *authkey = + &chan->conn->handshake_state->certs->ed_sign_auth->signed_key; + ed25519_signature_t sig; + tor_assert(authlen > ED25519_SIG_LEN); + memcpy(&sig.sig, auth + authlen - ED25519_SIG_LEN, ED25519_SIG_LEN); + if (ed25519_checksig(&sig, auth, authlen - ED25519_SIG_LEN, authkey)<0) { + ERR("Ed25519 signature wasn't valid."); + } + } + + /* Okay, we are authenticated. */ + chan->conn->handshake_state->received_authenticate = 1; + chan->conn->handshake_state->authenticated = 1; + chan->conn->handshake_state->authenticated_rsa = 1; + chan->conn->handshake_state->digest_received_data = 0; + { + tor_x509_cert_t *id_cert = chan->conn->handshake_state->certs->id_cert; + crypto_pk_t *identity_rcvd = tor_tls_cert_get_key(id_cert); + const common_digests_t *id_digests = tor_x509_cert_get_id_digests(id_cert); + const ed25519_public_key_t *ed_identity_received = NULL; + + if (! sig_is_rsa) { + chan->conn->handshake_state->authenticated_ed25519 = 1; + ed_identity_received = + &chan->conn->handshake_state->certs->ed_id_sign->signing_key; + memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id, + ed_identity_received, sizeof(ed25519_public_key_t)); + } + + /* This must exist; we checked key type when reading the cert. */ + tor_assert(id_digests); + + memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id, + id_digests->d[DIGEST_SHA1], DIGEST_LEN); + + channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd, + chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS); + crypto_pk_free(identity_rcvd); + + log_debug(LD_HANDSHAKE, + "Calling connection_or_init_conn_from_address for %s " + " from %s, with%s ed25519 id.", + safe_str(chan->conn->base_.address), + __func__, + ed_identity_received ? "" : "out"); + + connection_or_init_conn_from_address(chan->conn, + &(chan->conn->base_.addr), + chan->conn->base_.port, + (const char*)(chan->conn->handshake_state-> + authenticated_rsa_peer_id), + ed_identity_received, + 0); + + log_debug(LD_HANDSHAKE, + "Got an AUTHENTICATE cell from %s:%d, type %d: Looks good.", + safe_str(chan->conn->base_.address), + chan->conn->base_.port, + authtype); + } + + var_cell_free(expected_cell); + +#undef ERR +}

1 0

[tor/master] Restore proper behavior of netinfo skew check
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit cd6cb453720a5300d00d7996c5b3a03f054cd293 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 09:15:42 2019 -0400 Restore proper behavior of netinfo skew check My previous fix removed a comparison, which would have caused us to warn about every skew instead of skews of over an hour. --- src/or/channeltls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/channeltls.c b/src/or/channeltls.c index ea69792f1..d44f71913 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1721,7 +1721,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) /* Act on apparent skew. */ /** Warn when we get a netinfo skew with at least this value. */ #define NETINFO_NOTICE_SKEW 3600 - if (time_abs(apparent_skew) && + if (time_abs(apparent_skew) > NETINFO_NOTICE_SKEW && router_get_by_id_digest(chan->conn->identity_digest)) { int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest); clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL,

1 0

[tor/master] Merge branch 'ticket31343_029' into ticket31343_035
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit 70d0b97ee67381d901ed350fc7fdb1e98e1975c6 Merge: bc9492a93 cd6cb4537 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 09:17:23 2019 -0400 Merge branch 'ticket31343_029' into ticket31343_035

1 0

[tor/master] Fix another time_t/long warning for 31343.
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit 878f4409015f741c7075d0ccf3da794a6f313302 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 09:38:03 2019 -0400 Fix another time_t/long warning for 31343. --- src/or/routerlist.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/or/routerlist.c b/src/or/routerlist.c index f73ec9baa..f3b298006 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -5443,7 +5443,7 @@ int router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) { time_t r1pub, r2pub; - long time_difference; + time_t time_difference; tor_assert(r1 && r2); /* r1 should be the one that was published first. */ @@ -5506,7 +5506,9 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) * give or take some slop? */ r1pub = r1->cache_info.published_on; r2pub = r2->cache_info.published_on; - time_difference = labs(r2->uptime - (r1->uptime + (r2pub - r1pub))); + time_difference = r2->uptime - (r1->uptime + (r2pub - r1pub)); + if (time_difference < 0) + time_difference = - time_difference; if (time_difference > ROUTER_ALLOW_UPTIME_DRIFT && time_difference > r1->uptime * .05 && time_difference > r2->uptime * .05) @@ -5816,4 +5818,3 @@ refresh_all_country_info(void) nodelist_refresh_countries(); } -

1 0

[tor/master] Merge branch 'ticket31343_029' into ticket31343_035
by nickm＠torproject.org 08 Aug '19

08 Aug '19

commit a4400a77a5db2fc60db258857a23a11ec741e426 Merge: 70d0b97ee 878f44090 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 8 09:39:48 2019 -0400 Merge branch 'ticket31343_029' into ticket31343_035 src/feature/nodelist/routerlist.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --cc src/feature/nodelist/routerlist.c index 4a99427cd,000000000..456f930aa mode 100644,000000..100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@@ -1,3232 -1,0 +1,3234 @@@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerlist.c + * \brief Code to + * maintain and access the global list of routerinfos for known + * servers. + * + * A "routerinfo_t" object represents a single self-signed router + * descriptor, as generated by a Tor relay in order to tell the rest of + * the world about its keys, address, and capabilities. An + * "extrainfo_t" object represents an adjunct "extra-info" object, + * certified by a corresponding router descriptor, reporting more + * information about the relay that nearly all users will not need. + * + * Most users will not use router descriptors for most relays. Instead, + * they use the information in microdescriptors and in the consensus + * networkstatus. + * + * Right now, routerinfo_t objects are used in these ways: + * <ul> + * <li>By clients, in order to learn about bridge keys and capabilities. + * (Bridges aren't listed in the consensus networkstatus, so they + * can't have microdescriptors.) + * <li>By relays, since relays want more information about other relays + * than they can learn from microdescriptors. (TODO: Is this still true?) + * <li>By authorities, which receive them and use them to generate the + * consensus and the microdescriptors. + * <li>By all directory caches, which download them in case somebody + * else wants them. + * </ul> + * + * Routerinfos are mostly created by parsing them from a string, in + * routerparse.c. We store them to disk on receiving them, and + * periodically discard the ones we don't need. On restarting, we + * re-read them from disk. (This also applies to extrainfo documents, if + * we are configured to fetch them.) + * + * In order to keep our list of routerinfos up-to-date, we periodically + * check whether there are any listed in the latest consensus (or in the + * votes from other authorities, if we are an authority) that we don't + * have. (This also applies to extrainfo documents, if we are + * configured to fetch them.) + * + * Almost nothing in Tor should use a routerinfo_t to refer directly to + * a relay; instead, almost everything should use node_t (implemented in + * nodelist.c), which provides a common interface to routerinfo_t, + * routerstatus_t, and microdescriptor_t. + * + * + * + * This module also has some of the functions used for choosing random + * nodes according to different rules and weights. Historically, they + * were all in this module. Now, they are spread across this module, + * nodelist.c, and networkstatus.c. (TODO: Fix that.) + **/ + +#define ROUTERLIST_PRIVATE +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/or/policies.h" +#include "feature/client/bridges.h" +#include "feature/control/control.h" +#include "feature/dirauth/authmode.h" +#include "feature/dirauth/process_descs.h" +#include "feature/dirauth/reachability.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/dircommon/directory.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/describe.h" +#include "feature/nodelist/dirlist.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerinfo.h" +#include "feature/nodelist/routerlist.h" +#include "feature/dirparse/routerparse.h" +#include "feature/nodelist/routerset.h" +#include "feature/nodelist/torcert.h" +#include "feature/relay/routermode.h" +#include "feature/stats/rephist.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include "feature/dircommon/dir_connection_st.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/document_signature_st.h" +#include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/networkstatus_voter_info_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/routerlist_st.h" +#include "feature/nodelist/vote_routerstatus_st.h" + +#include "lib/crypt_ops/digestset.h" + +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif + +// #define DEBUG_ROUTERLIST + +/****************************************************************************/ + +/* Typed wrappers for different digestmap types; used to avoid type + * confusion. */ + +DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t) +DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t) +DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t) +#define SDMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \ + valvar) +#define RIMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar) +#define EIMAP_FOREACH(map, keyvar, valvar) \ + DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar) +#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn)) +#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn)) +#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn)) + +/* static function prototypes */ +static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); +static const char *signed_descriptor_get_body_impl( + const signed_descriptor_t *desc, + int with_annotations); +static void launch_dummy_descriptor_download_as_needed(time_t now, + const or_options_t *options); + +/****************************************************************************/ + +/** Global list of all of the routers that we know about. */ +static routerlist_t *routerlist = NULL; + +/** List of strings for nicknames we've already warned about and that are + * still unknown / unavailable. */ +static smartlist_t *warned_nicknames = NULL; + +/** The last time we tried to download any routerdesc, or 0 for "never". We + * use this to rate-limit download attempts when the number of routerdescs to + * download is low. */ +static time_t last_descriptor_download_attempted = 0; + +/* Router descriptor storage. + * + * Routerdescs are stored in a big file, named "cached-descriptors". As new + * routerdescs arrive, we append them to a journal file named + * "cached-descriptors.new". + * + * From time to time, we replace "cached-descriptors" with a new file + * containing only the live, non-superseded descriptors, and clear + * cached-routers.new. + * + * On startup, we read both files. + */ + +/** Helper: return 1 iff the router log is so big we want to rebuild the + * store. */ +static int +router_should_rebuild_store(desc_store_t *store) +{ + if (store->store_len > (1<<16)) + return (store->journal_len > store->store_len / 2 || + store->bytes_dropped > store->store_len / 2); + else + return store->journal_len > (1<<15); +} + +/** Return the desc_store_t in rl that should be used to store + * sd. */ +static inline desc_store_t * +desc_get_store(routerlist_t *rl, const signed_descriptor_t *sd) +{ + if (sd->is_extrainfo) + return &rl->extrainfo_store; + else + return &rl->desc_store; +} + +/** Add the signed_descriptor_t in desc to the router + * journal; change its saved_location to SAVED_IN_JOURNAL and set its + * offset appropriately. */ +static int +signed_desc_append_to_journal(signed_descriptor_t *desc, + desc_store_t *store) +{ + char *fname = get_cachedir_fname_suffix(store->fname_base, ".new"); + const char *body = signed_descriptor_get_body_impl(desc,1); + size_t len = desc->signed_descriptor_len + desc->annotations_len; + + if (append_bytes_to_file(fname, body, len, 1)) { + log_warn(LD_FS, "Unable to store router descriptor"); + tor_free(fname); + return -1; + } + desc->saved_location = SAVED_IN_JOURNAL; + tor_free(fname); + + desc->saved_offset = store->journal_len; + store->journal_len += len; + + return 0; +} + +/** Sorting helper: return <0, 0, or >0 depending on whether the + * signed_descriptor_t* in *a is older, the same age as, or newer than + * the signed_descriptor_t* in *b. */ +static int +compare_signed_descriptors_by_age_(const void **_a, const void **_b) +{ + const signed_descriptor_t *r1 = *_a, *r2 = *_b; + return (int)(r1->published_on - r2->published_on); +} + +#define RRS_FORCE 1 +#define RRS_DONT_REMOVE_OLD 2 + +/** If the journal of store is too long, or if RRS_FORCE is set in + * flags, then atomically replace the saved router store with the + * routers currently in our routerlist, and clear the journal. Unless + * RRS_DONT_REMOVE_OLD is set in flags, delete expired routers before + * rebuilding the store. Return 0 on success, -1 on failure. + */ +static int +router_rebuild_store(int flags, desc_store_t *store) +{ + smartlist_t *chunk_list = NULL; + char *fname = NULL, *fname_tmp = NULL; + int r = -1; + off_t offset = 0; + smartlist_t *signed_descriptors = NULL; + int nocache=0; + size_t total_expected_len = 0; + int had_any; + int force = flags & RRS_FORCE; + + if (!force && !router_should_rebuild_store(store)) { + r = 0; + goto done; + } + if (!routerlist) { + r = 0; + goto done; + } + + if (store->type == EXTRAINFO_STORE) + had_any = !eimap_isempty(routerlist->extra_info_map); + else + had_any = (smartlist_len(routerlist->routers)+ + smartlist_len(routerlist->old_routers))>0; + + /* Don't save deadweight. */ + if (!(flags & RRS_DONT_REMOVE_OLD)) + routerlist_remove_old_routers(); + + log_info(LD_DIR, "Rebuilding %s cache", store->description); + + fname = get_cachedir_fname(store->fname_base); + fname_tmp = get_cachedir_fname_suffix(store->fname_base, ".tmp"); + + chunk_list = smartlist_new(); + + /* We sort the routers by age to enhance locality on disk. */ + signed_descriptors = smartlist_new(); + if (store->type == EXTRAINFO_STORE) { + eimap_iter_t *iter; + for (iter = eimap_iter_init(routerlist->extra_info_map); + !eimap_iter_done(iter); + iter = eimap_iter_next(routerlist->extra_info_map, iter)) { + const char *key; + extrainfo_t *ei; + eimap_iter_get(iter, &key, &ei); + smartlist_add(signed_descriptors, &ei->cache_info); + } + } else { + SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, + smartlist_add(signed_descriptors, sd)); + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri, + smartlist_add(signed_descriptors, &ri->cache_info)); + } + + smartlist_sort(signed_descriptors, compare_signed_descriptors_by_age_); + + /* Now, add the appropriate members to chunk_list */ + SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) { + sized_chunk_t *c; + const char *body = signed_descriptor_get_body_impl(sd, 1); + if (!body) { + log_warn(LD_BUG, "No descriptor available for router."); + goto done; + } + if (sd->do_not_cache) { + ++nocache; + continue; + } + c = tor_malloc(sizeof(sized_chunk_t)); + c->bytes = body; + c->len = sd->signed_descriptor_len + sd->annotations_len; + total_expected_len += c->len; + smartlist_add(chunk_list, c); + } SMARTLIST_FOREACH_END(sd); + + if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) { + log_warn(LD_FS, "Error writing router store to disk."); + goto done; + } + + /* Our mmap is now invalid. */ + if (store->mmap) { + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Unable to munmap route store in %s", fname); + } + } + + if (replace_file(fname_tmp, fname)<0) { + log_warn(LD_FS, "Error replacing old router store: %s", strerror(errno)); + goto done; + } + + errno = 0; + store->mmap = tor_mmap_file(fname); + if (! store->mmap) { + if (errno == ERANGE) { + /* empty store.*/ + if (total_expected_len) { + log_warn(LD_FS, "We wrote some bytes to a new descriptor file at '%s'," + " but when we went to mmap it, it was empty!", fname); + } else if (had_any) { + log_info(LD_FS, "We just removed every descriptor in '%s'. This is " + "okay if we're just starting up after a long time. " + "Otherwise, it's a bug.", fname); + } + } else { + log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname); + } + } + + log_info(LD_DIR, "Reconstructing pointers into cache"); + + offset = 0; + SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) { + if (sd->do_not_cache) + continue; + sd->saved_location = SAVED_IN_CACHE; + if (store->mmap) { + tor_free(sd->signed_descriptor_body); // sets it to null + sd->saved_offset = offset; + } + offset += sd->signed_descriptor_len + sd->annotations_len; + signed_descriptor_get_body(sd); /* reconstruct and assert */ + } SMARTLIST_FOREACH_END(sd); + + tor_free(fname); + fname = get_cachedir_fname_suffix(store->fname_base, ".new"); + write_str_to_file(fname, "", 1); + + r = 0; + store->store_len = (size_t) offset; + store->journal_len = 0; + store->bytes_dropped = 0; + done: + smartlist_free(signed_descriptors); + tor_free(fname); + tor_free(fname_tmp); + if (chunk_list) { + SMARTLIST_FOREACH(chunk_list, sized_chunk_t *, c, tor_free(c)); + smartlist_free(chunk_list); + } + + return r; +} + +/** Helper: Reload a cache file and its associated journal, setting metadata + * appropriately. If extrainfo is true, reload the extrainfo store; + * else reload the router descriptor store. */ +static int +router_reload_router_list_impl(desc_store_t *store) +{ + char *fname = NULL, *contents = NULL; + struct stat st; + int extrainfo = (store->type == EXTRAINFO_STORE); + store->journal_len = store->store_len = 0; + + fname = get_cachedir_fname(store->fname_base); + + if (store->mmap) { + /* get rid of it first */ + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Failed to munmap %s", fname); + tor_free(fname); + return -1; + } + } + + store->mmap = tor_mmap_file(fname); + if (store->mmap) { + store->store_len = store->mmap->size; + if (extrainfo) + router_load_extrainfo_from_string(store->mmap->data, + store->mmap->data+store->mmap->size, + SAVED_IN_CACHE, NULL, 0); + else + router_load_routers_from_string(store->mmap->data, + store->mmap->data+store->mmap->size, + SAVED_IN_CACHE, NULL, 0, NULL); + } + + tor_free(fname); + fname = get_cachedir_fname_suffix(store->fname_base, ".new"); + /* don't load empty files - we wouldn't get any data, even if we tried */ + if (file_status(fname) == FN_FILE) + contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); + if (contents) { + if (extrainfo) + router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL, + NULL, 0); + else + router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL, + NULL, 0, NULL); + store->journal_len = (size_t) st.st_size; + tor_free(contents); + } + + tor_free(fname); + + if (store->journal_len) { + /* Always clear the journal on startup.*/ + router_rebuild_store(RRS_FORCE, store); + } else if (!extrainfo) { + /* Don't cache expired routers. (This is in an else because + * router_rebuild_store() also calls remove_old_routers().) */ + routerlist_remove_old_routers(); + } + + return 0; +} + +/** Load all cached router descriptors and extra-info documents from the + * store. Return 0 on success and -1 on failure. + */ +int +router_reload_router_list(void) +{ + routerlist_t *rl = router_get_routerlist(); + if (router_reload_router_list_impl(&rl->desc_store)) + return -1; + if (router_reload_router_list_impl(&rl->extrainfo_store)) + return -1; + return 0; +} + +/* When iterating through the routerlist, can OR address/port preference + * and reachability checks be skipped? + */ +int +router_skip_or_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or()); +} + +/* When iterating through the routerlist, can Dir address/port preference + * and reachability checks be skipped? + */ +int +router_skip_dir_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir()); +} + +/** Return true iff r1 and r2 have the same address and OR port. */ +int +routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2) +{ + return r1->addr == r2->addr && r1->or_port == r2->or_port && + tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) && + r1->ipv6_orport == r2->ipv6_orport; +} + +/** Add every suitable node from our nodelist to sl, so that + * we can pick a node for a circuit. + */ +void +router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime, + int need_capacity, int need_guard, + int need_desc, int pref_addr, + int direct_conn) +{ + const int check_reach = !router_skip_or_reachability(get_options(), + pref_addr); + /* XXXX MOVE */ + SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { + if (!node->is_running || !node->is_valid) + continue; + if (need_desc && !node_has_preferred_descriptor(node, direct_conn)) + continue; + if (node->ri && node->ri->purpose != ROUTER_PURPOSE_GENERAL) + continue; + if (node_is_unreliable(node, need_uptime, need_capacity, need_guard)) + continue; + /* Don't choose nodes if we are certain they can't do EXTEND2 cells */ + if (node->rs && !routerstatus_version_supports_extend2_cells(node->rs, 1)) + continue; + /* Don't choose nodes if we are certain they can't do ntor. */ + if ((node->ri || node->md) && !node_has_curve25519_onion_key(node)) + continue; + /* Choose a node with an OR address that matches the firewall rules */ + if (direct_conn && check_reach && + !fascist_firewall_allows_node(node, + FIREWALL_OR_CONNECTION, + pref_addr)) + continue; + + smartlist_add(sl, (void *)node); + } SMARTLIST_FOREACH_END(node); +} + +/** Look through the routerlist until we find a router that has my key. + Return it. */ +const routerinfo_t * +routerlist_find_my_routerinfo(void) +{ + if (!routerlist) + return NULL; + + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, router, + { + if (router_is_me(router)) + return router; + }); + return NULL; +} + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity. */ +uint32_t +router_get_advertised_bandwidth(const routerinfo_t *router) +{ + if (router->bandwidthcapacity < router->bandwidthrate) + return router->bandwidthcapacity; + return router->bandwidthrate; +} + +/** Do not weight any declared bandwidth more than this much when picking + * routers by bandwidth. */ +#define DEFAULT_MAX_BELIEVABLE_BANDWIDTH 10000000 /* 10 MB/sec */ + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity, capped by max-believe-bw. */ +uint32_t +router_get_advertised_bandwidth_capped(const routerinfo_t *router) +{ + uint32_t result = router->bandwidthcapacity; + if (result > router->bandwidthrate) + result = router->bandwidthrate; + if (result > DEFAULT_MAX_BELIEVABLE_BANDWIDTH) + result = DEFAULT_MAX_BELIEVABLE_BANDWIDTH; + return result; +} + +/** Helper: given an extended nickname in hexdigest try to decode it. + * Return 0 on success, -1 on failure. Store the result into the + * DIGEST_LEN-byte buffer at digest_out, the single character at + * nickname_qualifier_char_out, and the MAXNICKNAME_LEN+1-byte buffer + * at nickname_out. + * + * The recognized format is: + * HexName = Dollar? HexDigest NamePart? + * Dollar = '?' + * HexDigest = HexChar*20 + * HexChar = 'a'..'f' | 'A'..'F' | '0'..'9' + * NamePart = QualChar Name + * QualChar = '=' | '~' + * Name = NameChar*(1..MAX_NICKNAME_LEN) + * NameChar = Any ASCII alphanumeric character + */ +int +hex_digest_nickname_decode(const char *hexdigest, + char *digest_out, + char *nickname_qualifier_char_out, + char *nickname_out) +{ + size_t len; + + tor_assert(hexdigest); + if (hexdigest[0] == '$') + ++hexdigest; + + len = strlen(hexdigest); + if (len < HEX_DIGEST_LEN) { + return -1; + } else if (len > HEX_DIGEST_LEN && (hexdigest[HEX_DIGEST_LEN] == '=' || + hexdigest[HEX_DIGEST_LEN] == '~') && + len <= HEX_DIGEST_LEN+1+MAX_NICKNAME_LEN) { + *nickname_qualifier_char_out = hexdigest[HEX_DIGEST_LEN]; + strlcpy(nickname_out, hexdigest+HEX_DIGEST_LEN+1 , MAX_NICKNAME_LEN+1); + } else if (len == HEX_DIGEST_LEN) { + ; + } else { + return -1; + } + + if (base16_decode(digest_out, DIGEST_LEN, + hexdigest, HEX_DIGEST_LEN) != DIGEST_LEN) + return -1; + return 0; +} + +/** Helper: Return true iff the identity_digest and nickname + * combination of a router, encoded in hexadecimal, matches hexdigest + * (which is optionally prefixed with a single dollar sign). Return false if + * hexdigest is malformed, or it doesn't match. */ +int +hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest, + const char *nickname) +{ + char digest[DIGEST_LEN]; + char nn_char='\0'; + char nn_buf[MAX_NICKNAME_LEN+1]; + + if (hex_digest_nickname_decode(hexdigest, digest, &nn_char, nn_buf) == -1) + return 0; + + if (nn_char == '=') { + return 0; + } + + if (nn_char == '~') { + if (!nickname) // XXX This seems wrong. -NM + return 0; + if (strcasecmp(nn_buf, nickname)) + return 0; + } + + return tor_memeq(digest, identity_digest, DIGEST_LEN); +} + +/** If hexdigest is correctly formed, base16_decode it into + * digest, which must have DIGEST_LEN space in it. + * Return 0 on success, -1 on failure. + */ +int +hexdigest_to_digest(const char *hexdigest, char *digest) +{ + if (hexdigest[0]=='$') + ++hexdigest; + if (strlen(hexdigest) < HEX_DIGEST_LEN || + base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN) + return -1; + return 0; +} + +/** As router_get_by_id_digest,but return a pointer that you're allowed to + * modify */ +routerinfo_t * +router_get_mutable_by_digest(const char *digest) +{ + tor_assert(digest); + + if (!routerlist) return NULL; + + // routerlist_assert_ok(routerlist); + + return rimap_get(routerlist->identity_map, digest); +} + +/** Return the router in our routerlist whose 20-byte key digest + * is digest. Return NULL if no such router is known. */ +const routerinfo_t * +router_get_by_id_digest(const char *digest) +{ + return router_get_mutable_by_digest(digest); +} + +/** Return the router in our routerlist whose 20-byte descriptor + * is digest. Return NULL if no such router is known. */ +signed_descriptor_t * +router_get_by_descriptor_digest(const char *digest) +{ + tor_assert(digest); + + if (!routerlist) return NULL; + + return sdmap_get(routerlist->desc_digest_map, digest); +} + +/** Return the signed descriptor for the router in our routerlist whose + * 20-byte extra-info digest is digest. Return NULL if no such router + * is known. */ +MOCK_IMPL(signed_descriptor_t *, +router_get_by_extrainfo_digest,(const char *digest)) +{ + tor_assert(digest); + + if (!routerlist) return NULL; + + return sdmap_get(routerlist->desc_by_eid_map, digest); +} + +/** Return the signed descriptor for the extrainfo_t in our routerlist whose + * extra-info-digest is digest. Return NULL if no such extra-info + * document is known. */ +MOCK_IMPL(signed_descriptor_t *, +extrainfo_get_by_descriptor_digest,(const char *digest)) +{ + extrainfo_t *ei; + tor_assert(digest); + if (!routerlist) return NULL; + ei = eimap_get(routerlist->extra_info_map, digest); + return ei ? &ei->cache_info : NULL; +} + +/** Return a pointer to the signed textual representation of a descriptor. + * The returned string is not guaranteed to be NUL-terminated: the string's + * length will be in desc-\>signed_descriptor_len. + * + * If with_annotations is set, the returned string will include + * the annotations + * (if any) preceding the descriptor. This will increase the length of the + * string by desc-\>annotations_len. + * + * The caller must not free the string returned. + */ +static const char * +signed_descriptor_get_body_impl(const signed_descriptor_t *desc, + int with_annotations) +{ + const char *r = NULL; + size_t len = desc->signed_descriptor_len; + off_t offset = desc->saved_offset; + if (with_annotations) + len += desc->annotations_len; + else + offset += desc->annotations_len; + + tor_assert(len > 32); + if (desc->saved_location == SAVED_IN_CACHE && routerlist) { + desc_store_t *store = desc_get_store(router_get_routerlist(), desc); + if (store && store->mmap) { + tor_assert(desc->saved_offset + len <= store->mmap->size); + r = store->mmap->data + offset; + } else if (store) { + log_err(LD_DIR, "We couldn't read a descriptor that is supposedly " + "mmaped in our cache. Is another process running in our data " + "directory? Exiting."); + exit(1); // XXXX bad exit: should recover. + } + } + if (!r) /* no mmap, or not in cache. */ + r = desc->signed_descriptor_body + + (with_annotations ? 0 : desc->annotations_len); + + tor_assert(r); + if (!with_annotations) { + if (fast_memcmp("router ", r, 7) && fast_memcmp("extra-info ", r, 11)) { + char *cp = tor_strndup(r, 64); + log_err(LD_DIR, "descriptor at %p begins with unexpected string %s. " + "Is another process running in our data directory? Exiting.", + desc, escaped(cp)); + exit(1); // XXXX bad exit: should recover. + } + } + + return r; +} + +/** Return a pointer to the signed textual representation of a descriptor. + * The returned string is not guaranteed to be NUL-terminated: the string's + * length will be in desc-\>signed_descriptor_len. + * + * The caller must not free the string returned. + */ +const char * +signed_descriptor_get_body(const signed_descriptor_t *desc) +{ + return signed_descriptor_get_body_impl(desc, 0); +} + +/** As signed_descriptor_get_body(), but points to the beginning of the + * annotations section rather than the beginning of the descriptor. */ +const char * +signed_descriptor_get_annotations(const signed_descriptor_t *desc) +{ + return signed_descriptor_get_body_impl(desc, 1); +} + +/** Return the current list of all known routers. */ +routerlist_t * +router_get_routerlist(void) +{ + if (PREDICT_UNLIKELY(!routerlist)) { + routerlist = tor_malloc_zero(sizeof(routerlist_t)); + routerlist->routers = smartlist_new(); + routerlist->old_routers = smartlist_new(); + routerlist->identity_map = rimap_new(); + routerlist->desc_digest_map = sdmap_new(); + routerlist->desc_by_eid_map = sdmap_new(); + routerlist->extra_info_map = eimap_new(); + + routerlist->desc_store.fname_base = "cached-descriptors"; + routerlist->extrainfo_store.fname_base = "cached-extrainfo"; + + routerlist->desc_store.type = ROUTER_STORE; + routerlist->extrainfo_store.type = EXTRAINFO_STORE; + + routerlist->desc_store.description = "router descriptors"; + routerlist->extrainfo_store.description = "extra-info documents"; + } + return routerlist; +} + +/** Free all storage held by router. */ +void +routerinfo_free_(routerinfo_t *router) +{ + if (!router) + return; + + tor_free(router->cache_info.signed_descriptor_body); + tor_free(router->nickname); + tor_free(router->platform); + tor_free(router->protocol_list); + tor_free(router->contact_info); + if (router->onion_pkey) + tor_free(router->onion_pkey); + tor_free(router->onion_curve25519_pkey); + if (router->identity_pkey) + crypto_pk_free(router->identity_pkey); + tor_cert_free(router->cache_info.signing_key_cert); + if (router->declared_family) { + SMARTLIST_FOREACH(router->declared_family, char *, s, tor_free(s)); + smartlist_free(router->declared_family); + } + addr_policy_list_free(router->exit_policy); + short_policy_free(router->ipv6_exit_policy); + + memset(router, 77, sizeof(routerinfo_t)); + + tor_free(router); +} + +/** Release all storage held by extrainfo */ +void +extrainfo_free_(extrainfo_t *extrainfo) +{ + if (!extrainfo) + return; + tor_cert_free(extrainfo->cache_info.signing_key_cert); + tor_free(extrainfo->cache_info.signed_descriptor_body); + tor_free(extrainfo->pending_sig); + + memset(extrainfo, 88, sizeof(extrainfo_t)); /* debug bad memory usage */ + tor_free(extrainfo); +} + +#define signed_descriptor_free(val) \ + FREE_AND_NULL(signed_descriptor_t, signed_descriptor_free_, (val)) + +/** Release storage held by sd. */ +static void +signed_descriptor_free_(signed_descriptor_t *sd) +{ + if (!sd) + return; + + tor_free(sd->signed_descriptor_body); + tor_cert_free(sd->signing_key_cert); + + memset(sd, 99, sizeof(signed_descriptor_t)); /* Debug bad mem usage */ + tor_free(sd); +} + +/** Reset the given signed descriptor sd by freeing the allocated + * memory inside the object and by zeroing its content. */ +static void +signed_descriptor_reset(signed_descriptor_t *sd) +{ + tor_assert(sd); + tor_free(sd->signed_descriptor_body); + tor_cert_free(sd->signing_key_cert); + memset(sd, 0, sizeof(*sd)); +} + +/** Copy src into dest, and steal all references inside src so that when + * we free src, we don't mess up dest. */ +static void +signed_descriptor_move(signed_descriptor_t *dest, + signed_descriptor_t *src) +{ + tor_assert(dest != src); + /* Cleanup destination object before overwriting it.*/ + signed_descriptor_reset(dest); + memcpy(dest, src, sizeof(signed_descriptor_t)); + src->signed_descriptor_body = NULL; + src->signing_key_cert = NULL; + dest->routerlist_index = -1; +} + +/** Extract a signed_descriptor_t from a general routerinfo, and free the + * routerinfo. + */ +static signed_descriptor_t * +signed_descriptor_from_routerinfo(routerinfo_t *ri) +{ + signed_descriptor_t *sd; + tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL); + sd = tor_malloc_zero(sizeof(signed_descriptor_t)); + signed_descriptor_move(sd, &ri->cache_info); + routerinfo_free(ri); + return sd; +} + +/** Helper: free the storage held by the extrainfo_t in e. */ +static void +extrainfo_free_void(void *e) +{ + extrainfo_free_(e); +} + +/** Free all storage held by a routerlist rl. */ +void +routerlist_free_(routerlist_t *rl) +{ + if (!rl) + return; + rimap_free(rl->identity_map, NULL); + sdmap_free(rl->desc_digest_map, NULL); + sdmap_free(rl->desc_by_eid_map, NULL); + eimap_free(rl->extra_info_map, extrainfo_free_void); + SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, + routerinfo_free(r)); + SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd, + signed_descriptor_free(sd)); + smartlist_free(rl->routers); + smartlist_free(rl->old_routers); + if (rl->desc_store.mmap) { + int res = tor_munmap_file(routerlist->desc_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap"); + } + } + if (rl->extrainfo_store.mmap) { + int res = tor_munmap_file(routerlist->extrainfo_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap"); + } + } + tor_free(rl); + + router_dir_info_changed(); +} + +/** Log information about how much memory is being used for routerlist, + * at log level severity. */ +void +dump_routerlist_mem_usage(int severity) +{ + uint64_t livedescs = 0; + uint64_t olddescs = 0; + if (!routerlist) + return; + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, r, + livedescs += r->cache_info.signed_descriptor_len); + SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, + olddescs += sd->signed_descriptor_len); + + tor_log(severity, LD_DIR, + "In %d live descriptors: %"PRIu64" bytes. " + "In %d old descriptors: %"PRIu64" bytes.", + smartlist_len(routerlist->routers), (livedescs), + smartlist_len(routerlist->old_routers), (olddescs)); +} + +/** Debugging helper: If idx is nonnegative, assert that ri is + * in sl at position idx. Otherwise, search sl for + * ri. Return the index of ri in sl, or -1 if ri + * is not in sl. */ +static inline int +routerlist_find_elt_(smartlist_t *sl, void *ri, int idx) +{ + if (idx < 0) { + idx = -1; + SMARTLIST_FOREACH(sl, routerinfo_t *, r, + if (r == ri) { + idx = r_sl_idx; + break; + }); + } else { + tor_assert(idx < smartlist_len(sl)); + tor_assert(smartlist_get(sl, idx) == ri); + }; + return idx; +} + +/** Insert an item ri into the routerlist rl, updating indices + * as needed. There must be no previous member of rl with the same + * identity digest as ri: If there is, call routerlist_replace + * instead. + */ +static void +routerlist_insert(routerlist_t *rl, routerinfo_t *ri) +{ + routerinfo_t *ri_old; + signed_descriptor_t *sd_old; + { + const routerinfo_t *ri_generated = router_get_my_routerinfo(); + tor_assert(ri_generated != ri); + } + tor_assert(ri->cache_info.routerlist_index == -1); + + ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri); + tor_assert(!ri_old); + + sd_old = sdmap_set(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest, + &(ri->cache_info)); + if (sd_old) { + int idx = sd_old->routerlist_index; + sd_old->routerlist_index = -1; + smartlist_del(rl->old_routers, idx); + if (idx < smartlist_len(rl->old_routers)) { + signed_descriptor_t *d = smartlist_get(rl->old_routers, idx); + d->routerlist_index = idx; + } + rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len; + sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest); + signed_descriptor_free(sd_old); + } + + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest, + &ri->cache_info); + smartlist_add(rl->routers, ri); + ri->cache_info.routerlist_index = smartlist_len(rl->routers) - 1; + nodelist_set_routerinfo(ri, NULL); + router_dir_info_changed(); +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Adds the extrainfo_t ei to the routerlist rl, if there is a + * corresponding router in rl-\>routers or rl-\>old_routers. Return the status + * of inserting ei. Free ei if it isn't inserted. */ +MOCK_IMPL(STATIC was_router_added_t, +extrainfo_insert,(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)) +{ + was_router_added_t r; + const char *compatibility_error_msg; + routerinfo_t *ri = rimap_get(rl->identity_map, + ei->cache_info.identity_digest); + signed_descriptor_t *sd = + sdmap_get(rl->desc_by_eid_map, ei->cache_info.signed_descriptor_digest); + extrainfo_t *ei_tmp; + const int severity = warn_if_incompatible ? LOG_WARN : LOG_INFO; + + { + extrainfo_t *ei_generated = router_get_my_extrainfo(); + tor_assert(ei_generated != ei); + } + + if (!ri) { + /* This router is unknown; we can't even verify the signature. Give up.*/ + r = ROUTER_NOT_IN_CONSENSUS; + goto done; + } + if (! sd) { + /* The extrainfo router doesn't have a known routerdesc to attach it to. + * This just won't work. */; + static ratelim_t no_sd_ratelim = RATELIM_INIT(1800); + r = ROUTER_BAD_EI; + log_fn_ratelim(&no_sd_ratelim, severity, LD_BUG, + "No entry found in extrainfo map."); + goto done; + } + if (tor_memneq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN)) { + static ratelim_t digest_mismatch_ratelim = RATELIM_INIT(1800); + /* The sd we got from the map doesn't match the digest we used to look + * it up. This makes no sense. */ + r = ROUTER_BAD_EI; + log_fn_ratelim(&digest_mismatch_ratelim, severity, LD_BUG, + "Mismatch in digest in extrainfo map."); + goto done; + } + if (routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, sd, + &compatibility_error_msg)) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + r = (ri->cache_info.extrainfo_is_bogus) ? + ROUTER_BAD_EI : ROUTER_NOT_IN_CONSENSUS; + + base16_encode(d1, sizeof(d1), ri->cache_info.identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), ei->cache_info.identity_digest, DIGEST_LEN); + + log_fn(severity,LD_DIR, + "router info incompatible with extra info (ri id: %s, ei id %s, " + "reason: %s)", d1, d2, compatibility_error_msg); + + goto done; + } + + /* Okay, if we make it here, we definitely have a router corresponding to + * this extrainfo. */ + + ei_tmp = eimap_set(rl->extra_info_map, + ei->cache_info.signed_descriptor_digest, + ei); + r = ROUTER_ADDED_SUCCESSFULLY; + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + + done: + if (r != ROUTER_ADDED_SUCCESSFULLY) + extrainfo_free(ei); + +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif + return r; +} + +#define should_cache_old_descriptors() \ + directory_caches_dir_info(get_options()) + +/** If we're a directory cache and routerlist rl doesn't have + * a copy of router ri yet, add it to the list of old (not + * recommended but still served) descriptors. Else free it. */ +static void +routerlist_insert_old(routerlist_t *rl, routerinfo_t *ri) +{ + { + const routerinfo_t *ri_generated = router_get_my_routerinfo(); + tor_assert(ri_generated != ri); + } + tor_assert(ri->cache_info.routerlist_index == -1); + + if (should_cache_old_descriptors() && + ri->purpose == ROUTER_PURPOSE_GENERAL && + !sdmap_get(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest)) { + signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri); + sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd); + smartlist_add(rl->old_routers, sd); + sd->routerlist_index = smartlist_len(rl->old_routers)-1; + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); + } else { + routerinfo_free(ri); + } +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Remove an item ri from the routerlist rl, updating indices + * as needed. If idx is nonnegative and smartlist_get(rl->routers, + * idx) == ri, we don't need to do a linear search over the list to decide + * which to remove. We fill the gap in rl->routers with a later element in + * the list, if any exists. ri is freed. + * + * If make_old is true, instead of deleting the router, we try adding + * it to rl->old_routers. */ +void +routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, time_t now) +{ + routerinfo_t *ri_tmp; + extrainfo_t *ei_tmp; + int idx = ri->cache_info.routerlist_index; + tor_assert(0 <= idx && idx < smartlist_len(rl->routers)); + tor_assert(smartlist_get(rl->routers, idx) == ri); + + nodelist_remove_routerinfo(ri); + + /* make sure the rephist module knows that it's not running */ + rep_hist_note_router_unreachable(ri->cache_info.identity_digest, now); + + ri->cache_info.routerlist_index = -1; + smartlist_del(rl->routers, idx); + if (idx < smartlist_len(rl->routers)) { + routerinfo_t *r = smartlist_get(rl->routers, idx); + r->cache_info.routerlist_index = idx; + } + + ri_tmp = rimap_remove(rl->identity_map, ri->cache_info.identity_digest); + router_dir_info_changed(); + tor_assert(ri_tmp == ri); + + if (make_old && should_cache_old_descriptors() && + ri->purpose == ROUTER_PURPOSE_GENERAL) { + signed_descriptor_t *sd; + sd = signed_descriptor_from_routerinfo(ri); + smartlist_add(rl->old_routers, sd); + sd->routerlist_index = smartlist_len(rl->old_routers)-1; + sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd); + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); + } else { + signed_descriptor_t *sd_tmp; + sd_tmp = sdmap_remove(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest); + tor_assert(sd_tmp == &(ri->cache_info)); + rl->desc_store.bytes_dropped += ri->cache_info.signed_descriptor_len; + ei_tmp = eimap_remove(rl->extra_info_map, + ri->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) + sdmap_remove(rl->desc_by_eid_map, ri->cache_info.extra_info_digest); + routerinfo_free(ri); + } +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Remove a signed_descriptor_t sd from rl-\>old_routers, and + * adjust rl as appropriate. idx is -1, or the index of + * sd. */ +static void +routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx) +{ + signed_descriptor_t *sd_tmp; + extrainfo_t *ei_tmp; + desc_store_t *store; + if (idx == -1) { + idx = sd->routerlist_index; + } + tor_assert(0 <= idx && idx < smartlist_len(rl->old_routers)); + /* XXXX edmanm's bridge relay triggered the following assert while + * running 0.2.0.12-alpha. If anybody triggers this again, see if we + * can get a backtrace. */ + tor_assert(smartlist_get(rl->old_routers, idx) == sd); + tor_assert(idx == sd->routerlist_index); + + sd->routerlist_index = -1; + smartlist_del(rl->old_routers, idx); + if (idx < smartlist_len(rl->old_routers)) { + signed_descriptor_t *d = smartlist_get(rl->old_routers, idx); + d->routerlist_index = idx; + } + sd_tmp = sdmap_remove(rl->desc_digest_map, + sd->signed_descriptor_digest); + tor_assert(sd_tmp == sd); + store = desc_get_store(rl, sd); + if (store) + store->bytes_dropped += sd->signed_descriptor_len; + + ei_tmp = eimap_remove(rl->extra_info_map, + sd->extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_remove(rl->desc_by_eid_map, sd->extra_info_digest); + + signed_descriptor_free(sd); +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Remove ri_old from the routerlist rl, and replace it with + * ri_new, updating all index info. If idx is nonnegative and + * smartlist_get(rl->routers, idx) == ri, we don't need to do a linear + * search over the list to decide which to remove. We put ri_new in the same + * index as ri_old, if possible. ri is freed as appropriate. + * + * If should_cache_descriptors() is true, instead of deleting the router, + * we add it to rl->old_routers. */ +static void +routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, + routerinfo_t *ri_new) +{ + int idx; + int same_descriptors; + + routerinfo_t *ri_tmp; + extrainfo_t *ei_tmp; + { + const routerinfo_t *ri_generated = router_get_my_routerinfo(); + tor_assert(ri_generated != ri_new); + } + tor_assert(ri_old != ri_new); + tor_assert(ri_new->cache_info.routerlist_index == -1); + + idx = ri_old->cache_info.routerlist_index; + tor_assert(0 <= idx && idx < smartlist_len(rl->routers)); + tor_assert(smartlist_get(rl->routers, idx) == ri_old); + + { + routerinfo_t *ri_old_tmp=NULL; + nodelist_set_routerinfo(ri_new, &ri_old_tmp); + tor_assert(ri_old == ri_old_tmp); + } + + router_dir_info_changed(); + if (idx >= 0) { + smartlist_set(rl->routers, idx, ri_new); + ri_old->cache_info.routerlist_index = -1; + ri_new->cache_info.routerlist_index = idx; + /* Check that ri_old is not in rl->routers anymore: */ + tor_assert( routerlist_find_elt_(rl->routers, ri_old, -1) == -1 ); + } else { + log_warn(LD_BUG, "Appending entry from routerlist_replace."); + routerlist_insert(rl, ri_new); + return; + } + if (tor_memneq(ri_old->cache_info.identity_digest, + ri_new->cache_info.identity_digest, DIGEST_LEN)) { + /* digests don't match; digestmap_set won't replace */ + rimap_remove(rl->identity_map, ri_old->cache_info.identity_digest); + } + ri_tmp = rimap_set(rl->identity_map, + ri_new->cache_info.identity_digest, ri_new); + tor_assert(!ri_tmp || ri_tmp == ri_old); + sdmap_set(rl->desc_digest_map, + ri_new->cache_info.signed_descriptor_digest, + &(ri_new->cache_info)); + + if (!tor_digest_is_zero(ri_new->cache_info.extra_info_digest)) { + sdmap_set(rl->desc_by_eid_map, ri_new->cache_info.extra_info_digest, + &ri_new->cache_info); + } + + same_descriptors = tor_memeq(ri_old->cache_info.signed_descriptor_digest, + ri_new->cache_info.signed_descriptor_digest, + DIGEST_LEN); + + if (should_cache_old_descriptors() && + ri_old->purpose == ROUTER_PURPOSE_GENERAL && + !same_descriptors) { + /* ri_old is going to become a signed_descriptor_t and go into + * old_routers */ + signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old); + smartlist_add(rl->old_routers, sd); + sd->routerlist_index = smartlist_len(rl->old_routers)-1; + sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd); + if (!tor_digest_is_zero(sd->extra_info_digest)) + sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); + } else { + /* We're dropping ri_old. */ + if (!same_descriptors) { + /* digests don't match; The sdmap_set above didn't replace */ + sdmap_remove(rl->desc_digest_map, + ri_old->cache_info.signed_descriptor_digest); + + if (tor_memneq(ri_old->cache_info.extra_info_digest, + ri_new->cache_info.extra_info_digest, DIGEST_LEN)) { + ei_tmp = eimap_remove(rl->extra_info_map, + ri_old->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + } + + if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { + sdmap_remove(rl->desc_by_eid_map, + ri_old->cache_info.extra_info_digest); + } + } + rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len; + routerinfo_free(ri_old); + } +#ifdef DEBUG_ROUTERLIST + routerlist_assert_ok(rl); +#endif +} + +/** Extract the descriptor sd from old_routerlist, and re-parse + * it as a fresh routerinfo_t. */ +static routerinfo_t * +routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) +{ + routerinfo_t *ri; + const char *body; + + body = signed_descriptor_get_annotations(sd); + + ri = router_parse_entry_from_string(body, + body+sd->signed_descriptor_len+sd->annotations_len, + 0, 1, NULL, NULL); + if (!ri) + return NULL; + signed_descriptor_move(&ri->cache_info, sd); + + routerlist_remove_old(rl, sd, -1); + + return ri; +} + +/** Free all memory held by the routerlist module. + * Note: Calling routerlist_free_all() should always be paired with + * a call to nodelist_free_all(). These should only be called during + * cleanup. + */ +void +routerlist_free_all(void) +{ + routerlist_free(routerlist); + routerlist = NULL; + dirlist_free_all(); + if (warned_nicknames) { + SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); + smartlist_free(warned_nicknames); + warned_nicknames = NULL; + } + authcert_free_all(); +} + +/** Forget that we have issued any router-related warnings, so that we'll + * warn again if we see the same errors. */ +void +routerlist_reset_warnings(void) +{ + if (!warned_nicknames) + warned_nicknames = smartlist_new(); + SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); + smartlist_clear(warned_nicknames); /* now the list is empty. */ + + networkstatus_reset_warnings(); +} + +/** Return 1 if the signed descriptor of this router is older than + * seconds seconds. Otherwise return 0. */ +MOCK_IMPL(int, +router_descriptor_is_older_than,(const routerinfo_t *router, int seconds)) +{ + return router->cache_info.published_on < approx_time() - seconds; +} + +/** Add router to the routerlist, if we don't already have it. Replace + * older entries (if any) with the same key. Note: Callers should not hold + * their pointers to router if this function fails; router + * will either be inserted into the routerlist or freed. Similarly, even + * if this call succeeds, they should not hold their pointers to + * router after subsequent calls with other routerinfo's -- they + * might cause the original routerinfo to get freed. + * + * Returns the status for the operation. Might set *msg if it wants + * the poster of the router to know something. + * + * If from_cache, this descriptor came from our disk cache. If + * from_fetch, we received it in response to a request we made. + * (If both are false, that means it was uploaded to us as an auth dir + * server or via the controller.) + * + * This function should be called *after* + * routers_update_status_from_consensus_networkstatus; subsequently, you + * should call router_rebuild_store and routerlist_descriptors_added. + */ +was_router_added_t +router_add_to_routerlist(routerinfo_t *router, const char **msg, + int from_cache, int from_fetch) +{ + const char *id_digest; + const or_options_t *options = get_options(); + int authdir = authdir_mode_handles_descs(options, router->purpose); + int authdir_believes_valid = 0; + routerinfo_t *old_router; + networkstatus_t *consensus = + networkstatus_get_latest_consensus_by_flavor(FLAV_NS); + int in_consensus = 0; + + tor_assert(msg); + + if (!routerlist) + router_get_routerlist(); + + id_digest = router->cache_info.identity_digest; + + old_router = router_get_mutable_by_digest(id_digest); + + /* Make sure that it isn't expired. */ + if (router->cert_expiration_time < approx_time()) { + routerinfo_free(router); + *msg = "Some certs on this router are expired."; + return ROUTER_CERTS_EXPIRED; + } + + /* Make sure that we haven't already got this exact descriptor. */ + if (sdmap_get(routerlist->desc_digest_map, + router->cache_info.signed_descriptor_digest)) { + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + + /* Only members of routerlist->identity_map can be bridges; we don't + * put bridges in old_routers. */ + const int was_bridge = old_router && + old_router->purpose == ROUTER_PURPOSE_BRIDGE; + + if (routerinfo_is_a_configured_bridge(router) && + router->purpose == ROUTER_PURPOSE_BRIDGE && + !was_bridge) { + log_info(LD_DIR, "Replacing non-bridge descriptor with bridge " + "descriptor for router %s", + router_describe(router)); + } else { + log_info(LD_DIR, + "Dropping descriptor that we already have for router %s", + router_describe(router)); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_IS_ALREADY_KNOWN; + } + } + + if (authdir) { + if (authdir_wants_to_reject_router(router, msg, + !from_cache && !from_fetch, + &authdir_believes_valid)) { + tor_assert(*msg); + routerinfo_free(router); + return ROUTER_AUTHDIR_REJECTS; + } + } else if (from_fetch) { + /* Only check the descriptor digest against the network statuses when + * we are receiving in response to a fetch. */ + + if (!signed_desc_digest_is_recognized(&router->cache_info) && + !routerinfo_is_a_configured_bridge(router)) { + /* We asked for it, so some networkstatus must have listed it when we + * did. Save it if we're a cache in case somebody else asks for it. */ + log_info(LD_DIR, + "Received a no-longer-recognized descriptor for router %s", + router_describe(router)); + *msg = "Router descriptor is not referenced by any network-status."; + + /* Only journal this desc if we want to keep old descriptors */ + if (!from_cache && should_cache_old_descriptors()) + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + routerlist_insert_old(routerlist, router); + return ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS; + } + } + + /* We no longer need a router with this descriptor digest. */ + if (consensus) { + routerstatus_t *rs = networkstatus_vote_find_mutable_entry( + consensus, id_digest); + if (rs && tor_memeq(rs->descriptor_digest, + router->cache_info.signed_descriptor_digest, + DIGEST_LEN)) { + in_consensus = 1; + } + } + + if (router->purpose == ROUTER_PURPOSE_GENERAL && + consensus && !in_consensus && !authdir) { + /* If it's a general router not listed in the consensus, then don't + * consider replacing the latest router with it. */ + if (!from_cache && should_cache_old_descriptors()) + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + routerlist_insert_old(routerlist, router); + *msg = "Skipping router descriptor: not in consensus."; + return ROUTER_NOT_IN_CONSENSUS; + } + + /* If we're reading a bridge descriptor from our cache, and we don't + * recognize it as one of our currently configured bridges, drop the + * descriptor. Otherwise we could end up using it as one of our entry + * guards even if it isn't in our Bridge config lines. */ + if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache && + !authdir_mode_bridge(options) && + !routerinfo_is_a_configured_bridge(router)) { + log_info(LD_DIR, "Dropping bridge descriptor for %s because we have " + "no bridge configured at that address.", + safe_str_client(router_describe(router))); + *msg = "Router descriptor was not a configured bridge."; + routerinfo_free(router); + return ROUTER_WAS_NOT_WANTED; + } + + /* If we have a router with the same identity key, choose the newer one. */ + if (old_router) { + if (!in_consensus && (router->cache_info.published_on <= + old_router->cache_info.published_on)) { + /* Same key, but old. This one is not listed in the consensus. */ + log_debug(LD_DIR, "Not-new descriptor for router %s", + router_describe(router)); + /* Only journal this desc if we'll be serving it. */ + if (!from_cache && should_cache_old_descriptors()) + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + routerlist_insert_old(routerlist, router); + *msg = "Router descriptor was not new."; + return ROUTER_IS_ALREADY_KNOWN; + } else { + /* Same key, and either new, or listed in the consensus. */ + log_debug(LD_DIR, "Replacing entry for router %s", + router_describe(router)); + routerlist_replace(routerlist, old_router, router); + if (!from_cache) { + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + } + *msg = authdir_believes_valid ? "Valid server updated" : + ("Invalid server updated. (This dirserver is marking your " + "server as unapproved.)"); + return ROUTER_ADDED_SUCCESSFULLY; + } + } + + if (!in_consensus && from_cache && + router_descriptor_is_older_than(router, OLD_ROUTER_DESC_MAX_AGE)) { + *msg = "Router descriptor was really old."; + routerinfo_free(router); + return ROUTER_WAS_TOO_OLD; + } + + /* We haven't seen a router with this identity before. Add it to the end of + * the list. */ + routerlist_insert(routerlist, router); + if (!from_cache) { + signed_desc_append_to_journal(&router->cache_info, + &routerlist->desc_store); + } + return ROUTER_ADDED_SUCCESSFULLY; +} + +/** Insert ei into the routerlist, or free it. Other arguments are + * as for router_add_to_routerlist(). Return ROUTER_ADDED_SUCCESSFULLY iff + * we actually inserted it, ROUTER_BAD_EI otherwise. + */ +was_router_added_t +router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch) +{ + was_router_added_t inserted; + (void)from_fetch; + if (msg) *msg = NULL; + /*XXXX Do something with msg */ + + inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache); + + if (WRA_WAS_ADDED(inserted) && !from_cache) + signed_desc_append_to_journal(&ei->cache_info, + &routerlist->extrainfo_store); + + return inserted; +} + +/** Sorting helper: return <0, 0, or >0 depending on whether the + * signed_descriptor_t* in *a has an identity digest preceding, equal + * to, or later than that of *b. */ +static int +compare_old_routers_by_identity_(const void **_a, const void **_b) +{ + int i; + const signed_descriptor_t *r1 = *_a, *r2 = *_b; + if ((i = fast_memcmp(r1->identity_digest, r2->identity_digest, DIGEST_LEN))) + return i; + return (int)(r1->published_on - r2->published_on); +} + +/** Internal type used to represent how long an old descriptor was valid, + * where it appeared in the list of old descriptors, and whether it's extra + * old. Used only by routerlist_remove_old_cached_routers_with_id(). */ +struct duration_idx_t { + int duration; + int idx; + int old; +}; + +/** Sorting helper: compare two duration_idx_t by their duration. */ +static int +compare_duration_idx_(const void *_d1, const void *_d2) +{ + const struct duration_idx_t *d1 = _d1; + const struct duration_idx_t *d2 = _d2; + return d1->duration - d2->duration; +} + +/** The range lo through hi inclusive of routerlist->old_routers + * must contain routerinfo_t with the same identity and with publication time + * in ascending order. Remove members from this range until there are no more + * than max_descriptors_per_router() remaining. Start by removing the oldest + * members from before cutoff, then remove members which were current + * for the lowest amount of time. The order of members of old_routers at + * indices lo or higher may be changed. + */ +static void +routerlist_remove_old_cached_routers_with_id(time_t now, + time_t cutoff, int lo, int hi, + digestset_t *retain) +{ + int i, n = hi-lo+1; + unsigned n_extra, n_rmv = 0; + struct duration_idx_t *lifespans; + uint8_t *rmv, *must_keep; + smartlist_t *lst = routerlist->old_routers; +#if 1 + const char *ident; + tor_assert(hi < smartlist_len(lst)); + tor_assert(lo <= hi); + ident = ((signed_descriptor_t*)smartlist_get(lst, lo))->identity_digest; + for (i = lo+1; i <= hi; ++i) { + signed_descriptor_t *r = smartlist_get(lst, i); + tor_assert(tor_memeq(ident, r->identity_digest, DIGEST_LEN)); + } +#endif /* 1 */ + /* Check whether we need to do anything at all. */ + { + int mdpr = directory_caches_dir_info(get_options()) ? 2 : 1; + if (n <= mdpr) + return; + n_extra = n - mdpr; + } + + lifespans = tor_calloc(n, sizeof(struct duration_idx_t)); + rmv = tor_calloc(n, sizeof(uint8_t)); + must_keep = tor_calloc(n, sizeof(uint8_t)); + /* Set lifespans to contain the lifespan and index of each server. */ + /* Set rmv[i-lo]=1 if we're going to remove a server for being too old. */ + for (i = lo; i <= hi; ++i) { + signed_descriptor_t *r = smartlist_get(lst, i); + signed_descriptor_t *r_next; + lifespans[i-lo].idx = i; + if (r->last_listed_as_valid_until >= now || + (retain && digestset_probably_contains(retain, + r->signed_descriptor_digest))) { + must_keep[i-lo] = 1; + } + if (i < hi) { + r_next = smartlist_get(lst, i+1); + tor_assert(r->published_on <= r_next->published_on); + lifespans[i-lo].duration = (int)(r_next->published_on - r->published_on); + } else { + r_next = NULL; + lifespans[i-lo].duration = INT_MAX; + } + if (!must_keep[i-lo] && r->published_on < cutoff && n_rmv < n_extra) { + ++n_rmv; + lifespans[i-lo].old = 1; + rmv[i-lo] = 1; + } + } + + if (n_rmv < n_extra) { + /** + * We aren't removing enough servers for being old. Sort lifespans by + * the duration of liveness, and remove the ones we're not already going to + * remove based on how long they were alive. + **/ + qsort(lifespans, n, sizeof(struct duration_idx_t), compare_duration_idx_); + for (i = 0; i < n && n_rmv < n_extra; ++i) { + if (!must_keep[lifespans[i].idx-lo] && !lifespans[i].old) { + rmv[lifespans[i].idx-lo] = 1; + ++n_rmv; + } + } + } + + i = hi; + do { + if (rmv[i-lo]) + routerlist_remove_old(routerlist, smartlist_get(lst, i), i); + } while (--i >= lo); + tor_free(must_keep); + tor_free(rmv); + tor_free(lifespans); +} + +/** Deactivate any routers from the routerlist that are more than + * ROUTER_MAX_AGE seconds old and not recommended by any networkstatuses; + * remove old routers from the list of cached routers if we have too many. + */ +void +routerlist_remove_old_routers(void) +{ + int i, hi=-1; + const char *cur_id = NULL; + time_t now = time(NULL); + time_t cutoff; + routerinfo_t *router; + signed_descriptor_t *sd; + digestset_t *retain; + const networkstatus_t *consensus = networkstatus_get_latest_consensus(); + + trusted_dirs_remove_old_certs(); + + if (!routerlist || !consensus) + return; + + // routerlist_assert_ok(routerlist); + + /* We need to guess how many router descriptors we will wind up wanting to + retain, so that we can be sure to allocate a large enough Bloom filter + to hold the digest set. Overestimating is fine; underestimating is bad. + */ + { + /* We'll probably retain everything in the consensus. */ + int n_max_retain = smartlist_len(consensus->routerstatus_list); + retain = digestset_new(n_max_retain); + } + + cutoff = now - OLD_ROUTER_DESC_MAX_AGE; + /* Retain anything listed in the consensus. */ + if (consensus) { + SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, + if (rs->published_on >= cutoff) + digestset_add(retain, rs->descriptor_digest)); + } + + /* If we have a consensus, we should consider pruning current routers that + * are too old and that nobody recommends. (If we don't have a consensus, + * then we should get one before we decide to kill routers.) */ + + if (consensus) { + cutoff = now - ROUTER_MAX_AGE; + /* Remove too-old unrecommended members of routerlist->routers. */ + for (i = 0; i < smartlist_len(routerlist->routers); ++i) { + router = smartlist_get(routerlist->routers, i); + if (router->cache_info.published_on <= cutoff && + router->cache_info.last_listed_as_valid_until < now && + !digestset_probably_contains(retain, + router->cache_info.signed_descriptor_digest)) { + /* Too old: remove it. (If we're a cache, just move it into + * old_routers.) */ + log_info(LD_DIR, + "Forgetting obsolete (too old) routerinfo for router %s", + router_describe(router)); + routerlist_remove(routerlist, router, 1, now); + i--; + } + } + } + + //routerlist_assert_ok(routerlist); + + /* Remove far-too-old members of routerlist->old_routers. */ + cutoff = now - OLD_ROUTER_DESC_MAX_AGE; + for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) { + sd = smartlist_get(routerlist->old_routers, i); + if (sd->published_on <= cutoff && + sd->last_listed_as_valid_until < now && + !digestset_probably_contains(retain, sd->signed_descriptor_digest)) { + /* Too old. Remove it. */ + routerlist_remove_old(routerlist, sd, i--); + } + } + + //routerlist_assert_ok(routerlist); + + log_info(LD_DIR, "We have %d live routers and %d old router descriptors.", + smartlist_len(routerlist->routers), + smartlist_len(routerlist->old_routers)); + + /* Now we might have to look at routerlist->old_routers for extraneous + * members. (We'd keep all the members if we could, but we need to save + * space.) First, check whether we have too many router descriptors, total. + * We're okay with having too many for some given router, so long as the + * total number doesn't approach max_descriptors_per_router()*len(router). + */ + if (smartlist_len(routerlist->old_routers) < + smartlist_len(routerlist->routers)) + goto done; + + /* Sort by identity, then fix indices. */ + smartlist_sort(routerlist->old_routers, compare_old_routers_by_identity_); + /* Fix indices. */ + for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) { + signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i); + r->routerlist_index = i; + } + + /* Iterate through the list from back to front, so when we remove descriptors + * we don't mess up groups we haven't gotten to. */ + for (i = smartlist_len(routerlist->old_routers)-1; i >= 0; --i) { + signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i); + if (!cur_id) { + cur_id = r->identity_digest; + hi = i; + } + if (tor_memneq(cur_id, r->identity_digest, DIGEST_LEN)) { + routerlist_remove_old_cached_routers_with_id(now, + cutoff, i+1, hi, retain); + cur_id = r->identity_digest; + hi = i; + } + } + if (hi>=0) + routerlist_remove_old_cached_routers_with_id(now, cutoff, 0, hi, retain); + //routerlist_assert_ok(routerlist); + + done: + digestset_free(retain); + router_rebuild_store(RRS_DONT_REMOVE_OLD, &routerlist->desc_store); + router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store); +} + +/** We just added a new set of descriptors. Take whatever extra steps + * we need. */ +void +routerlist_descriptors_added(smartlist_t *sl, int from_cache) +{ + tor_assert(sl); + control_event_descriptors_changed(sl); + SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { + if (ri->purpose == ROUTER_PURPOSE_BRIDGE) + learned_bridge_descriptor(ri, from_cache); + if (ri->needs_retest_if_added) { + ri->needs_retest_if_added = 0; + dirserv_single_reachability_test(approx_time(), ri); + } + } SMARTLIST_FOREACH_END(ri); +} + +/** + * Code to parse a single router descriptor and insert it into the + * routerlist. Return -1 if the descriptor was ill-formed; 0 if the + * descriptor was well-formed but could not be added; and 1 if the + * descriptor was added. + * + * If we don't add it and msg is not NULL, then assign to + * *msg a static string describing the reason for refusing the + * descriptor. + * + * This is used only by the controller. + */ +int +router_load_single_router(const char *s, uint8_t purpose, int cache, + const char **msg) +{ + routerinfo_t *ri; + was_router_added_t r; + smartlist_t *lst; + char annotation_buf[ROUTER_ANNOTATION_BUF_LEN]; + tor_assert(msg); + *msg = NULL; + + tor_snprintf(annotation_buf, sizeof(annotation_buf), + "@source controller\n" + "@purpose %s\n", router_purpose_to_string(purpose)); + + if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, + annotation_buf, NULL))) { + log_warn(LD_DIR, "Error parsing router descriptor; dropping."); + *msg = "Couldn't parse router descriptor."; + return -1; + } + tor_assert(ri->purpose == purpose); + if (router_is_me(ri)) { + log_warn(LD_DIR, "Router's identity key matches mine; dropping."); + *msg = "Router's identity key matches mine."; + routerinfo_free(ri); + return 0; + } + + if (!cache) /* obey the preference of the controller */ + ri->cache_info.do_not_cache = 1; + + lst = smartlist_new(); + smartlist_add(lst, ri); + routers_update_status_from_consensus_networkstatus(lst, 0); + + r = router_add_to_routerlist(ri, msg, 0, 0); + if (!WRA_WAS_ADDED(r)) { + /* we've already assigned to *msg now, and ri is already freed */ + tor_assert(*msg); + if (r == ROUTER_AUTHDIR_REJECTS) + log_warn(LD_DIR, "Couldn't add router to list: %s Dropping.", *msg); + smartlist_free(lst); + return 0; + } else { + routerlist_descriptors_added(lst, 0); + smartlist_free(lst); + log_debug(LD_DIR, "Added router to list"); + return 1; + } +} + +/** Given a string s containing some routerdescs, parse it and put the + * routers into our directory. If saved_location is SAVED_NOWHERE, the routers + * are in response to a query to the network: cache them by adding them to + * the journal. + * + * Return the number of routers actually added. + * + * If requested_fingerprints is provided, it must contain a list of + * uppercased fingerprints. Do not update any router whose + * fingerprint is not on the list; after updating a router, remove its + * fingerprint from the list. + * + * If descriptor_digests is non-zero, then the requested_fingerprints + * are descriptor digests. Otherwise they are identity digests. + */ +int +router_load_routers_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests, + const char *prepend_annotations) +{ + smartlist_t *routers = smartlist_new(), *changed = smartlist_new(); + char fp[HEX_DIGEST_LEN+1]; + const char *msg; + int from_cache = (saved_location != SAVED_NOWHERE); + int allow_annotations = (saved_location != SAVED_NOWHERE); + int any_changed = 0; + smartlist_t *invalid_digests = smartlist_new(); + + router_parse_list_from_string(&s, eos, routers, saved_location, 0, + allow_annotations, prepend_annotations, + invalid_digests); + + routers_update_status_from_consensus_networkstatus(routers, !from_cache); + + log_info(LD_DIR, "%d elements to add", smartlist_len(routers)); + + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + was_router_added_t r; + char d[DIGEST_LEN]; + if (requested_fingerprints) { + base16_encode(fp, sizeof(fp), descriptor_digests ? + ri->cache_info.signed_descriptor_digest : + ri->cache_info.identity_digest, + DIGEST_LEN); + if (smartlist_contains_string(requested_fingerprints, fp)) { + smartlist_string_remove(requested_fingerprints, fp); + } else { + char *requested = + smartlist_join_strings(requested_fingerprints," ",0,NULL); + log_warn(LD_DIR, + "We received a router descriptor with a fingerprint (%s) " + "that we never requested. (We asked for: %s.) Dropping.", + fp, requested); + tor_free(requested); + routerinfo_free(ri); + continue; + } + } + + memcpy(d, ri->cache_info.signed_descriptor_digest, DIGEST_LEN); + r = router_add_to_routerlist(ri, &msg, from_cache, !from_cache); + if (WRA_WAS_ADDED(r)) { + any_changed++; + smartlist_add(changed, ri); + routerlist_descriptors_added(changed, from_cache); + smartlist_clear(changed); + } else if (WRA_NEVER_DOWNLOADABLE(r)) { + download_status_t *dl_status; + dl_status = router_get_dl_status_by_descriptor_digest(d); + if (dl_status) { + log_info(LD_GENERAL, "Marking router %s as never downloadable", + hex_str(d, DIGEST_LEN)); + download_status_mark_impossible(dl_status); + } + } + } SMARTLIST_FOREACH_END(ri); + + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints && descriptor_digests) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + download_status_t *dls; + dls = router_get_dl_status_by_descriptor_digest((char*)bad_digest); + if (dls) { + log_info(LD_GENERAL, "Marking router with descriptor %s as unparseable, " + "and therefore undownloadable", fp); + download_status_mark_impossible(dls); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + + routerlist_assert_ok(routerlist); + + if (any_changed) + router_rebuild_store(0, &routerlist->desc_store); + + smartlist_free(routers); + smartlist_free(changed); + + return any_changed; +} + +/** Parse one or more extrainfos from s (ending immediately before + * eos if eos is present). Other arguments are as for + * router_load_routers_from_string(). */ +void +router_load_extrainfo_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests) +{ + smartlist_t *extrainfo_list = smartlist_new(); + const char *msg; + int from_cache = (saved_location != SAVED_NOWHERE); + smartlist_t *invalid_digests = smartlist_new(); + + router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0, + NULL, invalid_digests); + + log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list)); + + SMARTLIST_FOREACH_BEGIN(extrainfo_list, extrainfo_t *, ei) { + uint8_t d[DIGEST_LEN]; + memcpy(d, ei->cache_info.signed_descriptor_digest, DIGEST_LEN); + was_router_added_t added = + router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache); + if (WRA_WAS_ADDED(added) && requested_fingerprints) { + char fp[HEX_DIGEST_LEN+1]; + base16_encode(fp, sizeof(fp), descriptor_digests ? + ei->cache_info.signed_descriptor_digest : + ei->cache_info.identity_digest, + DIGEST_LEN); + smartlist_string_remove(requested_fingerprints, fp); + /* We silently let relays stuff us with extrainfos we didn't ask for, + * so long as we would have wanted them anyway. Since we always fetch + * all the extrainfos we want, and we never actually act on them + * inside Tor, this should be harmless. */ + } else if (WRA_NEVER_DOWNLOADABLE(added)) { + signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", + hex_str((char*)d,DIGEST_LEN)); + download_status_mark_impossible(&sd->ei_dl_status); + } + } + } SMARTLIST_FOREACH_END(ei); + + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + char fp[HEX_DIGEST_LEN+1]; + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + signed_descriptor_t *sd = + router_get_by_extrainfo_digest((char*)bad_digest); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", fp); + download_status_mark_impossible(&sd->ei_dl_status); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + + routerlist_assert_ok(routerlist); + router_rebuild_store(0, &router_get_routerlist()->extrainfo_store); + + smartlist_free(extrainfo_list); +} + +/** Return true iff the latest ns-flavored consensus includes a descriptor + * whose digest is that of desc. */ +static int +signed_desc_digest_is_recognized(signed_descriptor_t *desc) +{ + const routerstatus_t *rs; + networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor( + FLAV_NS); + + if (consensus) { + rs = networkstatus_vote_find_entry(consensus, desc->identity_digest); + if (rs && tor_memeq(rs->descriptor_digest, + desc->signed_descriptor_digest, DIGEST_LEN)) + return 1; + } + return 0; +} + +/** Update downloads for router descriptors and/or microdescriptors as + * appropriate. */ +void +update_all_descriptor_downloads(time_t now) +{ + if (should_delay_dir_fetches(get_options(), NULL)) + return; + update_router_descriptor_downloads(now); + update_microdesc_downloads(now); + launch_dummy_descriptor_download_as_needed(now, get_options()); +} + +/** Clear all our timeouts for fetching v3 directory stuff, and then + * give it all a try again. */ +void +routerlist_retry_directory_downloads(time_t now) +{ + (void)now; + + log_debug(LD_GENERAL, + "In routerlist_retry_directory_downloads()"); + + router_reset_status_download_failures(); + router_reset_descriptor_download_failures(); + reschedule_directory_downloads(); +} + +/** Return true iff router does not permit exit streams. + */ +int +router_exit_policy_rejects_all(const routerinfo_t *router) +{ + return router->policy_is_reject_star; +} + +/** For every current directory connection whose purpose is purpose, + * and where the resource being downloaded begins with prefix, split + * rest of the resource into base16 fingerprints (or base64 fingerprints if + * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the + * corresponding elements of result to a nonzero value. + */ +void +list_pending_downloads(digestmap_t *result, digest256map_t *result256, + int purpose, const char *prefix) +{ + const size_t p_len = strlen(prefix); + smartlist_t *tmp = smartlist_new(); + smartlist_t *conns = get_connection_array(); + int flags = DSR_HEX; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) + flags = DSR_DIGEST256|DSR_BASE64; + + tor_assert(result || result256); + + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + if (conn->type == CONN_TYPE_DIR && + conn->purpose == purpose && + !conn->marked_for_close) { + const char *resource = TO_DIR_CONN(conn)->requested_resource; + if (!strcmpstart(resource, prefix)) + dir_split_resource_into_fingerprints(resource + p_len, + tmp, NULL, flags); + } + } SMARTLIST_FOREACH_END(conn); + + if (result) { + SMARTLIST_FOREACH(tmp, char *, d, + { + digestmap_set(result, d, (void*)1); + tor_free(d); + }); + } else if (result256) { + SMARTLIST_FOREACH(tmp, uint8_t *, d, + { + digest256map_set(result256, d, (void*)1); + tor_free(d); + }); + } + smartlist_free(tmp); +} + +/** For every router descriptor (or extra-info document if extrainfo is + * true) we are currently downloading by descriptor digest, set result[d] to + * (void*)1. */ +static void +list_pending_descriptor_downloads(digestmap_t *result, int extrainfo) +{ + int purpose = + extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC; + list_pending_downloads(result, NULL, purpose, "d/"); +} + +/** For every microdescriptor we are currently downloading by descriptor + * digest, set result[d] to (void*)1. + */ +void +list_pending_microdesc_downloads(digest256map_t *result) +{ + list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); +} + +/** Launch downloads for all the descriptors whose digests or digests256 + * are listed as digests[i] for lo <= i < hi. (Lo and hi may be out of + * range.) If source is given, download from source; + * otherwise, download from an appropriate random directory server. + */ +MOCK_IMPL(STATIC void, +initiate_descriptor_downloads,(const routerstatus_t *source, + int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)) +{ + char *resource, *cp; + int digest_len, enc_digest_len; + const char *sep; + int b64_256; + smartlist_t *tmp; + + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + /* Microdescriptors are downloaded by "-"-separated base64-encoded + * 256-bit digests. */ + digest_len = DIGEST256_LEN; + enc_digest_len = BASE64_DIGEST256_LEN + 1; + sep = "-"; + b64_256 = 1; + } else { + digest_len = DIGEST_LEN; + enc_digest_len = HEX_DIGEST_LEN + 1; + sep = "+"; + b64_256 = 0; + } + + if (lo < 0) + lo = 0; + if (hi > smartlist_len(digests)) + hi = smartlist_len(digests); + + if (hi-lo <= 0) + return; + + tmp = smartlist_new(); + + for (; lo < hi; ++lo) { + cp = tor_malloc(enc_digest_len); + if (b64_256) { + digest256_to_base64(cp, smartlist_get(digests, lo)); + } else { + base16_encode(cp, enc_digest_len, smartlist_get(digests, lo), + digest_len); + } + smartlist_add(tmp, cp); + } + + cp = smartlist_join_strings(tmp, sep, 0, NULL); + tor_asprintf(&resource, "d/%s.z", cp); + + SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1)); + smartlist_free(tmp); + tor_free(cp); + + if (source) { + /* We know which authority or directory mirror we want. */ + directory_request_t *req = directory_request_new(purpose); + directory_request_set_routerstatus(req, source); + directory_request_set_resource(req, resource); + directory_initiate_request(req); + directory_request_free(req); + } else { + directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource, + pds_flags, DL_WANT_ANY_DIRSERVER); + } + tor_free(resource); +} + +/** Return the max number of hashes to put in a URL for a given request. + */ +static int +max_dl_per_request(const or_options_t *options, int purpose) +{ + /* Since squid does not like URLs >= 4096 bytes we limit it to 96. + * 4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535 + * /tor/server/d/.z) == 4026 + * 4026/41 (40 for the hash and 1 for the + that separates them) => 98 + * So use 96 because it's a nice number. + * + * For microdescriptors, the calculation is + * 4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535 + * /tor/micro/d/.z) == 4027 + * 4027/44 (43 for the hash and 1 for the - that separates them) => 91 + * So use 90 because it's a nice number. + */ + int max = 96; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + max = 90; + } + /* If we're going to tunnel our connections, we can ask for a lot more + * in a request. */ + if (directory_must_use_begindir(options)) { + max = 500; + } + return max; +} + +/** Don't split our requests so finely that we are requesting fewer than + * this number per server. (Grouping more than this at once leads to + * diminishing returns.) */ +#define MIN_DL_PER_REQUEST 32 +/** To prevent a single screwy cache from confusing us by selective reply, + * try to split our requests into at least this many requests. */ +#define MIN_REQUESTS 3 +/** If we want fewer than this many descriptors, wait until we + * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */ +#define MAX_DL_TO_DELAY 16 + +/** Given a purpose (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of + * router descriptor digests or microdescriptor digest256s in + * downloadable, decide whether to delay fetching until we have more. + * If we don't want to delay, launch one or more requests to the appropriate + * directory authorities. + */ +void +launch_descriptor_downloads(int purpose, + smartlist_t *downloadable, + const routerstatus_t *source, time_t now) +{ + const or_options_t *options = get_options(); + const char *descname; + const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC); + int n_downloadable = smartlist_len(downloadable); + + int i, n_per_request, max_dl_per_req; + const char *req_plural = "", *rtr_plural = ""; + int pds_flags = PDS_RETRY_IF_NO_SERVERS; + + tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC); + descname = fetch_microdesc ? "microdesc" : "routerdesc"; + + if (!n_downloadable) + return; + + if (!directory_fetches_dir_info_early(options)) { + if (n_downloadable >= MAX_DL_TO_DELAY) { + log_debug(LD_DIR, + "There are enough downloadable %ss to launch requests.", + descname); + } else if (! router_have_minimum_dir_info()) { + log_debug(LD_DIR, + "We are only missing %d %ss, but we'll fetch anyway, since " + "we don't yet have enough directory info.", + n_downloadable, descname); + } else { + + /* should delay */ + if ((last_descriptor_download_attempted + + options->TestingClientMaxIntervalWithoutRequest) > now) + return; + + if (last_descriptor_download_attempted) { + log_info(LD_DIR, + "There are not many downloadable %ss, but we've " + "been waiting long enough (%d seconds). Downloading.", + descname, + (int)(now-last_descriptor_download_attempted)); + } else { + log_info(LD_DIR, + "There are not many downloadable %ss, but we haven't " + "tried downloading descriptors recently. Downloading.", + descname); + } + } + } + + if (!authdir_mode(options)) { + /* If we wind up going to the authorities, we want to only open one + * connection to each authority at a time, so that we don't overload + * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH + * regardless of whether we're a cache or not. + * + * Setting this flag can make initiate_descriptor_downloads() ignore + * requests. We need to make sure that we do in fact call + * update_router_descriptor_downloads() later on, once the connections + * have succeeded or failed. + */ + pds_flags |= fetch_microdesc ? + PDS_NO_EXISTING_MICRODESC_FETCH : + PDS_NO_EXISTING_SERVERDESC_FETCH; + } + + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); + max_dl_per_req = max_dl_per_request(options, purpose); + + if (n_per_request > max_dl_per_req) + n_per_request = max_dl_per_req; + + if (n_per_request < MIN_DL_PER_REQUEST) { + n_per_request = MIN(MIN_DL_PER_REQUEST, n_downloadable); + } + + if (n_downloadable > n_per_request) + req_plural = rtr_plural = "s"; + else if (n_downloadable > 1) + rtr_plural = "s"; + + log_info(LD_DIR, + "Launching %d request%s for %d %s%s, %d at a time", + CEIL_DIV(n_downloadable, n_per_request), req_plural, + n_downloadable, descname, rtr_plural, n_per_request); + smartlist_sort_digests(downloadable); + for (i=0; i < n_downloadable; i += n_per_request) { + initiate_descriptor_downloads(source, purpose, + downloadable, i, i+n_per_request, + pds_flags); + } + last_descriptor_download_attempted = now; +} + +/** For any descriptor that we want that's currently listed in + * consensus, download it as appropriate. */ +void +update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus) +{ + const or_options_t *options = get_options(); + digestmap_t *map = NULL; + smartlist_t *no_longer_old = smartlist_new(); + smartlist_t *downloadable = smartlist_new(); + routerstatus_t *source = NULL; + int authdir = authdir_mode(options); + int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0, + n_inprogress=0, n_in_oldrouters=0; + + if (directory_too_idle_to_fetch_descriptors(options, now)) + goto done; + if (!consensus) + goto done; + + if (is_vote) { + /* where's it from, so we know whom to ask for descriptors */ + dir_server_t *ds; + networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0); + tor_assert(voter); + ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest); + if (ds) + source = &(ds->fake_status); + else + log_warn(LD_DIR, "couldn't lookup source from vote?"); + } + + map = digestmap_new(); + list_pending_descriptor_downloads(map, 0); + SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, void *, rsp) { + routerstatus_t *rs = + is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp; + signed_descriptor_t *sd; + if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) { + const routerinfo_t *ri; + ++n_have; + if (!(ri = router_get_by_id_digest(rs->identity_digest)) || + tor_memneq(ri->cache_info.signed_descriptor_digest, + sd->signed_descriptor_digest, DIGEST_LEN)) { + /* We have a descriptor with this digest, but either there is no + * entry in routerlist with the same ID (!ri), or there is one, + * but the identity digest differs (memneq). + */ + smartlist_add(no_longer_old, sd); + ++n_in_oldrouters; /* We have it in old_routers. */ + } + continue; /* We have it already. */ + } + if (digestmap_get(map, rs->descriptor_digest)) { + ++n_inprogress; + continue; /* We have an in-progress download. */ + } + if (!download_status_is_ready(&rs->dl_status, now)) { + ++n_delayed; /* Not ready for retry. */ + continue; + } + if (authdir && dirserv_would_reject_router(rs)) { + ++n_would_reject; + continue; /* We would throw it out immediately. */ + } + if (!we_want_to_fetch_flavor(options, consensus->flavor) && + !client_would_use_router(rs, now)) { + ++n_wouldnt_use; + continue; /* We would never use it ourself. */ + } + if (is_vote && source) { + char time_bufnew[ISO_TIME_LEN+1]; + char time_bufold[ISO_TIME_LEN+1]; + const routerinfo_t *oldrouter; + oldrouter = router_get_by_id_digest(rs->identity_digest); + format_iso_time(time_bufnew, rs->published_on); + if (oldrouter) + format_iso_time(time_bufold, oldrouter->cache_info.published_on); + log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)", + routerstatus_describe(rs), + time_bufnew, + oldrouter ? time_bufold : "none", + source->nickname, oldrouter ? "known" : "unknown"); + } + smartlist_add(downloadable, rs->descriptor_digest); + } SMARTLIST_FOREACH_END(rsp); + + if (!authdir_mode_v3(options) + && smartlist_len(no_longer_old)) { + routerlist_t *rl = router_get_routerlist(); + log_info(LD_DIR, "%d router descriptors listed in consensus are " + "currently in old_routers; making them current.", + smartlist_len(no_longer_old)); + SMARTLIST_FOREACH_BEGIN(no_longer_old, signed_descriptor_t *, sd) { + const char *msg; + was_router_added_t r; + time_t tmp_cert_expiration_time; + routerinfo_t *ri = routerlist_reparse_old(rl, sd); + if (!ri) { + log_warn(LD_BUG, "Failed to re-parse a router."); + continue; + } + /* need to remember for below, since add_to_routerlist may free. */ + tmp_cert_expiration_time = ri->cert_expiration_time; + + r = router_add_to_routerlist(ri, &msg, 1, 0); + if (WRA_WAS_OUTDATED(r)) { + log_warn(LD_DIR, "Couldn't add re-parsed router: %s. This isn't " + "usually a big deal, but you should make sure that your " + "clock and timezone are set correctly.", + msg?msg:"???"); + if (r == ROUTER_CERTS_EXPIRED) { + char time_cons[ISO_TIME_LEN+1]; + char time_cert_expires[ISO_TIME_LEN+1]; + format_iso_time(time_cons, consensus->valid_after); + format_iso_time(time_cert_expires, tmp_cert_expiration_time); + log_warn(LD_DIR, " (I'm looking at a consensus from %s; This " + "router's certificates began expiring at %s.)", + time_cons, time_cert_expires); + } + } + } SMARTLIST_FOREACH_END(sd); + routerlist_assert_ok(rl); + } + + log_info(LD_DIR, + "%d router descriptors downloadable. %d delayed; %d present " + "(%d of those were in old_routers); %d would_reject; " + "%d wouldnt_use; %d in progress.", + smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, + n_would_reject, n_wouldnt_use, n_inprogress); + + launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC, + downloadable, source, now); + + digestmap_free(map, NULL); + done: + smartlist_free(downloadable); + smartlist_free(no_longer_old); +} + +/** How often should we launch a server/authority request to be sure of getting + * a guess for our IP? */ +/*XXXX+ this info should come from netinfo cells or something, or we should + * do this only when we aren't seeing incoming data. see bug 652. */ +#define DUMMY_DOWNLOAD_INTERVAL (20*60) + +/** As needed, launch a dummy router descriptor fetch to see if our + * address has changed. */ +static void +launch_dummy_descriptor_download_as_needed(time_t now, + const or_options_t *options) +{ + static time_t last_dummy_download = 0; + /* XXXX+ we could be smarter here; see notes on bug 652. */ + /* If we're a server that doesn't have a configured address, we rely on + * directory fetches to learn when our address changes. So if we haven't + * tried to get any routerdescs in a long time, try a dummy fetch now. */ + if (!options->Address && + server_mode(options) && + last_descriptor_download_attempted + DUMMY_DOWNLOAD_INTERVAL < now && + last_dummy_download + DUMMY_DOWNLOAD_INTERVAL < now) { + last_dummy_download = now; + /* XX/teor - do we want an authority here, because they are less likely + * to give us the wrong address? (See #17782) + * I'm leaving the previous behaviour intact, because I don't like + * the idea of some relays contacting an authority every 20 minutes. */ + directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC, + ROUTER_PURPOSE_GENERAL, "authority.z", + PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); + } +} + +/** Launch downloads for router status as needed. */ +void +update_router_descriptor_downloads(time_t now) +{ + const or_options_t *options = get_options(); + if (should_delay_dir_fetches(options, NULL)) + return; + if (!we_fetch_router_descriptors(options)) + return; + + update_consensus_router_descriptor_downloads(now, 0, + networkstatus_get_reasonably_live_consensus(now, FLAV_NS)); +} + +/** Launch extrainfo downloads as needed. */ +void +update_extrainfo_downloads(time_t now) +{ + const or_options_t *options = get_options(); + routerlist_t *rl; + smartlist_t *wanted; + digestmap_t *pending; + int old_routers, i, max_dl_per_req; + int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0, n_bogus[2] = {0,0}; + if (! options->DownloadExtraInfo) + return; + if (should_delay_dir_fetches(options, NULL)) + return; + if (!router_have_minimum_dir_info()) + return; + + pending = digestmap_new(); + list_pending_descriptor_downloads(pending, 1); + rl = router_get_routerlist(); + wanted = smartlist_new(); + for (old_routers = 0; old_routers < 2; ++old_routers) { + smartlist_t *lst = old_routers ? rl->old_routers : rl->routers; + for (i = 0; i < smartlist_len(lst); ++i) { + signed_descriptor_t *sd; + char *d; + if (old_routers) + sd = smartlist_get(lst, i); + else + sd = &((routerinfo_t*)smartlist_get(lst, i))->cache_info; + if (sd->is_extrainfo) + continue; /* This should never happen. */ + if (old_routers && !router_get_by_id_digest(sd->identity_digest)) + continue; /* Couldn't check the signature if we got it. */ + if (sd->extrainfo_is_bogus) + continue; + d = sd->extra_info_digest; + if (tor_digest_is_zero(d)) { + ++n_no_ei; + continue; + } + if (eimap_get(rl->extra_info_map, d)) { + ++n_have; + continue; + } + if (!download_status_is_ready(&sd->ei_dl_status, now)) { + ++n_delay; + continue; + } + if (digestmap_get(pending, d)) { + ++n_pending; + continue; + } + + const signed_descriptor_t *sd2 = router_get_by_extrainfo_digest(d); + if (sd2 != sd) { + if (sd2 != NULL) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + char d3[HEX_DIGEST_LEN+1], d4[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), sd2->identity_digest, DIGEST_LEN); + base16_encode(d3, sizeof(d3), d, DIGEST_LEN); + base16_encode(d4, sizeof(d3), sd2->extra_info_digest, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with mismatched " + "router_get_by_extrainfo_digest() value. This has ID %s " + "but the entry in the map has ID %s. This has EI digest " + "%s and the entry in the map has EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2, d3, d4); + } else { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), d, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with NULL " + "router_get_by_extrainfo_digest() value. This has ID %s " + "and EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2); + } + ++n_bogus[old_routers]; + continue; + } + smartlist_add(wanted, d); + } + } + digestmap_free(pending, NULL); + + log_info(LD_DIR, "Extrainfo download status: %d router with no ei, %d " + "with present ei, %d delaying, %d pending, %d downloadable, %d " + "bogus in routers, %d bogus in old_routers", + n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted), + n_bogus[0], n_bogus[1]); + + smartlist_shuffle(wanted); + + max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO); + for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) { + initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO, + wanted, i, i+max_dl_per_req, + PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH); + } + + smartlist_free(wanted); +} + +/** Reset the consensus and extra-info download failure count on all routers. + * When we get a new consensus, + * routers_update_status_from_consensus_networkstatus() will reset the + * download statuses on the descriptors in that consensus. + */ +void +router_reset_descriptor_download_failures(void) +{ + log_debug(LD_GENERAL, + "In router_reset_descriptor_download_failures()"); + + networkstatus_reset_download_failures(); + last_descriptor_download_attempted = 0; + if (!routerlist) + return; + /* We want to download *all* extra-info descriptors, not just those in + * the consensus we currently have (or are about to have) */ + SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri, + { + download_status_reset(&ri->cache_info.ei_dl_status); + }); + SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, + { + download_status_reset(&sd->ei_dl_status); + }); +} + +/** Any changes in a router descriptor's publication time larger than this are + * automatically non-cosmetic. */ +#define ROUTER_MAX_COSMETIC_TIME_DIFFERENCE (2*60*60) + +/** We allow uptime to vary from how much it ought to be by this much. */ +#define ROUTER_ALLOW_UPTIME_DRIFT (6*60*60) + +/** Return true iff the only differences between r1 and r2 are such that + * would not cause a recent (post 0.1.1.6) dirserver to republish. + */ +int +router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) +{ + time_t r1pub, r2pub; - long time_difference; ++ time_t time_difference; + tor_assert(r1 && r2); + + /* r1 should be the one that was published first. */ + if (r1->cache_info.published_on > r2->cache_info.published_on) { + const routerinfo_t *ri_tmp = r2; + r2 = r1; + r1 = ri_tmp; + } + + /* If any key fields differ, they're different. */ + if (r1->addr != r2->addr || + strcasecmp(r1->nickname, r2->nickname) || + r1->or_port != r2->or_port || + !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || + r1->ipv6_orport != r2->ipv6_orport || + r1->dir_port != r2->dir_port || + r1->purpose != r2->purpose || + r1->onion_pkey_len != r2->onion_pkey_len || + !tor_memeq(r1->onion_pkey, r2->onion_pkey, r1->onion_pkey_len) || + !crypto_pk_eq_keys(r1->identity_pkey, r2->identity_pkey) || + strcasecmp(r1->platform, r2->platform) || + (r1->contact_info && !r2->contact_info) || /* contact_info is optional */ + (!r1->contact_info && r2->contact_info) || + (r1->contact_info && r2->contact_info && + strcasecmp(r1->contact_info, r2->contact_info)) || + r1->is_hibernating != r2->is_hibernating || + ! addr_policies_eq(r1->exit_policy, r2->exit_policy) || + (r1->supports_tunnelled_dir_requests != + r2->supports_tunnelled_dir_requests)) + return 0; + if ((r1->declared_family == NULL) != (r2->declared_family == NULL)) + return 0; + if (r1->declared_family && r2->declared_family) { + int i, n; + if (smartlist_len(r1->declared_family)!=smartlist_len(r2->declared_family)) + return 0; + n = smartlist_len(r1->declared_family); + for (i=0; i < n; ++i) { + if (strcasecmp(smartlist_get(r1->declared_family, i), + smartlist_get(r2->declared_family, i))) + return 0; + } + } + + /* Did bandwidth change a lot? */ + if ((r1->bandwidthcapacity < r2->bandwidthcapacity/2) || + (r2->bandwidthcapacity < r1->bandwidthcapacity/2)) + return 0; + + /* Did the bandwidthrate or bandwidthburst change? */ + if ((r1->bandwidthrate != r2->bandwidthrate) || + (r1->bandwidthburst != r2->bandwidthburst)) + return 0; + + /* Did more than 12 hours pass? */ + if (r1->cache_info.published_on + ROUTER_MAX_COSMETIC_TIME_DIFFERENCE + < r2->cache_info.published_on) + return 0; + + /* Did uptime fail to increase by approximately the amount we would think, + * give or take some slop? */ + r1pub = r1->cache_info.published_on; + r2pub = r2->cache_info.published_on; - time_difference = labs(r2->uptime - (r1->uptime + (r2pub - r1pub))); ++ time_difference = r2->uptime - (r1->uptime + (r2pub - r1pub)); ++ if (time_difference < 0) ++ time_difference = - time_difference; + if (time_difference > ROUTER_ALLOW_UPTIME_DRIFT && + time_difference > r1->uptime * .05 && + time_difference > r2->uptime * .05) + return 0; + + /* Otherwise, the difference is cosmetic. */ + return 1; +} + +/** Check whether sd describes a router descriptor compatible with the + * extrainfo document ei. + * + * identity_pkey (which must also be provided) is RSA1024 identity key + * for the router. We use it to check the signature of the extrainfo document, + * if it has not already been checked. + * + * If no router is compatible with ei, ei should be + * dropped. Return 0 for "compatible", return 1 for "reject, and inform + * whoever uploaded ei, and return -1 for "reject silently.". If + * msg is present, set *msg to a description of the + * incompatibility (if any). + * + * Set the extrainfo_is_bogus field in sd if the digests matched + * but the extrainfo was nonetheless incompatible. + **/ +int +routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey, + extrainfo_t *ei, + signed_descriptor_t *sd, + const char **msg) +{ + int digest_matches, digest256_matches, r=1; + tor_assert(identity_pkey); + tor_assert(sd); + tor_assert(ei); + + if (ei->bad_sig) { + if (msg) *msg = "Extrainfo signature was bad, or signed with wrong key."; + return 1; + } + + digest_matches = tor_memeq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN); + /* Set digest256_matches to 1 if the digest is correct, or if no + * digest256 was in the ri. */ + digest256_matches = tor_memeq(ei->digest256, + sd->extra_info_digest256, DIGEST256_LEN); + digest256_matches |= + tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN); + + /* The identity must match exactly to have been generated at the same time + * by the same router. */ + if (tor_memneq(sd->identity_digest, + ei->cache_info.identity_digest, + DIGEST_LEN)) { + if (msg) *msg = "Extrainfo nickname or identity did not match routerinfo"; + goto err; /* different servers */ + } + + if (! tor_cert_opt_eq(sd->signing_key_cert, + ei->cache_info.signing_key_cert)) { + if (msg) *msg = "Extrainfo signing key cert didn't match routerinfo"; + goto err; /* different servers */ + } + + if (ei->pending_sig) { + char signed_digest[128]; + if (crypto_pk_public_checksig(identity_pkey, + signed_digest, sizeof(signed_digest), + ei->pending_sig, ei->pending_sig_len) != DIGEST_LEN || + tor_memneq(signed_digest, ei->cache_info.signed_descriptor_digest, + DIGEST_LEN)) { + ei->bad_sig = 1; + tor_free(ei->pending_sig); + if (msg) *msg = "Extrainfo signature bad, or signed with wrong key"; + goto err; /* Bad signature, or no match. */ + } + + ei->cache_info.send_unencrypted = sd->send_unencrypted; + tor_free(ei->pending_sig); + } + + if (ei->cache_info.published_on < sd->published_on) { + if (msg) *msg = "Extrainfo published time did not match routerdesc"; + goto err; + } else if (ei->cache_info.published_on > sd->published_on) { + if (msg) *msg = "Extrainfo published time did not match routerdesc"; + r = -1; + goto err; + } + + if (!digest256_matches && !digest_matches) { + if (msg) *msg = "Neither digest256 or digest matched " + "digest from routerdesc"; + goto err; + } + + if (!digest256_matches) { + if (msg) *msg = "Extrainfo digest did not match digest256 from routerdesc"; + goto err; /* Digest doesn't match declared value. */ + } + + if (!digest_matches) { + if (msg) *msg = "Extrainfo digest did not match value from routerdesc"; + goto err; /* Digest doesn't match declared value. */ + } + + return 0; + err: + if (digest_matches) { + /* This signature was okay, and the digest was right: This is indeed the + * corresponding extrainfo. But insanely, it doesn't match the routerinfo + * that lists it. Don't try to fetch this one again. */ + sd->extrainfo_is_bogus = 1; + } + + return r; +} + +/* Does ri have a valid ntor onion key? + * Valid ntor onion keys exist and have at least one non-zero byte. */ +int +routerinfo_has_curve25519_onion_key(const routerinfo_t *ri) +{ + if (!ri) { + return 0; + } + + if (!ri->onion_curve25519_pkey) { + return 0; + } + + if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key, + CURVE25519_PUBKEY_LEN)) { + return 0; + } + + return 1; +} + +/* Is rs running a tor version known to support EXTEND2 cells? + * If allow_unknown_versions is true, return true if we can't tell + * (from a versions line or a protocols line) whether it supports extend2 + * cells. + * Otherwise, return false if the version is unknown. */ +int +routerstatus_version_supports_extend2_cells(const routerstatus_t *rs, + int allow_unknown_versions) +{ + if (!rs) { + return allow_unknown_versions; + } + + if (!rs->pv.protocols_known) { + return allow_unknown_versions; + } + + return rs->pv.supports_extend2_cells; +} + +/** Assert that the internal representation of rl is + * self-consistent. */ +void +routerlist_assert_ok(const routerlist_t *rl) +{ + routerinfo_t *r2; + signed_descriptor_t *sd2; + if (!rl) + return; + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, r) { + r2 = rimap_get(rl->identity_map, r->cache_info.identity_digest); + tor_assert(r == r2); + sd2 = sdmap_get(rl->desc_digest_map, + r->cache_info.signed_descriptor_digest); + tor_assert(&(r->cache_info) == sd2); + tor_assert(r->cache_info.routerlist_index == r_sl_idx); + /* XXXX + * + * Hoo boy. We need to fix this one, and the fix is a bit tricky, so + * commenting this out is just a band-aid. + * + * The problem is that, although well-behaved router descriptors + * should never have the same value for their extra_info_digest, it's + * possible for ill-behaved routers to claim whatever they like there. + * + * The real answer is to trash desc_by_eid_map and instead have + * something that indicates for a given extra-info digest we want, + * what its download status is. We'll do that as a part of routerlist + * refactoring once consensus directories are in. For now, + * this rep violation is probably harmless: an adversary can make us + * reset our retry count for an extrainfo, but that's not the end + * of the world. Changing the representation in 0.2.0.x would just + * destabilize the codebase. + if (!tor_digest_is_zero(r->cache_info.extra_info_digest)) { + signed_descriptor_t *sd3 = + sdmap_get(rl->desc_by_eid_map, r->cache_info.extra_info_digest); + tor_assert(sd3 == &(r->cache_info)); + } + */ + } SMARTLIST_FOREACH_END(r); + SMARTLIST_FOREACH_BEGIN(rl->old_routers, signed_descriptor_t *, sd) { + r2 = rimap_get(rl->identity_map, sd->identity_digest); + tor_assert(!r2 || sd != &(r2->cache_info)); + sd2 = sdmap_get(rl->desc_digest_map, sd->signed_descriptor_digest); + tor_assert(sd == sd2); + tor_assert(sd->routerlist_index == sd_sl_idx); + /* XXXX see above. + if (!tor_digest_is_zero(sd->extra_info_digest)) { + signed_descriptor_t *sd3 = + sdmap_get(rl->desc_by_eid_map, sd->extra_info_digest); + tor_assert(sd3 == sd); + } + */ + } SMARTLIST_FOREACH_END(sd); + + RIMAP_FOREACH(rl->identity_map, d, r) { + tor_assert(tor_memeq(r->cache_info.identity_digest, d, DIGEST_LEN)); + } DIGESTMAP_FOREACH_END; + SDMAP_FOREACH(rl->desc_digest_map, d, sd) { + tor_assert(tor_memeq(sd->signed_descriptor_digest, d, DIGEST_LEN)); + } DIGESTMAP_FOREACH_END; + SDMAP_FOREACH(rl->desc_by_eid_map, d, sd) { + tor_assert(!tor_digest_is_zero(d)); + tor_assert(sd); + tor_assert(tor_memeq(sd->extra_info_digest, d, DIGEST_LEN)); + } DIGESTMAP_FOREACH_END; + EIMAP_FOREACH(rl->extra_info_map, d, ei) { + signed_descriptor_t *sd; + tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest, + d, DIGEST_LEN)); + sd = sdmap_get(rl->desc_by_eid_map, + ei->cache_info.signed_descriptor_digest); + // tor_assert(sd); // XXXX see above + if (sd) { + tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN)); + } + } DIGESTMAP_FOREACH_END; +} + +/** Allocate and return a new string representing the contact info + * and platform string for router, + * surrounded by quotes and using standard C escapes. + * + * THIS FUNCTION IS NOT REENTRANT. Don't call it from outside the main + * thread. Also, each call invalidates the last-returned value, so don't + * try log_warn(LD_GENERAL, "%s %s", esc_router_info(a), esc_router_info(b)); + * + * If router is NULL, it just frees its internal memory and returns. + */ +const char * +esc_router_info(const routerinfo_t *router) +{ + static char *info=NULL; + char *esc_contact, *esc_platform; + tor_free(info); + + if (!router) + return NULL; /* we're exiting; just free the memory we use */ + + esc_contact = esc_for_log(router->contact_info); + esc_platform = esc_for_log(router->platform); + + tor_asprintf(&info, "Contact %s, Platform %s", esc_contact, esc_platform); + tor_free(esc_contact); + tor_free(esc_platform); + + return info; +} + +/** Helper for sorting: compare two routerinfos by their identity + * digest. */ +static int +compare_routerinfo_by_id_digest_(const void **a, const void **b) +{ + routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; + return fast_memcmp(first->cache_info.identity_digest, + second->cache_info.identity_digest, + DIGEST_LEN); +} + +/** Sort a list of routerinfo_t in ascending order of identity digest. */ +void +routers_sort_by_identity(smartlist_t *routers) +{ + smartlist_sort(routers, compare_routerinfo_by_id_digest_); +} + +/** Called when we change a node set, or when we reload the geoip IPv4 list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ +void +refresh_all_country_info(void) +{ + const or_options_t *options = get_options(); + + if (options->EntryNodes) + routerset_refresh_countries(options->EntryNodes); + if (options->ExitNodes) + routerset_refresh_countries(options->ExitNodes); + if (options->ExcludeNodes) + routerset_refresh_countries(options->ExcludeNodes); + if (options->ExcludeExitNodes) + routerset_refresh_countries(options->ExcludeExitNodes); + if (options->ExcludeExitNodesUnion_) + routerset_refresh_countries(options->ExcludeExitNodesUnion_); + + nodelist_refresh_countries(); +}

1 0