commit 94b04d6c64ec998a9117d65a156888fa3af188e5 Merge: 1c62adb65 81f422332 Author: Nick Mathewson nickm@torproject.org Date: Tue Sep 4 10:44:36 2018 -0400

Merge branch 'bug24104_029_squashed'

changes/bug24104 | 4 ++ src/feature/relay/router.c | 27 +++++++-- src/feature/stats/rephist.c | 4 +- src/feature/stats/rephist.h | 2 +- src/test/log_test_helpers.c | 20 +++++++ src/test/log_test_helpers.h | 5 ++ src/test/test_router.c | 130 ++++++++++++++++++++++++++++++++++++++++++-- 7 files changed, 180 insertions(+), 12 deletions(-)

diff --cc src/feature/relay/router.c index ad97d534c,000000000..b9a930dbe mode 100644,000000..100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@@ -1,3879 -1,0 +1,3896 @@@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define ROUTER_PRIVATE + +#include "core/or/or.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "feature/control/control.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/crypt_ops/crypto_curve25519.h" +#include "feature/dircache/directory.h" +#include "feature/dircache/dirserv.h" +#include "feature/relay/dns.h" +#include "feature/stats/geoip.h" +#include "feature/hibernate/hibernate.h" +#include "core/mainloop/main.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "core/or/policies.h" +#include "core/or/protover.h" +#include "core/or/relay.h" +#include "feature/stats/rephist.h" +#include "feature/relay/router.h" +#include "feature/relay/routerkeys.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerparse.h" +#include "app/config/statefile.h" +#include "feature/nodelist/torcert.h" +#include "feature/client/transports.h" +#include "feature/nodelist/routerset.h" + +#include "feature/dirauth/mode.h" + +#include "feature/nodelist/authority_cert_st.h" +#include "core/or/crypt_path_st.h" +#include "feature/dircommon/dir_connection_st.h" +#include "feature/dirclient/dir_server_st.h" +#include "core/or/extend_info_st.h" +#include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/origin_circuit_st.h" +#include "app/config/or_state_st.h" +#include "core/or/port_cfg_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "lib/osinfo/uname.h" +#include "lib/tls/tortls.h" +#include "lib/encoding/confline.h" +#include "lib/crypt_ops/crypto_format.h" + +/** + * \file router.c + * \brief Miscellaneous relay functionality, including RSA key maintenance, + * generating and uploading server descriptors, picking an address to + * advertise, and so on. + * + * This module handles the job of deciding whether we are a Tor relay, and if + * so what kind. (Mostly through functions like server_mode() that inspect an + * or_options_t, but in some cases based on our own capabilities, such as when + * we are deciding whether to be a directory cache in + * router_has_bandwidth_to_be_dirserver().) + * + * Also in this module are the functions to generate our own routerinfo_t and + * extrainfo_t, and to encode those to signed strings for upload to the + * directory authorities. + * + * This module also handles key maintenance for RSA and Curve25519-ntor keys, + * and for our TLS context. (These functions should eventually move to + * routerkeys.c along with the code that handles Ed25519 keys now.) + **/ + +/************************************************************/ + +/***** + * Key management: ORs only. + *****/ + +/** Private keys for this OR. There is also an SSL key managed by tortls.c. + */ +static tor_mutex_t *key_lock=NULL; +static time_t onionkey_set_at=0; /**< When was onionkey last changed? */ +/** Current private onionskin decryption key: used to decode CREATE cells. */ +static crypto_pk_t *onionkey=NULL; +/** Previous private onionskin decryption key: used to decode CREATE cells + * generated by clients that have an older version of our descriptor. */ +static crypto_pk_t *lastonionkey=NULL; +/** Current private ntor secret key: used to perform the ntor handshake. */ +static curve25519_keypair_t curve25519_onion_key; +/** Previous private ntor secret key: used to perform the ntor handshake + * with clients that have an older version of our descriptor. */ +static curve25519_keypair_t last_curve25519_onion_key; +/** Private server "identity key": used to sign directory info and TLS + * certificates. Never changes. */ +static crypto_pk_t *server_identitykey=NULL; +/** Digest of server_identitykey. */ +static char server_identitykey_digest[DIGEST_LEN]; +/** Private client "identity key": used to sign bridges' and clients' + * outbound TLS certificates. Regenerated on startup and on IP address + * change. */ +static crypto_pk_t *client_identitykey=NULL; +/** Signing key used for v3 directory material; only set for authorities. */ +static crypto_pk_t *authority_signing_key = NULL; +/** Key certificate to authenticate v3 directory material; only set for + * authorities. */ +static authority_cert_t *authority_key_certificate = NULL; + +/** For emergency V3 authority key migration: An extra signing key that we use + * with our old (obsolete) identity key for a while. */ +static crypto_pk_t *legacy_signing_key = NULL; +/** For emergency V3 authority key migration: An extra certificate to + * authenticate legacy_signing_key with our obsolete identity key.*/ +static authority_cert_t *legacy_key_certificate = NULL; + +/* (Note that v3 authorities also have a separate "authority identity key", + * but this key is never actually loaded by the Tor process. Instead, it's + * used by tor-gencert to sign new signing keys and make new key + * certificates. */ + +const char *format_node_description(char *buf, + const char *id_digest, + int is_named, + const char *nickname, + const tor_addr_t *addr, + uint32_t addr32h); + +/** Return a readonly string with human readable description + * of <b>err</b>. + */ +const char * +routerinfo_err_to_string(int err) +{ + switch (err) { + case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR: + return "No known exit address yet"; + case TOR_ROUTERINFO_ERROR_CANNOT_PARSE: + return "Cannot parse descriptor"; + case TOR_ROUTERINFO_ERROR_NOT_A_SERVER: + return "Not running in server mode"; + case TOR_ROUTERINFO_ERROR_DIGEST_FAILED: + return "Key digest failed"; + case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE: + return "Cannot generate descriptor"; + case TOR_ROUTERINFO_ERROR_DESC_REBUILDING: + return "Descriptor still rebuilding - not ready yet"; + } + + log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err); + tor_assert_unreached(); + + return "Unknown error"; +} + +/** Return true if we expect given error to be transient. + * Return false otherwise. + */ +int +routerinfo_err_is_transient(int err) +{ + /** + * For simplicity, we consider all errors other than + * "not a server" transient - see discussion on + * https://trac.torproject.org/projects/tor/ticket/27034 + */ + return err != TOR_ROUTERINFO_ERROR_NOT_A_SERVER; +} + +/** Replace the current onion key with <b>k</b>. Does not affect + * lastonionkey; to update lastonionkey correctly, call rotate_onion_key(). + */ +static void +set_onion_key(crypto_pk_t *k) +{ + if (onionkey && crypto_pk_eq_keys(onionkey, k)) { + /* k is already our onion key; free it and return */ + crypto_pk_free(k); + return; + } + tor_mutex_acquire(key_lock); + crypto_pk_free(onionkey); + onionkey = k; + tor_mutex_release(key_lock); + mark_my_descriptor_dirty("set onion key"); +} + +/** Return the current onion key. Requires that the onion key has been + * loaded or generated. */ +crypto_pk_t * +get_onion_key(void) +{ + tor_assert(onionkey); + return onionkey; +} + +/** Store a full copy of the current onion key into *<b>key</b>, and a full + * copy of the most recent onion key into *<b>last</b>. Store NULL into + * a pointer if the corresponding key does not exist. + */ +void +dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last) +{ + tor_assert(key); + tor_assert(last); + tor_mutex_acquire(key_lock); + if (onionkey) + *key = crypto_pk_copy_full(onionkey); + else + *key = NULL; + if (lastonionkey) + *last = crypto_pk_copy_full(lastonionkey); + else + *last = NULL; + tor_mutex_release(key_lock); +} + +/** Expire our old set of onion keys. This is done by setting + * last_curve25519_onion_key and lastonionkey to all zero's and NULL + * respectively. + * + * This function does not perform any grace period checks for the old onion + * keys. + */ +void +expire_old_onion_keys(void) +{ + char *fname = NULL; + + tor_mutex_acquire(key_lock); + + /* Free lastonionkey and set it to NULL. */ + if (lastonionkey) { + crypto_pk_free(lastonionkey); + lastonionkey = NULL; + } + + /* We zero out the keypair. See the tor_mem_is_zero() check made in + * construct_ntor_key_map() below. */ + memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); + + tor_mutex_release(key_lock); + + fname = get_keydir_fname("secret_onion_key.old"); + if (file_status(fname) == FN_FILE) { + if (tor_unlink(fname) != 0) { + log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s", + fname, strerror(errno)); + } + } + tor_free(fname); + + fname = get_keydir_fname("secret_onion_key_ntor.old"); + if (file_status(fname) == FN_FILE) { + if (tor_unlink(fname) != 0) { + log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s", + fname, strerror(errno)); + } + } + tor_free(fname); +} + +/** Return the current secret onion key for the ntor handshake. Must only + * be called from the main thread. */ +static const curve25519_keypair_t * +get_current_curve25519_keypair(void) +{ + return &curve25519_onion_key; +} +/** Return a map from KEYID (the key itself) to keypairs for use in the ntor + * handshake. Must only be called from the main thread. */ +di_digest256_map_t * +construct_ntor_key_map(void) +{ + di_digest256_map_t *m = NULL; + + if (!tor_mem_is_zero((const char*) + curve25519_onion_key.pubkey.public_key, + CURVE25519_PUBKEY_LEN)) { + dimap_add_entry(&m, + curve25519_onion_key.pubkey.public_key, + tor_memdup(&curve25519_onion_key, + sizeof(curve25519_keypair_t))); + } + if (!tor_mem_is_zero((const char*) + last_curve25519_onion_key.pubkey.public_key, + CURVE25519_PUBKEY_LEN)) { + dimap_add_entry(&m, + last_curve25519_onion_key.pubkey.public_key, + tor_memdup(&last_curve25519_onion_key, + sizeof(curve25519_keypair_t))); + } + + return m; +} +/** Helper used to deallocate a di_digest256_map_t returned by + * construct_ntor_key_map. */ +static void +ntor_key_map_free_helper(void *arg) +{ + curve25519_keypair_t *k = arg; + memwipe(k, 0, sizeof(*k)); + tor_free(k); +} +/** Release all storage from a keymap returned by construct_ntor_key_map. */ +void +ntor_key_map_free_(di_digest256_map_t *map) +{ + if (!map) + return; + dimap_free(map, ntor_key_map_free_helper); +} + +/** Return the time when the onion key was last set. This is either the time + * when the process launched, or the time of the most recent key rotation since + * the process launched. + */ +time_t +get_onion_key_set_at(void) +{ + return onionkey_set_at; +} + +/** Set the current server identity key to <b>k</b>. + */ +void +set_server_identity_key(crypto_pk_t *k) +{ + crypto_pk_free(server_identitykey); + server_identitykey = k; + if (crypto_pk_get_digest(server_identitykey, + server_identitykey_digest) < 0) { + log_err(LD_BUG, "Couldn't compute our own identity key digest."); + tor_assert(0); + } +} + +/** Make sure that we have set up our identity keys to match or not match as + * appropriate, and die with an assertion if we have not. */ +static void +assert_identity_keys_ok(void) +{ + if (1) + return; + tor_assert(client_identitykey); + if (public_server_mode(get_options())) { + /* assert that we have set the client and server keys to be equal */ + tor_assert(server_identitykey); + tor_assert(crypto_pk_eq_keys(client_identitykey, server_identitykey)); + } else { + /* assert that we have set the client and server keys to be unequal */ + if (server_identitykey) + tor_assert(!crypto_pk_eq_keys(client_identitykey, server_identitykey)); + } +} + +/** Returns the current server identity key; requires that the key has + * been set, and that we are running as a Tor server. + */ +crypto_pk_t * +get_server_identity_key(void) +{ + tor_assert(server_identitykey); + tor_assert(server_mode(get_options())); + assert_identity_keys_ok(); + return server_identitykey; +} + +/** Return true iff we are a server and the server identity key + * has been set. */ +int +server_identity_key_is_set(void) +{ + return server_mode(get_options()) && server_identitykey != NULL; +} + +/** Set the current client identity key to <b>k</b>. + */ +void +set_client_identity_key(crypto_pk_t *k) +{ + crypto_pk_free(client_identitykey); + client_identitykey = k; +} + +/** Returns the current client identity key for use on outgoing TLS + * connections; requires that the key has been set. + */ +crypto_pk_t * +get_tlsclient_identity_key(void) +{ + tor_assert(client_identitykey); + assert_identity_keys_ok(); + return client_identitykey; +} + +/** Return true iff the client identity key has been set. */ +int +client_identity_key_is_set(void) +{ + return client_identitykey != NULL; +} + +/** Return the key certificate for this v3 (voting) authority, or NULL + * if we have no such certificate. */ +MOCK_IMPL(authority_cert_t *, +get_my_v3_authority_cert, (void)) +{ + return authority_key_certificate; +} + +/** Return the v3 signing key for this v3 (voting) authority, or NULL + * if we have no such key. */ +crypto_pk_t * +get_my_v3_authority_signing_key(void) +{ + return authority_signing_key; +} + +/** If we're an authority, and we're using a legacy authority identity key for + * emergency migration purposes, return the certificate associated with that + * key. */ +authority_cert_t * +get_my_v3_legacy_cert(void) +{ + return legacy_key_certificate; +} + +/** If we're an authority, and we're using a legacy authority identity key for + * emergency migration purposes, return that key. */ +crypto_pk_t * +get_my_v3_legacy_signing_key(void) +{ + return legacy_signing_key; +} + +/** Replace the previous onion key with the current onion key, and generate + * a new previous onion key. Immediately after calling this function, + * the OR should: + * - schedule all previous cpuworkers to shut down _after_ processing + * pending work. (This will cause fresh cpuworkers to be generated.) + * - generate and upload a fresh routerinfo. + */ +void +rotate_onion_key(void) +{ + char *fname, *fname_prev; + crypto_pk_t *prkey = NULL; + or_state_t *state = get_or_state(); + curve25519_keypair_t new_curve25519_keypair; + time_t now; + fname = get_keydir_fname("secret_onion_key"); + fname_prev = get_keydir_fname("secret_onion_key.old"); + /* There isn't much point replacing an old key with an empty file */ + if (file_status(fname) == FN_FILE) { + if (replace_file(fname, fname_prev)) + goto error; + } + if (!(prkey = crypto_pk_new())) { + log_err(LD_GENERAL,"Error constructing rotated onion key"); + goto error; + } + if (crypto_pk_generate_key(prkey)) { + log_err(LD_BUG,"Error generating onion key"); + goto error; + } + if (crypto_pk_write_private_key_to_filename(prkey, fname)) { + log_err(LD_FS,"Couldn't write generated onion key to "%s".", fname); + goto error; + } + tor_free(fname); + tor_free(fname_prev); + fname = get_keydir_fname("secret_onion_key_ntor"); + fname_prev = get_keydir_fname("secret_onion_key_ntor.old"); + if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0) + goto error; + /* There isn't much point replacing an old key with an empty file */ + if (file_status(fname) == FN_FILE) { + if (replace_file(fname, fname_prev)) + goto error; + } + if (curve25519_keypair_write_to_file(&new_curve25519_keypair, fname, + "onion") < 0) { + log_err(LD_FS,"Couldn't write curve25519 onion key to "%s".",fname); + goto error; + } + log_info(LD_GENERAL, "Rotating onion key"); + tor_mutex_acquire(key_lock); + crypto_pk_free(lastonionkey); + lastonionkey = onionkey; + onionkey = prkey; + memcpy(&last_curve25519_onion_key, &curve25519_onion_key, + sizeof(curve25519_keypair_t)); + memcpy(&curve25519_onion_key, &new_curve25519_keypair, + sizeof(curve25519_keypair_t)); + now = time(NULL); + state->LastRotatedOnionKey = onionkey_set_at = now; + tor_mutex_release(key_lock); + mark_my_descriptor_dirty("rotated onion key"); + or_state_mark_dirty(state, get_options()->AvoidDiskWrites ? now+3600 : 0); + goto done; + error: + log_warn(LD_GENERAL, "Couldn't rotate onion key."); + if (prkey) + crypto_pk_free(prkey); + done: + memwipe(&new_curve25519_keypair, 0, sizeof(new_curve25519_keypair)); + tor_free(fname); + tor_free(fname_prev); +} + +/** Log greeting message that points to new relay lifecycle document the + * first time this function has been called. + */ +static void +log_new_relay_greeting(void) +{ + static int already_logged = 0; + + if (already_logged) + return; + + tor_log(LOG_NOTICE, LD_GENERAL, "You are running a new relay. " + "Thanks for helping the Tor network! If you wish to know " + "what will happen in the upcoming weeks regarding its usage, " + "have a look at https://blog.torproject.org/blog/lifecycle-of" + "-a-new-relay"); + + already_logged = 1; +} + +/** Try to read an RSA key from <b>fname</b>. If <b>fname</b> doesn't exist + * and <b>generate</b> is true, create a new RSA key and save it in + * <b>fname</b>. Return the read/created key, or NULL on error. Log all + * errors at level <b>severity</b>. If <b>log_greeting</b> is non-zero and a + * new key was created, log_new_relay_greeting() is called. + */ +crypto_pk_t * +init_key_from_file(const char *fname, int generate, int severity, + int log_greeting) +{ + crypto_pk_t *prkey = NULL; + + if (!(prkey = crypto_pk_new())) { + tor_log(severity, LD_GENERAL,"Error constructing key"); + goto error; + } + + switch (file_status(fname)) { + case FN_DIR: + case FN_ERROR: + tor_log(severity, LD_FS,"Can't read key from "%s"", fname); + goto error; + /* treat empty key files as if the file doesn't exist, and, + * if generate is set, replace the empty file in + * crypto_pk_write_private_key_to_filename() */ + case FN_NOENT: + case FN_EMPTY: + if (generate) { + if (!have_lockfile()) { + if (try_locking(get_options(), 0)<0) { + /* Make sure that --list-fingerprint only creates new keys + * if there is no possibility for a deadlock. */ + tor_log(severity, LD_FS, "Another Tor process has locked "%s". " + "Not writing any new keys.", fname); + /*XXXX The 'other process' might make a key in a second or two; + * maybe we should wait for it. */ + goto error; + } + } + log_info(LD_GENERAL, "No key found in "%s"; generating fresh key.", + fname); + if (crypto_pk_generate_key(prkey)) { + tor_log(severity, LD_GENERAL,"Error generating onion key"); + goto error; + } + if (crypto_pk_check_key(prkey) <= 0) { + tor_log(severity, LD_GENERAL,"Generated key seems invalid"); + goto error; + } + log_info(LD_GENERAL, "Generated key seems valid"); + if (log_greeting) { + log_new_relay_greeting(); + } + if (crypto_pk_write_private_key_to_filename(prkey, fname)) { + tor_log(severity, LD_FS, + "Couldn't write generated key to "%s".", fname); + goto error; + } + } else { + tor_log(severity, LD_GENERAL, "No key found in "%s"", fname); + goto error; + } + return prkey; + case FN_FILE: + if (crypto_pk_read_private_key_from_filename(prkey, fname)) { + tor_log(severity, LD_GENERAL,"Error loading private key."); + goto error; + } + return prkey; + default: + tor_assert(0); + } + + error: + if (prkey) + crypto_pk_free(prkey); + return NULL; +} + +/** Load a curve25519 keypair from the file <b>fname</b>, writing it into + * <b>keys_out</b>. If the file isn't found, or is empty, and <b>generate</b> + * is true, create a new keypair and write it into the file. If there are + * errors, log them at level <b>severity</b>. Generate files using <b>tag</b> + * in their ASCII wrapper. */ +static int +init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, + const char *fname, + int generate, + int severity, + const char *tag) +{ + switch (file_status(fname)) { + case FN_DIR: + case FN_ERROR: + tor_log(severity, LD_FS,"Can't read key from "%s"", fname); + goto error; + /* treat empty key files as if the file doesn't exist, and, if generate + * is set, replace the empty file in curve25519_keypair_write_to_file() */ + case FN_NOENT: + case FN_EMPTY: + if (generate) { + if (!have_lockfile()) { + if (try_locking(get_options(), 0)<0) { + /* Make sure that --list-fingerprint only creates new keys + * if there is no possibility for a deadlock. */ + tor_log(severity, LD_FS, "Another Tor process has locked "%s". " + "Not writing any new keys.", fname); + /*XXXX The 'other process' might make a key in a second or two; + * maybe we should wait for it. */ + goto error; + } + } + log_info(LD_GENERAL, "No key found in "%s"; generating fresh key.", + fname); + if (curve25519_keypair_generate(keys_out, 1) < 0) + goto error; + if (curve25519_keypair_write_to_file(keys_out, fname, tag)<0) { + tor_log(severity, LD_FS, + "Couldn't write generated key to "%s".", fname); + memwipe(keys_out, 0, sizeof(*keys_out)); + goto error; + } + } else { + log_info(LD_GENERAL, "No key found in "%s"", fname); + } + return 0; + case FN_FILE: + { + char *tag_in=NULL; + if (curve25519_keypair_read_from_file(keys_out, &tag_in, fname) < 0) { + tor_log(severity, LD_GENERAL,"Error loading private key."); + tor_free(tag_in); + goto error; + } + if (!tag_in || strcmp(tag_in, tag)) { + tor_log(severity, LD_GENERAL,"Unexpected tag %s on private key.", + escaped(tag_in)); + tor_free(tag_in); + goto error; + } + tor_free(tag_in); + return 0; + } + default: + tor_assert(0); + } + + error: + return -1; +} + +/** Try to load the vote-signing private key and certificate for being a v3 + * directory authority, and make sure they match. If <b>legacy</b>, load a + * legacy key/cert set for emergency key migration; otherwise load the regular + * key/cert set. On success, store them into *<b>key_out</b> and + * *<b>cert_out</b> respectively, and return 0. On failure, return -1. */ +static int +load_authority_keyset(int legacy, crypto_pk_t **key_out, + authority_cert_t **cert_out) +{ + int r = -1; + char *fname = NULL, *cert = NULL; + const char *eos = NULL; + crypto_pk_t *signing_key = NULL; + authority_cert_t *parsed = NULL; + + fname = get_keydir_fname( + legacy ? "legacy_signing_key" : "authority_signing_key"); + signing_key = init_key_from_file(fname, 0, LOG_ERR, 0); + if (!signing_key) { + log_warn(LD_DIR, "No version 3 directory key found in %s", fname); + goto done; + } + tor_free(fname); + fname = get_keydir_fname( + legacy ? "legacy_certificate" : "authority_certificate"); + cert = read_file_to_str(fname, 0, NULL); + if (!cert) { + log_warn(LD_DIR, "Signing key found, but no certificate found in %s", + fname); + goto done; + } + parsed = authority_cert_parse_from_string(cert, &eos); + if (!parsed) { + log_warn(LD_DIR, "Unable to parse certificate in %s", fname); + goto done; + } + if (!crypto_pk_eq_keys(signing_key, parsed->signing_key)) { + log_warn(LD_DIR, "Stored signing key does not match signing key in " + "certificate"); + goto done; + } + + crypto_pk_free(*key_out); + authority_cert_free(*cert_out); + + *key_out = signing_key; + *cert_out = parsed; + r = 0; + signing_key = NULL; + parsed = NULL; + + done: + tor_free(fname); + tor_free(cert); + crypto_pk_free(signing_key); + authority_cert_free(parsed); + return r; +} + +/** Load the v3 (voting) authority signing key and certificate, if they are + * present. Return -1 if anything is missing, mismatched, or unloadable; + * return 0 on success. */ +static int +init_v3_authority_keys(void) +{ + if (load_authority_keyset(0, &authority_signing_key, + &authority_key_certificate)<0) + return -1; + + if (get_options()->V3AuthUseLegacyKey && + load_authority_keyset(1, &legacy_signing_key, + &legacy_key_certificate)<0) + return -1; + + return 0; +} + +/** If we're a v3 authority, check whether we have a certificate that's + * likely to expire soon. Warn if we do, but not too often. */ +void +v3_authority_check_key_expiry(void) +{ + time_t now, expires; + static time_t last_warned = 0; + int badness, time_left, warn_interval; + if (!authdir_mode_v3(get_options()) || !authority_key_certificate) + return; + + now = time(NULL); + expires = authority_key_certificate->expires; + time_left = (int)( expires - now ); + if (time_left <= 0) { + badness = LOG_ERR; + warn_interval = 60*60; + } else if (time_left <= 24*60*60) { + badness = LOG_WARN; + warn_interval = 60*60; + } else if (time_left <= 24*60*60*7) { + badness = LOG_WARN; + warn_interval = 24*60*60; + } else if (time_left <= 24*60*60*30) { + badness = LOG_WARN; + warn_interval = 24*60*60*5; + } else { + return; + } + + if (last_warned + warn_interval > now) + return; + + if (time_left <= 0) { + tor_log(badness, LD_DIR, "Your v3 authority certificate has expired." + " Generate a new one NOW."); + } else if (time_left <= 24*60*60) { + tor_log(badness, LD_DIR, "Your v3 authority certificate expires in %d " + "hours; Generate a new one NOW.", time_left/(60*60)); + } else { + tor_log(badness, LD_DIR, "Your v3 authority certificate expires in %d " + "days; Generate a new one soon.", time_left/(24*60*60)); + } + last_warned = now; +} + +/** Get the lifetime of an onion key in days. This value is defined by the + * network consesus parameter "onion-key-rotation-days". Always returns a value + * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and + * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>. + */ +static int +get_onion_key_rotation_days_(void) +{ + return networkstatus_get_param(NULL, + "onion-key-rotation-days", + DEFAULT_ONION_KEY_LIFETIME_DAYS, + MIN_ONION_KEY_LIFETIME_DAYS, + MAX_ONION_KEY_LIFETIME_DAYS); +} + +/** Get the current lifetime of an onion key in seconds. This value is defined + * by the network consesus parameter "onion-key-rotation-days", but the value + * is converted to seconds. + */ +int +get_onion_key_lifetime(void) +{ + return get_onion_key_rotation_days_()*24*60*60; +} + +/** Get the grace period of an onion key in seconds. This value is defined by + * the network consesus parameter "onion-key-grace-period-days", but the value + * is converted to seconds. + */ +int +get_onion_key_grace_period(void) +{ + int grace_period; + grace_period = networkstatus_get_param(NULL, + "onion-key-grace-period-days", + DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS, + MIN_ONION_KEY_GRACE_PERIOD_DAYS, + get_onion_key_rotation_days_()); + return grace_period*24*60*60; +} + +/** Set up Tor's TLS contexts, based on our configuration and keys. Return 0 + * on success, and -1 on failure. */ +int +router_initialize_tls_context(void) +{ + unsigned int flags = 0; + const or_options_t *options = get_options(); + int lifetime = options->SSLKeyLifetime; + if (public_server_mode(options)) + flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER; + if (!lifetime) { /* we should guess a good ssl cert lifetime */ + + /* choose between 5 and 365 days, and round to the day */ + unsigned int five_days = 5*24*3600; + unsigned int one_year = 365*24*3600; + lifetime = crypto_rand_int_range(five_days, one_year); + lifetime -= lifetime % (24*3600); + + if (crypto_rand_int(2)) { + /* Half the time we expire at midnight, and half the time we expire + * one second before midnight. (Some CAs wobble their expiry times a + * bit in practice, perhaps to reduce collision attacks; see ticket + * 8443 for details about observed certs in the wild.) */ + lifetime--; + } + } + + /* It's ok to pass lifetime in as an unsigned int, since + * config_parse_interval() checked it. */ + return tor_tls_context_init(flags, + get_tlsclient_identity_key(), + server_mode(options) ? + get_server_identity_key() : NULL, + (unsigned int)lifetime); +} + +/** Compute fingerprint (or hashed fingerprint if hashed is 1) and write + * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or + * -1 if Tor should die, + */ +STATIC int +router_write_fingerprint(int hashed) +{ + char *keydir = NULL, *cp = NULL; + const char *fname = hashed ? "hashed-fingerprint" : + "fingerprint"; + char fingerprint[FINGERPRINT_LEN+1]; + const or_options_t *options = get_options(); + char *fingerprint_line = NULL; + int result = -1; + + keydir = get_datadir_fname(fname); + log_info(LD_GENERAL,"Dumping %sfingerprint to "%s"...", + hashed ? "hashed " : "", keydir); + if (!hashed) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { + log_err(LD_GENERAL,"Error computing fingerprint"); + goto done; + } + } else { + if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(), + fingerprint) < 0) { + log_err(LD_GENERAL,"Error computing hashed fingerprint"); + goto done; + } + } + + tor_asprintf(&fingerprint_line, "%s %s\n", options->Nickname, fingerprint); + + /* Check whether we need to write the (hashed-)fingerprint file. */ + + cp = read_file_to_str(keydir, RFTS_IGNORE_MISSING, NULL); + if (!cp || strcmp(cp, fingerprint_line)) { + if (write_str_to_file(keydir, fingerprint_line, 0)) { + log_err(LD_FS, "Error writing %sfingerprint line to file", + hashed ? "hashed " : ""); + goto done; + } + } + + log_notice(LD_GENERAL, "Your Tor %s identity key fingerprint is '%s %s'", + hashed ? "bridge's hashed" : "server's", options->Nickname, + fingerprint); + + result = 0; + done: + tor_free(cp); + tor_free(keydir); + tor_free(fingerprint_line); + return result; +} + +static int +init_keys_common(void) +{ + if (!key_lock) + key_lock = tor_mutex_new(); + + /* There are a couple of paths that put us here before we've asked + * openssl to initialize itself. */ + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { + log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); + return -1; + } + + return 0; +} + +int +init_keys_client(void) +{ + crypto_pk_t *prkey; + if (init_keys_common() < 0) + return -1; + + if (!(prkey = crypto_pk_new())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_pk_free(prkey); + return -1; + } + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (router_initialize_tls_context() < 0) { + log_err(LD_GENERAL,"Error creating TLS context for Tor client."); + return -1; + } + return 0; +} + +/** Initialize all OR private keys, and the TLS context, as necessary. + * On OPs, this only initializes the tls context. Return 0 on success, + * or -1 if Tor should die. + */ +int +init_keys(void) +{ + char *keydir; + const char *mydesc; + crypto_pk_t *prkey; + char digest[DIGEST_LEN]; + char v3_digest[DIGEST_LEN]; + const or_options_t *options = get_options(); + dirinfo_type_t type; + time_t now = time(NULL); + dir_server_t *ds; + int v3_digest_set = 0; + authority_cert_t *cert = NULL; + + /* OP's don't need persistent keys; just make up an identity and + * initialize the TLS context. */ + if (!server_mode(options)) { + return init_keys_client(); + } + if (init_keys_common() < 0) + return -1; + + if (create_keys_directory(options) < 0) + return -1; + + /* 1a. Read v3 directory authority key/cert information. */ + memset(v3_digest, 0, sizeof(v3_digest)); + if (authdir_mode_v3(options)) { + if (init_v3_authority_keys()<0) { + log_err(LD_GENERAL, "We're configured as a V3 authority, but we " + "were unable to load our v3 authority keys and certificate! " + "Use tor-gencert to generate them. Dying."); + return -1; + } + cert = get_my_v3_authority_cert(); + if (cert) { + if (crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key, + v3_digest) < 0) { + log_err(LD_BUG, "Couldn't compute my v3 authority identity key " + "digest."); + return -1; + } + v3_digest_set = 1; + } + } + + /* 1b. Read identity key. Make it if none is found. */ + keydir = get_keydir_fname("secret_id_key"); + log_info(LD_GENERAL,"Reading/making identity key "%s"...",keydir); + prkey = init_key_from_file(keydir, 1, LOG_ERR, 1); + tor_free(keydir); + if (!prkey) return -1; + set_server_identity_key(prkey); + + /* 1c. If we are configured as a bridge, generate a client key; + * otherwise, set the server identity key as our client identity + * key. */ + if (public_server_mode(options)) { + set_client_identity_key(crypto_pk_dup_key(prkey)); /* set above */ + } else { + if (!(prkey = crypto_pk_new())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_pk_free(prkey); + return -1; + } + set_client_identity_key(prkey); + } + + /* 1d. Load all ed25519 keys */ + const int new_signing_key = load_ed_keys(options,now); + if (new_signing_key < 0) + return -1; + + /* 2. Read onion key. Make it if none is found. */ + keydir = get_keydir_fname("secret_onion_key"); + log_info(LD_GENERAL,"Reading/making onion key "%s"...",keydir); + prkey = init_key_from_file(keydir, 1, LOG_ERR, 1); + tor_free(keydir); + if (!prkey) return -1; + set_onion_key(prkey); + if (options->command == CMD_RUN_TOR) { + /* only mess with the state file if we're actually running Tor */ + or_state_t *state = get_or_state(); + if (state->LastRotatedOnionKey > 100 && state->LastRotatedOnionKey < now) { + /* We allow for some parsing slop, but we don't want to risk accepting + * values in the distant future. If we did, we might never rotate the + * onion key. */ + onionkey_set_at = state->LastRotatedOnionKey; + } else { + /* We have no LastRotatedOnionKey set; either we just created the key + * or it's a holdover from 0.1.2.4-alpha-dev or earlier. In either case, + * start the clock ticking now so that we will eventually rotate it even + * if we don't stay up for the full lifetime of an onion key. */ + state->LastRotatedOnionKey = onionkey_set_at = now; + or_state_mark_dirty(state, options->AvoidDiskWrites ? + time(NULL)+3600 : 0); + } + } + + keydir = get_keydir_fname("secret_onion_key.old"); + if (!lastonionkey && file_status(keydir) == FN_FILE) { + /* Load keys from non-empty files only. + * Missing old keys won't be replaced with freshly generated keys. */ + prkey = init_key_from_file(keydir, 0, LOG_ERR, 0); + if (prkey) + lastonionkey = prkey; + } + tor_free(keydir); + + { + /* 2b. Load curve25519 onion keys. */ + int r; + keydir = get_keydir_fname("secret_onion_key_ntor"); + r = init_curve25519_keypair_from_file(&curve25519_onion_key, + keydir, 1, LOG_ERR, "onion"); + tor_free(keydir); + if (r<0) + return -1; + + keydir = get_keydir_fname("secret_onion_key_ntor.old"); + if (tor_mem_is_zero((const char *) + last_curve25519_onion_key.pubkey.public_key, + CURVE25519_PUBKEY_LEN) && + file_status(keydir) == FN_FILE) { + /* Load keys from non-empty files only. + * Missing old keys won't be replaced with freshly generated keys. */ + init_curve25519_keypair_from_file(&last_curve25519_onion_key, + keydir, 0, LOG_ERR, "onion"); + } + tor_free(keydir); + } + + /* 3. Initialize link key and TLS context. */ + if (router_initialize_tls_context() < 0) { + log_err(LD_GENERAL,"Error initializing TLS context"); + return -1; + } + + /* 3b. Get an ed25519 link certificate. Note that we need to do this + * after we set up the TLS context */ + if (generate_ed_link_cert(options, now, new_signing_key > 0) < 0) { + log_err(LD_GENERAL,"Couldn't make link cert"); + return -1; + } + + /* 4. Build our router descriptor. */ + /* Must be called after keys are initialized. */ + mydesc = router_get_my_descriptor(); + if (authdir_mode_v3(options)) { + const char *m = NULL; + routerinfo_t *ri; + /* We need to add our own fingerprint so it gets recognized. */ + if (dirserv_add_own_fingerprint(get_server_identity_key())) { + log_err(LD_GENERAL,"Error adding own fingerprint to set of relays"); + return -1; + } + if (mydesc) { + was_router_added_t added; + ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL, NULL); + if (!ri) { + log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse."); + return -1; + } + added = dirserv_add_descriptor(ri, &m, "self"); + if (!WRA_WAS_ADDED(added)) { + if (!WRA_WAS_OUTDATED(added)) { + log_err(LD_GENERAL, "Unable to add own descriptor to directory: %s", + m?m:"<unknown error>"); + return -1; + } else { + /* If the descriptor was outdated, that's ok. This can happen + * when some config options are toggled that affect workers, but + * we don't really need new keys yet so the descriptor doesn't + * change and the old one is still fresh. */ + log_info(LD_GENERAL, "Couldn't add own descriptor to directory " + "after key init: %s This is usually not a problem.", + m?m:"<unknown error>"); + } + } + } + } + + /* 5. Dump fingerprint and possibly hashed fingerprint to files. */ + if (router_write_fingerprint(0)) { + log_err(LD_FS, "Error writing fingerprint to file"); + return -1; + } + if (!public_server_mode(options) && router_write_fingerprint(1)) { + log_err(LD_FS, "Error writing hashed fingerprint to file"); + return -1; + } + + if (!authdir_mode(options)) + return 0; + /* 6. [authdirserver only] load approved-routers file */ + if (dirserv_load_fingerprint_file() < 0) { + log_err(LD_GENERAL,"Error loading fingerprints"); + return -1; + } + /* 6b. [authdirserver only] add own key to approved directories. */ + crypto_pk_get_digest(get_server_identity_key(), digest); + type = ((options->V3AuthoritativeDir ? + (V3_DIRINFO|MICRODESC_DIRINFO|EXTRAINFO_DIRINFO) : NO_DIRINFO) | + (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO)); + + ds = router_get_trusteddirserver_by_digest(digest); + if (!ds) { + ds = trusted_dir_server_new(options->Nickname, NULL, + router_get_advertised_dir_port(options, 0), + router_get_advertised_or_port(options), + NULL, + digest, + v3_digest, + type, 0.0); + if (!ds) { + log_err(LD_GENERAL,"We want to be a directory authority, but we " + "couldn't add ourselves to the authority list. Failing."); + return -1; + } + dir_server_add(ds); + } + if (ds->type != type) { + log_warn(LD_DIR, "Configured authority type does not match authority " + "type in DirAuthority list. Adjusting. (%d v %d)", + type, ds->type); + ds->type = type; + } + if (v3_digest_set && (ds->type & V3_DIRINFO) && + tor_memneq(v3_digest, ds->v3_identity_digest, DIGEST_LEN)) { + log_warn(LD_DIR, "V3 identity key does not match identity declared in " + "DirAuthority line. Adjusting."); + memcpy(ds->v3_identity_digest, v3_digest, DIGEST_LEN); + } + + if (cert) { /* add my own cert to the list of known certs */ + log_info(LD_DIR, "adding my own v3 cert"); + if (trusted_dirs_load_certs_from_string( + cert->cache_info.signed_descriptor_body, + TRUSTED_DIRS_CERTS_SRC_SELF, 0, + NULL)<0) { + log_warn(LD_DIR, "Unable to parse my own v3 cert! Failing."); + return -1; + } + } + + return 0; /* success */ +} + +/* Keep track of whether we should upload our server descriptor, + * and what type of server we are. + */ + +/** Whether we can reach our ORPort from the outside. */ +static int can_reach_or_port = 0; +/** Whether we can reach our DirPort from the outside. */ +static int can_reach_dir_port = 0; + +/** Forget what we have learned about our reachability status. */ +void +router_reset_reachability(void) +{ + can_reach_or_port = can_reach_dir_port = 0; +} + +/** Return 1 if we won't do reachability checks, because: + * - AssumeReachable is set, or + * - the network is disabled. + * Otherwise, return 0. + */ +static int +router_reachability_checks_disabled(const or_options_t *options) +{ + return options->AssumeReachable || + net_is_disabled(); +} + +/** Return 0 if we need to do an ORPort reachability check, because: + * - no reachability check has been done yet, or + * - we've initiated reachability checks, but none have succeeded. + * Return 1 if we don't need to do an ORPort reachability check, because: + * - we've seen a successful reachability check, or + * - AssumeReachable is set, or + * - the network is disabled. + */ +int +check_whether_orport_reachable(const or_options_t *options) +{ + int reach_checks_disabled = router_reachability_checks_disabled(options); + return reach_checks_disabled || + can_reach_or_port; +} + +/** Return 0 if we need to do a DirPort reachability check, because: + * - no reachability check has been done yet, or + * - we've initiated reachability checks, but none have succeeded. + * Return 1 if we don't need to do a DirPort reachability check, because: + * - we've seen a successful reachability check, or + * - there is no DirPort set, or + * - AssumeReachable is set, or + * - the network is disabled. + */ +int +check_whether_dirport_reachable(const or_options_t *options) +{ + int reach_checks_disabled = router_reachability_checks_disabled(options) || + !options->DirPort_set; + return reach_checks_disabled || + can_reach_dir_port; +} + +/** The lower threshold of remaining bandwidth required to advertise (or + * automatically provide) directory services */ +/* XXX Should this be increased? */ +#define MIN_BW_TO_ADVERTISE_DIRSERVER 51200 + +/** Return true iff we have enough configured bandwidth to advertise or + * automatically provide directory services from cache directory + * information. */ +static int +router_has_bandwidth_to_be_dirserver(const or_options_t *options) +{ + if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) { + return 0; + } + if (options->RelayBandwidthRate > 0 && + options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) { + return 0; + } + return 1; +} + +/** Helper: Return 1 if we have sufficient resources for serving directory + * requests, return 0 otherwise. + * dir_port is either 0 or the configured DirPort number. + * If AccountingMax is set less than our advertised bandwidth, then don't + * serve requests. Likewise, if our advertised bandwidth is less than + * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests. + */ +static int +router_should_be_dirserver(const or_options_t *options, int dir_port) +{ + static int advertising=1; /* start out assuming we will advertise */ + int new_choice=1; + const char *reason = NULL; + + if (accounting_is_enabled(options) && + get_options()->AccountingRule != ACCT_IN) { + /* Don't spend bytes for directory traffic if we could end up hibernating, + * but allow DirPort otherwise. Some relay operators set AccountingMax + * because they're confused or to get statistics. Directory traffic has a + * much larger effect on output than input so there is no reason to turn it + * off if using AccountingRule in. */ + int interval_length = accounting_get_interval_length(); + uint32_t effective_bw = get_effective_bwrate(options); + uint64_t acc_bytes; + if (!interval_length) { + log_warn(LD_BUG, "An accounting interval is not allowed to be zero " + "seconds long. Raising to 1."); + interval_length = 1; + } + log_info(LD_GENERAL, "Calculating whether to advertise %s: effective " + "bwrate: %u, AccountingMax: %"PRIu64", " + "accounting interval length %d", + dir_port ? "dirport" : "begindir", + effective_bw, (options->AccountingMax), + interval_length); + + acc_bytes = options->AccountingMax; + if (get_options()->AccountingRule == ACCT_SUM) + acc_bytes /= 2; + if (effective_bw >= + acc_bytes / interval_length) { + new_choice = 0; + reason = "AccountingMax enabled"; + } + } else if (! router_has_bandwidth_to_be_dirserver(options)) { + /* if we're advertising a small amount */ + new_choice = 0; + reason = "BandwidthRate under 50KB"; + } + + if (advertising != new_choice) { + if (new_choice == 1) { + if (dir_port > 0) + log_notice(LD_DIR, "Advertising DirPort as %d", dir_port); + else + log_notice(LD_DIR, "Advertising directory service support"); + } else { + tor_assert(reason); + log_notice(LD_DIR, "Not advertising Dir%s (Reason: %s)", + dir_port ? "Port" : "ectory Service support", reason); + } + advertising = new_choice; + } + + return advertising; +} + +/** Return 1 if we are configured to accept either relay or directory requests + * from clients and we aren't at risk of exceeding our bandwidth limits, thus + * we should be a directory server. If not, return 0. + */ +int +dir_server_mode(const or_options_t *options) +{ + if (!options->DirCache) + return 0; + return options->DirPort_set || + (server_mode(options) && router_has_bandwidth_to_be_dirserver(options)); +} + +/** Look at a variety of factors, and return 0 if we don't want to + * advertise the fact that we have a DirPort open or begindir support, else + * return 1. + * + * Where dir_port or supports_tunnelled_dir_requests are not relevant, they + * must be 0. + * + * Log a helpful message if we change our mind about whether to publish. + */ +static int +decide_to_advertise_dir_impl(const or_options_t *options, + uint16_t dir_port, + int supports_tunnelled_dir_requests) +{ + /* Part one: reasons to publish or not publish that aren't + * worth mentioning to the user, either because they're obvious + * or because they're normal behavior. */ + + /* short circuit the rest of the function */ + if (!dir_port && !supports_tunnelled_dir_requests) + return 0; + if (authdir_mode(options)) /* always publish */ + return 1; + if (net_is_disabled()) + return 0; + if (dir_port && !router_get_advertised_dir_port(options, dir_port)) + return 0; + if (supports_tunnelled_dir_requests && + !router_get_advertised_or_port(options)) + return 0; + + /* Part two: consider config options that could make us choose to + * publish or not publish that the user might find surprising. */ + return router_should_be_dirserver(options, dir_port); +} + +/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to + * advertise the fact that we have a DirPort open, else return the + * DirPort we want to advertise. + */ +static int +router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port) +{ + /* supports_tunnelled_dir_requests is not relevant, pass 0 */ + return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0; +} + +/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to + * advertise the fact that we support begindir requests, else return 1. + */ +static int +router_should_advertise_begindir(const or_options_t *options, + int supports_tunnelled_dir_requests) +{ + /* dir_port is not relevant, pass 0 */ + return decide_to_advertise_dir_impl(options, 0, + supports_tunnelled_dir_requests); +} + +/** Allocate and return a new extend_info_t that can be used to build + * a circuit to or through the router <b>r</b>. Uses the primary + * address of the router, so should only be called on a server. */ +static extend_info_t * +extend_info_from_router(const routerinfo_t *r) +{ + crypto_pk_t *rsa_pubkey; + extend_info_t *info; + tor_addr_port_t ap; + tor_assert(r); + + /* Make sure we don't need to check address reachability */ + tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0)); + + const ed25519_public_key_t *ed_id_key; + if (r->cache_info.signing_key_cert) + ed_id_key = &r->cache_info.signing_key_cert->signing_key; + else + ed_id_key = NULL; + + router_get_prim_orport(r, &ap); + rsa_pubkey = router_get_rsa_onion_pkey(r->onion_pkey, r->onion_pkey_len); + info = extend_info_new(r->nickname, r->cache_info.identity_digest, + ed_id_key, + rsa_pubkey, r->onion_curve25519_pkey, + &ap.addr, ap.port); + crypto_pk_free(rsa_pubkey); + return info; +} + +/**See if we currently believe our ORPort or DirPort to be + * unreachable. If so, return 1 else return 0. + */ +static int +router_should_check_reachability(int test_or, int test_dir) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); + + if (!me) + return 0; + + if (routerset_contains_router(options->ExcludeNodes, me, -1) && + options->StrictNodes) { + /* If we've excluded ourself, and StrictNodes is set, we can't test + * ourself. */ + if (test_or || test_dir) { +#define SELF_EXCLUDED_WARN_INTERVAL 3600 + static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL); + log_fn_ratelim(&warning_limit, LOG_WARN, LD_CIRC, + "Can't peform self-tests for this relay: we have " + "listed ourself in ExcludeNodes, and StrictNodes is set. " + "We cannot learn whether we are usable, and will not " + "be able to advertise ourself."); + } + return 0; + } + return 1; +} + +/** Some time has passed, or we just got new directory information. + * See if we currently believe our ORPort or DirPort to be + * unreachable. If so, launch a new test for it. + * + * For ORPort, we simply try making a circuit that ends at ourselves. + * Success is noticed in onionskin_answer(). + * + * For DirPort, we make a connection via Tor to our DirPort and ask + * for our own server descriptor. + * Success is noticed in connection_dir_client_reached_eof(). + */ +void +router_do_reachability_checks(int test_or, int test_dir) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); + int orport_reachable = check_whether_orport_reachable(options); + tor_addr_t addr; + + if (router_should_check_reachability(test_or, test_dir)) { + if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { + extend_info_t *ei = extend_info_from_router(me); + /* XXX IPv6 self testing */ + log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", + !orport_reachable ? "reachability" : "bandwidth", + fmt_addr32(me->addr), me->or_port); + circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, + CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); + extend_info_free(ei); + } + + /* XXX IPv6 self testing */ + tor_addr_from_ipv4h(&addr, me->addr); + if (test_dir && !check_whether_dirport_reachable(options) && + !connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, me->dir_port, + DIR_PURPOSE_FETCH_SERVERDESC)) { + tor_addr_port_t my_orport, my_dirport; + memcpy(&my_orport.addr, &addr, sizeof(addr)); + memcpy(&my_dirport.addr, &addr, sizeof(addr)); + my_orport.port = me->or_port; + my_dirport.port = me->dir_port; + /* ask myself, via tor, for my server descriptor. */ + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &my_orport); + directory_request_set_dir_addr_port(req, &my_dirport); + directory_request_set_directory_id_digest(req, + me->cache_info.identity_digest); + // ask via an anon circuit, connecting to our dirport. + directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); + directory_request_set_resource(req, "authority.z"); + directory_initiate_request(req); + directory_request_free(req); + } + } +} + +/** Annotate that we found our ORPort reachable. */ +void +router_orport_found_reachable(void) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); + if (!can_reach_or_port && me) { + char *address = tor_dup_ip(me->addr); + log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " + "the outside. Excellent.%s", + options->PublishServerDescriptor_ != NO_DIRINFO + && check_whether_dirport_reachable(options) ? + " Publishing server descriptor." : ""); + can_reach_or_port = 1; + mark_my_descriptor_dirty("ORPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (options->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } + control_event_server_status(LOG_NOTICE, + "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", + address, me->or_port); + tor_free(address); + } +} + +/** Annotate that we found our DirPort reachable. */ +void +router_dirport_found_reachable(void) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); + if (!can_reach_dir_port && me) { + char *address = tor_dup_ip(me->addr); + log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " + "from the outside. Excellent.%s", + options->PublishServerDescriptor_ != NO_DIRINFO + && check_whether_orport_reachable(options) ? + " Publishing server descriptor." : ""); + can_reach_dir_port = 1; + if (router_should_advertise_dirport(options, me->dir_port)) { + mark_my_descriptor_dirty("DirPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (options->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } + } + control_event_server_status(LOG_NOTICE, + "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", + address, me->dir_port); + tor_free(address); + } +} + +/** We have enough testing circuits open. Send a bunch of "drop" + * cells down each of them, to exercise our bandwidth. */ +void +router_perform_bandwidth_test(int num_circs, time_t now) +{ + int num_cells = (int)(get_options()->BandwidthRate * 10 / + CELL_MAX_NETWORK_SIZE); + int max_cells = num_cells < CIRCWINDOW_START ? + num_cells : CIRCWINDOW_START; + int cells_per_circuit = max_cells / num_circs; + origin_circuit_t *circ = NULL; + + log_notice(LD_OR,"Performing bandwidth self-test...done."); + while ((circ = circuit_get_next_by_pk_and_purpose(circ, NULL, + CIRCUIT_PURPOSE_TESTING))) { + /* dump cells_per_circuit drop cells onto this circ */ + int i = cells_per_circuit; + if (circ->base_.state != CIRCUIT_STATE_OPEN) + continue; + circ->base_.timestamp_dirty = now; + while (i-- > 0) { + if (relay_send_command_from_edge(0, TO_CIRCUIT(circ), + RELAY_COMMAND_DROP, + NULL, 0, circ->cpath->prev)<0) { + return; /* stop if error */ + } + } + } +} + +/** Return true iff our network is in some sense disabled or shutting down: + * either we're hibernating, entering hibernation, or the network is turned + * off with DisableNetwork. */ +int +net_is_disabled(void) +{ + return get_options()->DisableNetwork || we_are_hibernating(); +} + +/** Return true iff our network is in some sense "completely disabled" either + * we're fully hibernating or the network is turned off with + * DisableNetwork. */ +int +net_is_completely_disabled(void) +{ + return get_options()->DisableNetwork || we_are_fully_hibernating(); +} + +/** Return true iff we believe ourselves to be an authoritative + * directory server. + */ +int +authdir_mode(const or_options_t *options) +{ + return options->AuthoritativeDir != 0; +} +/** Return true iff we are an authoritative directory server that is + * authoritative about receiving and serving descriptors of type + * <b>purpose</b> on its dirport. + */ +int +authdir_mode_handles_descs(const or_options_t *options, int purpose) +{ + if (BUG(purpose < 0)) /* Deprecated. */ + return authdir_mode(options); + else if (purpose == ROUTER_PURPOSE_GENERAL) + return authdir_mode_v3(options); + else if (purpose == ROUTER_PURPOSE_BRIDGE) + return authdir_mode_bridge(options); + else + return 0; +} +/** Return true iff we are an authoritative directory server that + * publishes its own network statuses. + */ +int +authdir_mode_publishes_statuses(const or_options_t *options) +{ + if (authdir_mode_bridge(options)) + return 0; + return authdir_mode(options); +} +/** Return true iff we are an authoritative directory server that + * tests reachability of the descriptors it learns about. + */ +int +authdir_mode_tests_reachability(const or_options_t *options) +{ + return authdir_mode(options); +} +/** Return true iff we believe ourselves to be a bridge authoritative + * directory server. + */ +int +authdir_mode_bridge(const or_options_t *options) +{ + return authdir_mode(options) && options->BridgeAuthoritativeDir != 0; +} + +/** Return true iff we are trying to be a server. + */ +MOCK_IMPL(int, +server_mode,(const or_options_t *options)) +{ + if (options->ClientOnly) return 0; + return (options->ORPort_set); +} + +/** Return true iff we are trying to be a non-bridge server. + */ +MOCK_IMPL(int, +public_server_mode,(const or_options_t *options)) +{ + if (!server_mode(options)) return 0; + return (!options->BridgeRelay); +} + +/** Return true iff the combination of options in <b>options</b> and parameters + * in the consensus mean that we don't want to allow exits from circuits + * we got from addresses not known to be servers. */ +int +should_refuse_unknown_exits(const or_options_t *options) +{ + if (options->RefuseUnknownExits != -1) { + return options->RefuseUnknownExits; + } else { + return networkstatus_get_param(NULL, "refuseunknownexits", 1, 0, 1); + } +} + +/** Remember if we've advertised ourselves to the dirservers. */ +static int server_is_advertised=0; + +/** Return true iff we have published our descriptor lately. + */ +MOCK_IMPL(int, +advertised_server_mode,(void)) +{ + return server_is_advertised; +} + +/** + * Called with a boolean: set whether we have recently published our + * descriptor. + */ +static void +set_server_advertised(int s) +{ + server_is_advertised = s; +} + +/** Return true iff we are trying to proxy client connections. */ +int +proxy_mode(const or_options_t *options) +{ + (void)options; + SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) { + if (p->type == CONN_TYPE_AP_LISTENER || + p->type == CONN_TYPE_AP_TRANS_LISTENER || + p->type == CONN_TYPE_AP_DNS_LISTENER || + p->type == CONN_TYPE_AP_NATD_LISTENER) + return 1; + } SMARTLIST_FOREACH_END(p); + return 0; +} + +/** Decide if we're a publishable server. We are a publishable server if: + * - We don't have the ClientOnly option set + * and + * - We have the PublishServerDescriptor option set to non-empty + * and + * - We have ORPort set + * and + * - We believe our ORPort and DirPort (if present) are reachable from + * the outside; or + * - We believe our ORPort is reachable from the outside, and we can't + * check our DirPort because the consensus has no exits; or + * - We are an authoritative directory server. + */ +static int +decide_if_publishable_server(void) +{ + const or_options_t *options = get_options(); + + if (options->ClientOnly) + return 0; + if (options->PublishServerDescriptor_ == NO_DIRINFO) + return 0; + if (!server_mode(options)) + return 0; + if (authdir_mode(options)) + return 1; + if (!router_get_advertised_or_port(options)) + return 0; + if (!check_whether_orport_reachable(options)) + return 0; + if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL) { + /* All set: there are no exits in the consensus (maybe this is a tiny + * test network), so we can't check our DirPort reachability. */ + return 1; + } else { + return check_whether_dirport_reachable(options); + } +} + +/** Initiate server descriptor upload as reasonable (if server is publishable, + * etc). <b>force</b> is as for router_upload_dir_desc_to_dirservers. + * + * We need to rebuild the descriptor if it's dirty even if we're not + * uploading, because our reachability testing *uses* our descriptor to + * determine what IP address and ports to test. + */ +void +consider_publishable_server(int force) +{ + int rebuilt; + + if (!server_mode(get_options())) + return; + + rebuilt = router_rebuild_descriptor(0); + if (decide_if_publishable_server()) { + set_server_advertised(1); + if (rebuilt == 0) + router_upload_dir_desc_to_dirservers(force); + } else { + set_server_advertised(0); + } +} + +/** Return the port of the first active listener of type + * <b>listener_type</b>. */ +/** XXX not a very good interface. it's not reliable when there are + multiple listeners. */ +uint16_t +router_get_active_listener_port_by_type_af(int listener_type, + sa_family_t family) +{ + /* Iterate all connections, find one of the right kind and return + the port. Not very sophisticated or fast, but effective. */ + smartlist_t *conns = get_connection_array(); + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + if (conn->type == listener_type && !conn->marked_for_close && + conn->socket_family == family) { + return conn->port; + } + } SMARTLIST_FOREACH_END(conn); + + return 0; +} + +/** Return the port that we should advertise as our ORPort; this is either + * the one configured in the ORPort option, or the one we actually bound to + * if ORPort is "auto". + */ +uint16_t +router_get_advertised_or_port(const or_options_t *options) +{ + return router_get_advertised_or_port_by_af(options, AF_INET); +} + +/** As router_get_advertised_or_port(), but allows an address family argument. + */ +uint16_t +router_get_advertised_or_port_by_af(const or_options_t *options, + sa_family_t family) +{ + int port = get_first_advertised_port_by_type_af(CONN_TYPE_OR_LISTENER, + family); + (void)options; + + /* If the port is in 'auto' mode, we have to use + router_get_listener_port_by_type(). */ + if (port == CFG_AUTO_PORT) + return router_get_active_listener_port_by_type_af(CONN_TYPE_OR_LISTENER, + family); + + return port; +} + +/** Return the port that we should advertise as our DirPort; + * this is one of three possibilities: + * The one that is passed as <b>dirport</b> if the DirPort option is 0, or + * the one configured in the DirPort option, + * or the one we actually bound to if DirPort is "auto". */ +uint16_t +router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport) +{ + int dirport_configured = get_primary_dir_port(); + (void)options; + + if (!dirport_configured) + return dirport; + + if (dirport_configured == CFG_AUTO_PORT) + return router_get_active_listener_port_by_type_af(CONN_TYPE_DIR_LISTENER, + AF_INET); + + return dirport_configured; +} + +/* + * OR descriptor generation. + */ + +/** My routerinfo. */ +static routerinfo_t *desc_routerinfo = NULL; +/** My extrainfo */ +static extrainfo_t *desc_extrainfo = NULL; +/** Why did we most recently decide to regenerate our descriptor? Used to + * tell the authorities why we're sending it to them. */ +static const char *desc_gen_reason = "uninitialized reason"; +/** Since when has our descriptor been "clean"? 0 if we need to regenerate it + * now. */ +static time_t desc_clean_since = 0; +/** Why did we mark the descriptor dirty? */ +static const char *desc_dirty_reason = "Tor just started"; +/** Boolean: do we need to regenerate the above? */ +static int desc_needs_upload = 0; + +/** OR only: If <b>force</b> is true, or we haven't uploaded this + * descriptor successfully yet, try to upload our signed descriptor to + * all the directory servers we know about. + */ +void +router_upload_dir_desc_to_dirservers(int force) +{ + const routerinfo_t *ri; + extrainfo_t *ei; + char *msg; + size_t desc_len, extra_len = 0, total_len; + dirinfo_type_t auth = get_options()->PublishServerDescriptor_; + + ri = router_get_my_routerinfo(); + if (!ri) { + log_info(LD_GENERAL, "No descriptor; skipping upload"); + return; + } + ei = router_get_my_extrainfo(); + if (auth == NO_DIRINFO) + return; + if (!force && !desc_needs_upload) + return; + + log_info(LD_OR, "Uploading relay descriptor to directory authorities%s", + force ? " (forced)" : ""); + + desc_needs_upload = 0; + + desc_len = ri->cache_info.signed_descriptor_len; + extra_len = ei ? ei->cache_info.signed_descriptor_len : 0; + total_len = desc_len + extra_len + 1; + msg = tor_malloc(total_len); + memcpy(msg, ri->cache_info.signed_descriptor_body, desc_len); + if (ei) { + memcpy(msg+desc_len, ei->cache_info.signed_descriptor_body, extra_len); + } + msg[desc_len+extra_len] = 0; + + directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_DIR, + (auth & BRIDGE_DIRINFO) ? + ROUTER_PURPOSE_BRIDGE : + ROUTER_PURPOSE_GENERAL, + auth, msg, desc_len, extra_len); + tor_free(msg); +} + +/** OR only: Check whether my exit policy says to allow connection to + * conn. Return 0 if we accept; non-0 if we reject. + */ +int +router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + if (!me) /* make sure routerinfo exists */ + return -1; + + /* make sure it's resolved to something. this way we can't get a + 'maybe' below. */ + if (tor_addr_is_null(addr)) + return -1; + + /* look at router_get_my_routerinfo()->exit_policy for both the v4 and the + * v6 policies. The exit_policy field in router_get_my_routerinfo() is a + * bit unusual, in that it contains IPv6 and IPv6 entries. We don't want to + * look at router_get_my_routerinfo()->ipv6_exit_policy, since that's a port + * summary. */ + if ((tor_addr_family(addr) == AF_INET || + tor_addr_family(addr) == AF_INET6)) { + return compare_tor_addr_to_addr_policy(addr, port, + me->exit_policy) != ADDR_POLICY_ACCEPTED; +#if 0 + } else if (tor_addr_family(addr) == AF_INET6) { + return get_options()->IPv6Exit && + desc_routerinfo->ipv6_exit_policy && + compare_tor_addr_to_short_policy(addr, port, + me->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED; +#endif /* 0 */ + } else { + return -1; + } +} + +/** Return true iff my exit policy is reject *:*. Return -1 if we don't + * have a descriptor */ +MOCK_IMPL(int, +router_my_exit_policy_is_reject_star,(void)) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + if (!me) /* make sure routerinfo exists */ + return -1; + + return me->policy_is_reject_star; +} + +/** Return true iff I'm a server and <b>digest</b> is equal to + * my server identity key digest. */ +int +router_digest_is_me(const char *digest) +{ + return (server_identitykey && + tor_memeq(server_identitykey_digest, digest, DIGEST_LEN)); +} + +/** Return my identity digest. */ +const uint8_t * +router_get_my_id_digest(void) +{ + return (const uint8_t *)server_identitykey_digest; +} + +/** Return true iff I'm a server and <b>digest</b> is equal to + * my identity digest. */ +int +router_extrainfo_digest_is_me(const char *digest) +{ + extrainfo_t *ei = router_get_my_extrainfo(); + if (!ei) + return 0; + + return tor_memeq(digest, + ei->cache_info.signed_descriptor_digest, + DIGEST_LEN); +} + +/** A wrapper around router_digest_is_me(). */ +int +router_is_me(const routerinfo_t *router) +{ + return router_digest_is_me(router->cache_info.identity_digest); +} + +/** Return a routerinfo for this OR, rebuilding a fresh one if + * necessary. Return NULL on error, or if called on an OP. */ +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo,(void)) +{ + return router_get_my_routerinfo_with_err(NULL); +} + +/** Return routerinfo of this OR. Rebuild it from + * scratch if needed. Set <b>*err</b> to 0 on success or to + * appropriate TOR_ROUTERINFO_ERROR_* value on failure. + */ +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo_with_err,(int *err)) +{ + if (!server_mode(get_options())) { + if (err) + *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER; + + return NULL; + } + + if (!desc_clean_since) { + int rebuild_err = router_rebuild_descriptor(0); + if (rebuild_err < 0) { + if (err) + *err = rebuild_err; + + return NULL; + } + } + + if (!desc_routerinfo) { + if (err) + *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING; + + return NULL; + } + + if (err) + *err = 0; + + return desc_routerinfo; +} + +/** OR only: Return a signed server descriptor for this OR, rebuilding a fresh + * one if necessary. Return NULL on error. + */ +const char * +router_get_my_descriptor(void) +{ + const char *body; + const routerinfo_t *me = router_get_my_routerinfo(); + if (! me) + return NULL; + tor_assert(me->cache_info.saved_location == SAVED_NOWHERE); + body = signed_descriptor_get_body(&me->cache_info); + /* Make sure this is nul-terminated. */ + tor_assert(!body[me->cache_info.signed_descriptor_len]); + log_debug(LD_GENERAL,"my desc is '%s'", body); + return body; +} + +/** Return the extrainfo document for this OR, or NULL if we have none. + * Rebuilt it (and the server descriptor) if necessary. */ +extrainfo_t * +router_get_my_extrainfo(void) +{ + if (!server_mode(get_options())) + return NULL; + if (router_rebuild_descriptor(0)) + return NULL; + return desc_extrainfo; +} + +/** Return a human-readable string describing what triggered us to generate + * our current descriptor, or NULL if we don't know. */ +const char * +router_get_descriptor_gen_reason(void) +{ + return desc_gen_reason; +} + +/** A list of nicknames that we've warned about including in our family + * declaration verbatim rather than as digests. */ +static smartlist_t *warned_nonexistent_family = NULL; + +static int router_guess_address_from_dir_headers(uint32_t *guess); + +/** Make a current best guess at our address, either because + * it's configured in torrc, or because we've learned it from + * dirserver headers. Place the answer in *<b>addr</b> and return + * 0 on success, else return -1 if we have no guess. + * + * If <b>cache_only</b> is true, just return any cached answers, and + * don't try to get any new answers. + */ +MOCK_IMPL(int, +router_pick_published_address,(const or_options_t *options, uint32_t *addr, + int cache_only)) +{ + /* First, check the cached output from resolve_my_address(). */ + *addr = get_last_resolved_addr(); + if (*addr) + return 0; + + /* Second, consider doing a resolve attempt right here. */ + if (!cache_only) { + if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) { + log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr)); + return 0; + } + } + + /* Third, check the cached output from router_new_address_suggestion(). */ + if (router_guess_address_from_dir_headers(addr) >= 0) + return 0; + + /* We have no useful cached answers. Return failure. */ + return -1; +} + +/* Like router_check_descriptor_address_consistency, but specifically for the + * ORPort or DirPort. + * listener_type is either CONN_TYPE_OR_LISTENER or CONN_TYPE_DIR_LISTENER. */ +static void +router_check_descriptor_address_port_consistency(uint32_t ipv4h_desc_addr, + int listener_type) +{ + tor_assert(listener_type == CONN_TYPE_OR_LISTENER || + listener_type == CONN_TYPE_DIR_LISTENER); + + /* The first advertised Port may be the magic constant CFG_AUTO_PORT. + */ + int port_v4_cfg = get_first_advertised_port_by_type_af(listener_type, + AF_INET); + if (port_v4_cfg != 0 && + !port_exists_by_type_addr32h_port(listener_type, + ipv4h_desc_addr, port_v4_cfg, 1)) { + const tor_addr_t *port_addr = get_first_advertised_addr_by_type_af( + listener_type, + AF_INET); + /* If we're building a descriptor with no advertised address, + * something is terribly wrong. */ + tor_assert(port_addr); + + tor_addr_t desc_addr; + char port_addr_str[TOR_ADDR_BUF_LEN]; + char desc_addr_str[TOR_ADDR_BUF_LEN]; + + tor_addr_to_str(port_addr_str, port_addr, TOR_ADDR_BUF_LEN, 0); + + tor_addr_from_ipv4h(&desc_addr, ipv4h_desc_addr); + tor_addr_to_str(desc_addr_str, &desc_addr, TOR_ADDR_BUF_LEN, 0); + + const char *listener_str = (listener_type == CONN_TYPE_OR_LISTENER ? + "OR" : "Dir"); + log_warn(LD_CONFIG, "The IPv4 %sPort address %s does not match the " + "descriptor address %s. If you have a static public IPv4 " + "address, use 'Address <IPv4>' and 'OutboundBindAddress " + "<IPv4>'. If you are behind a NAT, use two %sPort lines: " + "'%sPort <PublicPort> NoListen' and '%sPort <InternalPort> " + "NoAdvertise'.", + listener_str, port_addr_str, desc_addr_str, listener_str, + listener_str, listener_str); + } +} + +/* Tor relays only have one IPv4 address in the descriptor, which is derived + * from the Address torrc option, or guessed using various methods in + * router_pick_published_address(). + * Warn the operator if there is no ORPort on the descriptor address + * ipv4h_desc_addr. + * Warn the operator if there is no DirPort on the descriptor address. + * This catches a few common config errors: + * - operators who expect ORPorts and DirPorts to be advertised on the + * ports' listen addresses, rather than the torrc Address (or guessed + * addresses in the absence of an Address config). This includes + * operators who attempt to put their ORPort and DirPort on different + * addresses; + * - discrepancies between guessed addresses and configured listen + * addresses (when the Address option isn't set). + * If a listener is listening on all IPv4 addresses, it is assumed that it + * is listening on the configured Address, and no messages are logged. + * If an operators has specified NoAdvertise ORPorts in a NAT setting, + * no messages are logged, unless they have specified other advertised + * addresses. + * The message tells operators to configure an ORPort and DirPort that match + * the Address (using NoListen if needed). + */ +static void +router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) +{ + router_check_descriptor_address_port_consistency(ipv4h_desc_addr, + CONN_TYPE_OR_LISTENER); + router_check_descriptor_address_port_consistency(ipv4h_desc_addr, + CONN_TYPE_DIR_LISTENER); +} + +/** Build a fresh routerinfo, signed server descriptor, and extra-info document + * for this OR. Set r to the generated routerinfo, e to the generated + * extra-info document. Return 0 on success, -1 on temporary error. Failure to + * generate an extra-info document is not an error and is indicated by setting + * e to NULL. Caller is responsible for freeing generated documents if 0 is + * returned. + */ +int +router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) +{ + routerinfo_t *ri; + extrainfo_t *ei; + uint32_t addr; + char platform[256]; + int hibernating = we_are_hibernating(); + const or_options_t *options = get_options(); + + if (router_pick_published_address(options, &addr, 0) < 0) { + log_warn(LD_CONFIG, "Don't know my address while generating descriptor"); + return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; + } + + /* Log a message if the address in the descriptor doesn't match the ORPort + * and DirPort addresses configured by the operator. */ + router_check_descriptor_address_consistency(addr); + + ri = tor_malloc_zero(sizeof(routerinfo_t)); + ri->cache_info.routerlist_index = -1; + ri->nickname = tor_strdup(options->Nickname); + ri->addr = addr; + ri->or_port = router_get_advertised_or_port(options); + ri->dir_port = router_get_advertised_dir_port(options, 0); + ri->supports_tunnelled_dir_requests = + directory_permits_begindir_requests(options); + ri->cache_info.published_on = time(NULL); + /* get_onion_key() must invoke from main thread */ + router_set_rsa_onion_pkey(get_onion_key(), &ri->onion_pkey, + &ri->onion_pkey_len); + + ri->onion_curve25519_pkey = + tor_memdup(&get_current_curve25519_keypair()->pubkey, + sizeof(curve25519_public_key_t)); + + /* For now, at most one IPv6 or-address is being advertised. */ + { + const port_cfg_t *ipv6_orport = NULL; + SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) { + if (p->type == CONN_TYPE_OR_LISTENER && + ! p->server_cfg.no_advertise && + ! p->server_cfg.bind_ipv4_only && + tor_addr_family(&p->addr) == AF_INET6) { + /* Like IPv4, if the relay is configured using the default + * authorities, disallow internal IPs. Otherwise, allow them. */ + const int default_auth = using_default_dir_authorities(options); + if (! tor_addr_is_internal(&p->addr, 0) || ! default_auth) { + ipv6_orport = p; + break; + } else { + char addrbuf[TOR_ADDR_BUF_LEN]; + log_warn(LD_CONFIG, + "Unable to use configured IPv6 address "%s" in a " + "descriptor. Skipping it. " + "Try specifying a globally reachable address explicitly.", + tor_addr_to_str(addrbuf, &p->addr, sizeof(addrbuf), 1)); + } + } + } SMARTLIST_FOREACH_END(p); + if (ipv6_orport) { + tor_addr_copy(&ri->ipv6_addr, &ipv6_orport->addr); + ri->ipv6_orport = ipv6_orport->port; + } + } + + ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key()); + if (BUG(crypto_pk_get_digest(ri->identity_pkey, + ri->cache_info.identity_digest) < 0)) { + routerinfo_free(ri); + return TOR_ROUTERINFO_ERROR_DIGEST_FAILED; + } + ri->cache_info.signing_key_cert = + tor_cert_dup(get_master_signing_key_cert()); + + get_platform_str(platform, sizeof(platform)); + ri->platform = tor_strdup(platform); + + ri->protocol_list = tor_strdup(protover_get_supported_protocols()); + + /* compute ri->bandwidthrate as the min of various options */ + ri->bandwidthrate = get_effective_bwrate(options); + + /* and compute ri->bandwidthburst similarly */ + ri->bandwidthburst = get_effective_bwburst(options); + + /* Report bandwidth, unless we're hibernating or shutting down */ + ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); + + if (dns_seems_to_be_broken() || has_dns_init_failed()) { + /* DNS is screwed up; don't claim to be an exit. */ + policies_exit_policy_append_reject_star(&ri->exit_policy); + } else { + policies_parse_exit_policy_from_options(options,ri->addr,&ri->ipv6_addr, + &ri->exit_policy); + } + ri->policy_is_reject_star = + policy_is_reject_star(ri->exit_policy, AF_INET, 1) && + policy_is_reject_star(ri->exit_policy, AF_INET6, 1); + + if (options->IPv6Exit) { + char *p_tmp = policy_summarize(ri->exit_policy, AF_INET6); + if (p_tmp) + ri->ipv6_exit_policy = parse_short_policy(p_tmp); + tor_free(p_tmp); + } + + if (options->MyFamily && ! options->BridgeRelay) { + if (!warned_nonexistent_family) + warned_nonexistent_family = smartlist_new(); + ri->declared_family = smartlist_new(); + config_line_t *family; + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; + const node_t *member; + if (!strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself, that's redundant */ + else + member = node_get_by_nickname(name, 0); + if (!member) { + int is_legal = is_legal_nickname_or_hexdigest(name); + if (!smartlist_contains_string(warned_nonexistent_family, name) && + !is_legal_hexdigest(name)) { + if (is_legal) + log_warn(LD_CONFIG, + "I have no descriptor for the router named "%s" in my " + "declared family; I'll use the nickname as is, but " + "this may confuse clients.", name); + else + log_warn(LD_CONFIG, "There is a router named "%s" in my " + "declared family, but that isn't a legal nickname. " + "Skipping it.", escaped(name)); + smartlist_add_strdup(warned_nonexistent_family, name); + } + if (is_legal) { + smartlist_add_strdup(ri->declared_family, name); + } + } else if (router_digest_is_me(member->identity)) { + /* Don't list ourself in our own family; that's redundant */ + /* XXX shouldn't be possible */ + } else { + char *fp = tor_malloc(HEX_DIGEST_LEN+2); + fp[0] = '$'; + base16_encode(fp+1,HEX_DIGEST_LEN+1, + member->identity, DIGEST_LEN); + smartlist_add(ri->declared_family, fp); + if (smartlist_contains_string(warned_nonexistent_family, name)) + smartlist_string_remove(warned_nonexistent_family, name); + } + } + + /* remove duplicates from the list */ + smartlist_sort_strings(ri->declared_family); + smartlist_uniq_strings(ri->declared_family); + } + + /* Now generate the extrainfo. */ + ei = tor_malloc_zero(sizeof(extrainfo_t)); + ei->cache_info.is_extrainfo = 1; + strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname)); + ei->cache_info.published_on = ri->cache_info.published_on; + ei->cache_info.signing_key_cert = + tor_cert_dup(get_master_signing_key_cert()); + + memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, + DIGEST_LEN); + if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, + ei, get_server_identity_key(), + get_master_signing_keypair()) < 0) { + log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); + extrainfo_free(ei); + ei = NULL; + } else { + ei->cache_info.signed_descriptor_len = + strlen(ei->cache_info.signed_descriptor_body); + router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_len, + ei->cache_info.signed_descriptor_digest); + crypto_digest256((char*) ei->digest256, + ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_len, + DIGEST_SHA256); + } + + /* Now finish the router descriptor. */ + if (ei) { + memcpy(ri->cache_info.extra_info_digest, + ei->cache_info.signed_descriptor_digest, + DIGEST_LEN); + memcpy(ri->cache_info.extra_info_digest256, + ei->digest256, + DIGEST256_LEN); + } else { + /* ri was allocated with tor_malloc_zero, so there is no need to + * zero ri->cache_info.extra_info_digest here. */ + } + if (! (ri->cache_info.signed_descriptor_body = + router_dump_router_to_string(ri, get_server_identity_key(), + get_onion_key(), + get_current_curve25519_keypair(), + get_master_signing_keypair())) ) { + log_warn(LD_BUG, "Couldn't generate router descriptor."); + routerinfo_free(ri); + extrainfo_free(ei); + return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE; + } + ri->cache_info.signed_descriptor_len = + strlen(ri->cache_info.signed_descriptor_body); + + ri->purpose = + options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; + if (options->BridgeRelay) { + /* Bridges shouldn't be able to send their descriptors unencrypted, + anyway, since they don't have a DirPort, and always connect to the + bridge authority anonymously. But just in case they somehow think of + sending them on an unencrypted connection, don't allow them to try. */ + ri->cache_info.send_unencrypted = 0; + if (ei) + ei->cache_info.send_unencrypted = 0; + } else { + ri->cache_info.send_unencrypted = 1; + if (ei) + ei->cache_info.send_unencrypted = 1; + } + + router_get_router_hash(ri->cache_info.signed_descriptor_body, + strlen(ri->cache_info.signed_descriptor_body), + ri->cache_info.signed_descriptor_digest); + + if (ei) { + tor_assert(! + routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, + &ri->cache_info, NULL)); + } + + *r = ri; + *e = ei; + return 0; +} + +/** If <b>force</b> is true, or our descriptor is out-of-date, rebuild a fresh + * routerinfo, signed server descriptor, and extra-info document for this OR. + * Return 0 on success, -1 on temporary error. + */ +int +router_rebuild_descriptor(int force) +{ + int err = 0; + routerinfo_t *ri; + extrainfo_t *ei; + uint32_t addr; + const or_options_t *options = get_options(); + + if (desc_clean_since && !force) + return 0; + + if (router_pick_published_address(options, &addr, 0) < 0 || + router_get_advertised_or_port(options) == 0) { + /* Stop trying to rebuild our descriptor every second. We'll + * learn that it's time to try again when ip_address_changed() + * marks it dirty. */ + desc_clean_since = time(NULL); + return TOR_ROUTERINFO_ERROR_DESC_REBUILDING; + } + + log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : ""); + + err = router_build_fresh_descriptor(&ri, &ei); + if (err < 0) { + return err; + } + + routerinfo_free(desc_routerinfo); + desc_routerinfo = ri; + extrainfo_free(desc_extrainfo); + desc_extrainfo = ei; + + desc_clean_since = time(NULL); + desc_needs_upload = 1; + desc_gen_reason = desc_dirty_reason; + if (BUG(desc_gen_reason == NULL)) { + desc_gen_reason = "descriptor was marked dirty earlier, for no reason."; + } + desc_dirty_reason = NULL; + control_event_my_descriptor_changed(); + return 0; +} + +/** If our router descriptor ever goes this long without being regenerated + * because something changed, we force an immediate regenerate-and-upload. */ +#define FORCE_REGENERATE_DESCRIPTOR_INTERVAL (18*60*60) + +/** If our router descriptor seems to be missing or unacceptable according + * to the authorities, regenerate and reupload it _this_ often. */ +#define FAST_RETRY_DESCRIPTOR_INTERVAL (90*60) + +/** Mark descriptor out of date if it's been "too long" since we last tried + * to upload one. */ +void +mark_my_descriptor_dirty_if_too_old(time_t now) +{ + networkstatus_t *ns; + const routerstatus_t *rs; + const char *retry_fast_reason = NULL; /* Set if we should retry frequently */ + const time_t slow_cutoff = now - FORCE_REGENERATE_DESCRIPTOR_INTERVAL; + const time_t fast_cutoff = now - FAST_RETRY_DESCRIPTOR_INTERVAL; + + /* If it's already dirty, don't mark it. */ + if (! desc_clean_since) + return; + + /* If it's older than FORCE_REGENERATE_DESCRIPTOR_INTERVAL, it's always + * time to rebuild it. */ + if (desc_clean_since < slow_cutoff) { + mark_my_descriptor_dirty("time for new descriptor"); + return; + } + /* Now we see whether we want to be retrying frequently or no. The + * rule here is that we'll retry frequently if we aren't listed in the + * live consensus we have, or if the publication time of the + * descriptor listed for us in the consensus is very old. */ + ns = networkstatus_get_live_consensus(now); + if (ns) { + rs = networkstatus_vote_find_entry(ns, server_identitykey_digest); + if (rs == NULL) + retry_fast_reason = "not listed in consensus"; + else if (rs->published_on < slow_cutoff) + retry_fast_reason = "version listed in consensus is quite old"; + } + + if (retry_fast_reason && desc_clean_since < fast_cutoff) + mark_my_descriptor_dirty(retry_fast_reason); +} + +/** Call when the current descriptor is out of date. */ +void +mark_my_descriptor_dirty(const char *reason) +{ + const or_options_t *options = get_options(); + if (BUG(reason == NULL)) { + reason = "marked descriptor dirty for unspecified reason"; + } + if (server_mode(options) && options->PublishServerDescriptor_) + log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason); + desc_clean_since = 0; + if (!desc_dirty_reason) + desc_dirty_reason = reason; +} + +/** How frequently will we republish our descriptor because of large (factor + * of 2) shifts in estimated bandwidth? Note: We don't use this constant + * if our previous bandwidth estimate was exactly 0. */ +#define MAX_BANDWIDTH_CHANGE_FREQ (3*60*60) + ++/** Maximum uptime to republish our descriptor because of large shifts in ++ * estimated bandwidth. */ ++#define MAX_UPTIME_BANDWIDTH_CHANGE (24*60*60) ++ ++/** By which factor bandwidth shifts have to change to be considered large. */ ++#define BANDWIDTH_CHANGE_FACTOR 2 ++ +/** Check whether bandwidth has changed a lot since the last time we announced - * bandwidth. If so, mark our descriptor dirty. */ ++ * bandwidth while the uptime is smaller than MAX_UPTIME_BANDWIDTH_CHANGE. ++ * If so, mark our descriptor dirty. */ +void +check_descriptor_bandwidth_changed(time_t now) +{ + static time_t last_changed = 0; + uint64_t prev, cur; ++ const int hibernating = we_are_hibernating(); ++ ++ /* If the relay uptime is bigger than MAX_UPTIME_BANDWIDTH_CHANGE, ++ * the next regularly scheduled descriptor update (18h) will be enough */ ++ if (get_uptime() > MAX_UPTIME_BANDWIDTH_CHANGE && !hibernating) ++ return; ++ + const routerinfo_t *my_ri = router_get_my_routerinfo(); - if (!my_ri) /* make sure routerinfo exists */ ++ ++ if (!my_ri) + return; + + prev = my_ri->bandwidthcapacity; + + /* Consider ourselves to have zero bandwidth if we're hibernating or + * shutting down. */ - cur = we_are_hibernating() ? 0 : rep_hist_bandwidth_assess(); ++ cur = hibernating ? 0 : rep_hist_bandwidth_assess(); ++ + if ((prev != cur && (!prev || !cur)) || - cur > prev*2 || - cur < prev/2) { ++ cur > (prev * BANDWIDTH_CHANGE_FACTOR) || ++ cur < (prev / BANDWIDTH_CHANGE_FACTOR) ) { + if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) { + log_info(LD_GENERAL, + "Measured bandwidth has changed; rebuilding descriptor."); + mark_my_descriptor_dirty("bandwidth has changed"); + last_changed = now; + } + } +} + +/** Note at log level severity that our best guess of address has changed from + * <b>prev</b> to <b>cur</b>. */ +static void +log_addr_has_changed(int severity, + const tor_addr_t *prev, + const tor_addr_t *cur, + const char *source) +{ + char addrbuf_prev[TOR_ADDR_BUF_LEN]; + char addrbuf_cur[TOR_ADDR_BUF_LEN]; + + if (BUG(!server_mode(get_options()))) + return; + + if (tor_addr_to_str(addrbuf_prev, prev, sizeof(addrbuf_prev), 1) == NULL) + strlcpy(addrbuf_prev, "???", TOR_ADDR_BUF_LEN); + if (tor_addr_to_str(addrbuf_cur, cur, sizeof(addrbuf_cur), 1) == NULL) + strlcpy(addrbuf_cur, "???", TOR_ADDR_BUF_LEN); + + if (!tor_addr_is_null(prev)) + log_fn(severity, LD_GENERAL, + "Our IP Address has changed from %s to %s; " + "rebuilding descriptor (source: %s).", + addrbuf_prev, addrbuf_cur, source); + else + log_notice(LD_GENERAL, + "Guessed our IP address as %s (source: %s).", + addrbuf_cur, source); +} + +/** Check whether our own address as defined by the Address configuration + * has changed. This is for routers that get their address from a service + * like dyndns. If our address has changed, mark our descriptor dirty. */ +void +check_descriptor_ipaddress_changed(time_t now) +{ + uint32_t prev, cur; + const or_options_t *options = get_options(); + const char *method = NULL; + char *hostname = NULL; + const routerinfo_t *my_ri = router_get_my_routerinfo(); + + (void) now; + + if (my_ri == NULL) /* make sure routerinfo exists */ + return; + + /* XXXX ipv6 */ + prev = my_ri->addr; + if (resolve_my_address(LOG_INFO, options, &cur, &method, &hostname) < 0) { + log_info(LD_CONFIG,"options->Address didn't resolve into an IP."); + return; + } + + if (prev != cur) { + char *source; + tor_addr_t tmp_prev, tmp_cur; + + tor_addr_from_ipv4h(&tmp_prev, prev); + tor_addr_from_ipv4h(&tmp_cur, cur); + + tor_asprintf(&source, "METHOD=%s%s%s", method, + hostname ? " HOSTNAME=" : "", + hostname ? hostname : ""); + + log_addr_has_changed(LOG_NOTICE, &tmp_prev, &tmp_cur, source); + tor_free(source); + + ip_address_changed(0); + } + + tor_free(hostname); +} + +/** The most recently guessed value of our IP address, based on directory + * headers. */ +static tor_addr_t last_guessed_ip = TOR_ADDR_NULL; + +/** A directory server <b>d_conn</b> told us our IP address is + * <b>suggestion</b>. + * If this address is different from the one we think we are now, and + * if our computer doesn't actually know its IP address, then switch. */ +void +router_new_address_suggestion(const char *suggestion, + const dir_connection_t *d_conn) +{ + tor_addr_t addr; + uint32_t cur = 0; /* Current IPv4 address. */ + const or_options_t *options = get_options(); + + /* first, learn what the IP address actually is */ + if (tor_addr_parse(&addr, suggestion) == -1) { + log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.", + escaped(suggestion)); + return; + } + + log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion); + + if (!server_mode(options)) { + tor_addr_copy(&last_guessed_ip, &addr); + return; + } + + /* XXXX ipv6 */ + cur = get_last_resolved_addr(); + if (cur || + resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) { + /* We're all set -- we already know our address. Great. */ + tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we + need it later */ + return; + } + if (tor_addr_is_internal(&addr, 0)) { + /* Don't believe anybody who says our IP is, say, 127.0.0.1. */ + return; + } + if (tor_addr_eq(&d_conn->base_.addr, &addr)) { + /* Don't believe anybody who says our IP is their IP. */ + log_debug(LD_DIR, "A directory server told us our IP address is %s, " + "but they are just reporting their own IP address. Ignoring.", + suggestion); + return; + } + + /* Okay. We can't resolve our own address, and X-Your-Address-Is is giving + * us an answer different from what we had the last time we managed to + * resolve it. */ + if (!tor_addr_eq(&last_guessed_ip, &addr)) { + control_event_server_status(LOG_NOTICE, + "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV", + suggestion); + log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr, + d_conn->base_.address); + ip_address_changed(0); + tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor() + will fetch it */ + } +} + +/** We failed to resolve our address locally, but we'd like to build + * a descriptor and publish / test reachability. If we have a guess + * about our address based on directory headers, answer it and return + * 0; else return -1. */ +static int +router_guess_address_from_dir_headers(uint32_t *guess) +{ + if (!tor_addr_is_null(&last_guessed_ip)) { + *guess = tor_addr_to_ipv4h(&last_guessed_ip); + return 0; + } + return -1; +} + +/** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short + * string describing the version of Tor and the operating system we're + * currently running on. + */ +STATIC void +get_platform_str(char *platform, size_t len) +{ + tor_snprintf(platform, len, "Tor %s on %s", + get_short_version(), get_uname()); +} + +/* XXX need to audit this thing and count fenceposts. maybe + * refactor so we don't have to keep asking if we're + * near the end of maxlen? + */ +#define DEBUG_ROUTER_DUMP_ROUTER_TO_STRING + +/** OR only: Given a routerinfo for this router, and an identity key to sign + * with, encode the routerinfo as a signed server descriptor and return a new + * string encoding the result, or NULL on failure. + */ +char * +router_dump_router_to_string(routerinfo_t *router, + const crypto_pk_t *ident_key, + const crypto_pk_t *tap_key, + const curve25519_keypair_t *ntor_keypair, + const ed25519_keypair_t *signing_keypair) +{ + char *address = NULL; + char *onion_pkey = NULL; /* Onion key, PEM-encoded. */ + crypto_pk_t *rsa_pubkey = NULL; + char *identity_pkey = NULL; /* Identity key, PEM-encoded. */ + char digest[DIGEST256_LEN]; + char published[ISO_TIME_LEN+1]; + char fingerprint[FINGERPRINT_LEN+1]; + char *extra_info_line = NULL; + size_t onion_pkeylen, identity_pkeylen; + char *family_line = NULL; + char *extra_or_address = NULL; + const or_options_t *options = get_options(); + smartlist_t *chunks = NULL; + char *output = NULL; + const int emit_ed_sigs = signing_keypair && + router->cache_info.signing_key_cert; + char *ed_cert_line = NULL; + char *rsa_tap_cc_line = NULL; + char *ntor_cc_line = NULL; + char *proto_line = NULL; + + /* Make sure the identity key matches the one in the routerinfo. */ + if (!crypto_pk_eq_keys(ident_key, router->identity_pkey)) { + log_warn(LD_BUG,"Tried to sign a router with a private key that didn't " + "match router's public key!"); + goto err; + } + if (emit_ed_sigs) { + if (!router->cache_info.signing_key_cert->signing_key_included || + !ed25519_pubkey_eq(&router->cache_info.signing_key_cert->signed_key, + &signing_keypair->pubkey)) { + log_warn(LD_BUG, "Tried to sign a router descriptor with a mismatched " + "ed25519 key chain %d", + router->cache_info.signing_key_cert->signing_key_included); + goto err; + } + } + + /* record our fingerprint, so we can include it in the descriptor */ + if (crypto_pk_get_fingerprint(router->identity_pkey, fingerprint, 1)<0) { + log_err(LD_BUG,"Error computing fingerprint"); + goto err; + } + + if (emit_ed_sigs) { + /* Encode ed25519 signing cert */ + char ed_cert_base64[256]; + char ed_fp_base64[ED25519_BASE64_LEN+1]; + if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64), + (const char*)router->cache_info.signing_key_cert->encoded, + router->cache_info.signing_key_cert->encoded_len, + BASE64_ENCODE_MULTILINE) < 0) { + log_err(LD_BUG,"Couldn't base64-encode signing key certificate!"); + goto err; + } + if (ed25519_public_to_base64(ed_fp_base64, + &router->cache_info.signing_key_cert->signing_key)<0) { + log_err(LD_BUG,"Couldn't base64-encode identity key\n"); + goto err; + } + tor_asprintf(&ed_cert_line, "identity-ed25519\n" + "-----BEGIN ED25519 CERT-----\n" + "%s" + "-----END ED25519 CERT-----\n" + "master-key-ed25519 %s\n", + ed_cert_base64, ed_fp_base64); + } + + /* PEM-encode the onion key */ + rsa_pubkey = router_get_rsa_onion_pkey(router->onion_pkey, + router->onion_pkey_len); + if (crypto_pk_write_public_key_to_string(rsa_pubkey, + &onion_pkey,&onion_pkeylen)<0) { + log_warn(LD_BUG,"write onion_pkey to string failed!"); + goto err; + } + + /* PEM-encode the identity key */ + if (crypto_pk_write_public_key_to_string(router->identity_pkey, + &identity_pkey,&identity_pkeylen)<0) { + log_warn(LD_BUG,"write identity_pkey to string failed!"); + goto err; + } + + /* Cross-certify with RSA key */ + if (tap_key && router->cache_info.signing_key_cert && + router->cache_info.signing_key_cert->signing_key_included) { + char buf[256]; + int tap_cc_len = 0; + uint8_t *tap_cc = + make_tap_onion_key_crosscert(tap_key, + &router->cache_info.signing_key_cert->signing_key, + router->identity_pkey, + &tap_cc_len); + if (!tap_cc) { + log_warn(LD_BUG,"make_tap_onion_key_crosscert failed!"); + goto err; + } + + if (base64_encode(buf, sizeof(buf), (const char*)tap_cc, tap_cc_len, + BASE64_ENCODE_MULTILINE) < 0) { + log_warn(LD_BUG,"base64_encode(rsa_crosscert) failed!"); + tor_free(tap_cc); + goto err; + } + tor_free(tap_cc); + + tor_asprintf(&rsa_tap_cc_line, + "onion-key-crosscert\n" + "-----BEGIN CROSSCERT-----\n" + "%s" + "-----END CROSSCERT-----\n", buf); + } + + /* Cross-certify with onion keys */ + if (ntor_keypair && router->cache_info.signing_key_cert && + router->cache_info.signing_key_cert->signing_key_included) { + int sign = 0; + char buf[256]; + /* XXXX Base the expiration date on the actual onion key expiration time?*/ + tor_cert_t *cert = + make_ntor_onion_key_crosscert(ntor_keypair, + &router->cache_info.signing_key_cert->signing_key, + router->cache_info.published_on, + get_onion_key_lifetime(), &sign); + if (!cert) { + log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!"); + goto err; + } + tor_assert(sign == 0 || sign == 1); + + if (base64_encode(buf, sizeof(buf), + (const char*)cert->encoded, cert->encoded_len, + BASE64_ENCODE_MULTILINE)<0) { + log_warn(LD_BUG,"base64_encode(ntor_crosscert) failed!"); + tor_cert_free(cert); + goto err; + } + tor_cert_free(cert); + + tor_asprintf(&ntor_cc_line, + "ntor-onion-key-crosscert %d\n" + "-----BEGIN ED25519 CERT-----\n" + "%s" + "-----END ED25519 CERT-----\n", sign, buf); + } + + /* Encode the publication time. */ + format_iso_time(published, router->cache_info.published_on); + + if (router->declared_family && smartlist_len(router->declared_family)) { + char *family = smartlist_join_strings(router->declared_family, + " ", 0, NULL); + tor_asprintf(&family_line, "family %s\n", family); + tor_free(family); + } else { + family_line = tor_strdup(""); + } + + if (!tor_digest_is_zero(router->cache_info.extra_info_digest)) { + char extra_info_digest[HEX_DIGEST_LEN+1]; + base16_encode(extra_info_digest, sizeof(extra_info_digest), + router->cache_info.extra_info_digest, DIGEST_LEN); + if (!tor_digest256_is_zero(router->cache_info.extra_info_digest256)) { + char d256_64[BASE64_DIGEST256_LEN+1]; + digest256_to_base64(d256_64, router->cache_info.extra_info_digest256); + tor_asprintf(&extra_info_line, "extra-info-digest %s %s\n", + extra_info_digest, d256_64); + } else { + tor_asprintf(&extra_info_line, "extra-info-digest %s\n", + extra_info_digest); + } + } + + if (router->ipv6_orport && + tor_addr_family(&router->ipv6_addr) == AF_INET6) { + char addr[TOR_ADDR_BUF_LEN]; + const char *a; + a = tor_addr_to_str(addr, &router->ipv6_addr, sizeof(addr), 1); + if (a) { + tor_asprintf(&extra_or_address, + "or-address %s:%d\n", a, router->ipv6_orport); + log_debug(LD_OR, "My or-address line is <%s>", extra_or_address); + } + } + + if (router->protocol_list) { + tor_asprintf(&proto_line, "proto %s\n", router->protocol_list); + } else { + proto_line = tor_strdup(""); + } + + address = tor_dup_ip(router->addr); + chunks = smartlist_new(); + + /* Generate the easy portion of the router descriptor. */ + smartlist_add_asprintf(chunks, + "router %s %s %d 0 %d\n" + "%s" + "%s" + "platform %s\n" + "%s" + "published %s\n" + "fingerprint %s\n" + "uptime %ld\n" + "bandwidth %d %d %d\n" + "%s%s" + "onion-key\n%s" + "signing-key\n%s" + "%s%s" + "%s%s%s", + router->nickname, + address, + router->or_port, + router_should_advertise_dirport(options, router->dir_port), + ed_cert_line ? ed_cert_line : "", + extra_or_address ? extra_or_address : "", + router->platform, + proto_line, + published, + fingerprint, + get_uptime(), + (int) router->bandwidthrate, + (int) router->bandwidthburst, + (int) router->bandwidthcapacity, + extra_info_line ? extra_info_line : "", + (options->DownloadExtraInfo || options->V3AuthoritativeDir) ? + "caches-extra-info\n" : "", + onion_pkey, identity_pkey, + rsa_tap_cc_line ? rsa_tap_cc_line : "", + ntor_cc_line ? ntor_cc_line : "", + family_line, + we_are_hibernating() ? "hibernating 1\n" : "", + "hidden-service-dir\n"); + + if (options->ContactInfo && strlen(options->ContactInfo)) { + const char *ci = options->ContactInfo; + if (strchr(ci, '\n') || strchr(ci, '\r')) + ci = escaped(ci); + smartlist_add_asprintf(chunks, "contact %s\n", ci); + } + + if (options->BridgeRelay) { + const char *bd; + if (options->BridgeDistribution && strlen(options->BridgeDistribution)) { + bd = options->BridgeDistribution; + } else { + bd = "any"; + } + if (strchr(bd, '\n') || strchr(bd, '\r')) + bd = escaped(bd); + smartlist_add_asprintf(chunks, "bridge-distribution-request %s\n", bd); + } + + if (router->onion_curve25519_pkey) { + char kbuf[128]; + base64_encode(kbuf, sizeof(kbuf), + (const char *)router->onion_curve25519_pkey->public_key, + CURVE25519_PUBKEY_LEN, BASE64_ENCODE_MULTILINE); + smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf); + } else { + /* Authorities will start rejecting relays without ntor keys in 0.2.9 */ + log_err(LD_BUG, "A relay must have an ntor onion key"); + goto err; + } + + /* Write the exit policy to the end of 's'. */ + if (!router->exit_policy || !smartlist_len(router->exit_policy)) { + smartlist_add_strdup(chunks, "reject *:*\n"); + } else if (router->exit_policy) { + char *exit_policy = router_dump_exit_policy_to_string(router,1,0); + + if (!exit_policy) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", exit_policy); + tor_free(exit_policy); + } + + if (router->ipv6_exit_policy) { + char *p6 = write_short_policy(router->ipv6_exit_policy); + if (p6 && strcmp(p6, "reject 1-65535")) { + smartlist_add_asprintf(chunks, + "ipv6-policy %s\n", p6); + } + tor_free(p6); + } + + if (router_should_advertise_begindir(options, + router->supports_tunnelled_dir_requests)) { + smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); + } + + /* Sign the descriptor with Ed25519 */ + if (emit_ed_sigs) { + smartlist_add_strdup(chunks, "router-sig-ed25519 "); + crypto_digest_smartlist_prefix(digest, DIGEST256_LEN, + ED_DESC_SIGNATURE_PREFIX, + chunks, "", DIGEST_SHA256); + ed25519_signature_t sig; + char buf[ED25519_SIG_BASE64_LEN+1]; + if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN, + signing_keypair) < 0) + goto err; + if (ed25519_signature_to_base64(buf, &sig) < 0) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", buf); + } + + /* Sign the descriptor with RSA */ + smartlist_add_strdup(chunks, "router-signature\n"); + + crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1); + + { + char *sig; + if (!(sig = router_get_dirobj_signature(digest, DIGEST_LEN, ident_key))) { + log_warn(LD_BUG, "Couldn't sign router descriptor"); + goto err; + } + smartlist_add(chunks, sig); + } + + /* include a last '\n' */ + smartlist_add_strdup(chunks, "\n"); + + output = smartlist_join_strings(chunks, "", 0, NULL); + +#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING + { + char *s_dup; + const char *cp; + routerinfo_t *ri_tmp; + cp = s_dup = tor_strdup(output); + ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL, NULL); + if (!ri_tmp) { + log_err(LD_BUG, + "We just generated a router descriptor we can't parse."); + log_err(LD_BUG, "Descriptor was: <<%s>>", output); + goto err; + } + tor_free(s_dup); + routerinfo_free(ri_tmp); + } +#endif /* defined(DEBUG_ROUTER_DUMP_ROUTER_TO_STRING) */ + + goto done; + + err: + tor_free(output); /* sets output to NULL */ + done: + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + } + crypto_pk_free(rsa_pubkey); + tor_free(address); + tor_free(family_line); + tor_free(onion_pkey); + tor_free(identity_pkey); + tor_free(extra_or_address); + tor_free(ed_cert_line); + tor_free(rsa_tap_cc_line); + tor_free(ntor_cc_line); + tor_free(extra_info_line); + tor_free(proto_line); + + return output; +} + +/** + * OR only: Given <b>router</b>, produce a string with its exit policy. + * If <b>include_ipv4</b> is true, include IPv4 entries. + * If <b>include_ipv6</b> is true, include IPv6 entries. + */ +char * +router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6) +{ + if ((!router->exit_policy) || (router->policy_is_reject_star)) { + return tor_strdup("reject *:*"); + } + + return policy_dump_to_string(router->exit_policy, + include_ipv4, + include_ipv6); +} + +/** Copy the primary (IPv4) OR port (IP address and TCP port) for + * <b>router</b> into *<b>ap_out</b>. */ +void +router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *ap_out) +{ + tor_assert(ap_out != NULL); + tor_addr_from_ipv4h(&ap_out->addr, router->addr); + ap_out->port = router->or_port; +} + +/** Return 1 if any of <b>router</b>'s addresses are <b>addr</b>. + * Otherwise return 0. */ +int +router_has_addr(const routerinfo_t *router, const tor_addr_t *addr) +{ + return + tor_addr_eq_ipv4h(addr, router->addr) || + tor_addr_eq(&router->ipv6_addr, addr); +} + +int +router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport) +{ + return + (tor_addr_eq_ipv4h(&orport->addr, router->addr) && + orport->port == router->or_port) || + (tor_addr_eq(&orport->addr, &router->ipv6_addr) && + orport->port == router->ipv6_orport); +} + +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist or is empty, or -1 + * if the file does not contain a line matching these criteria or other + * failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + /* treat empty stats files as if the file doesn't exist */ + case FN_NOENT: + case FN_EMPTY: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + +/** Write the contents of <b>extrainfo</b> and aggregated statistics to + * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on + * success, negative on failure. */ +int +extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, + crypto_pk_t *ident_key, + const ed25519_keypair_t *signing_keypair) +{ + const or_options_t *options = get_options(); + char identity[HEX_DIGEST_LEN+1]; + char published[ISO_TIME_LEN+1]; + char digest[DIGEST_LEN]; + char *bandwidth_usage; + int result; + static int write_stats_to_extrainfo = 1; + char sig[DIROBJ_MAX_SIG_LEN+1]; + char *s = NULL, *pre, *contents, *cp, *s_dup = NULL; + time_t now = time(NULL); + smartlist_t *chunks = smartlist_new(); + extrainfo_t *ei_tmp = NULL; + const int emit_ed_sigs = signing_keypair && + extrainfo->cache_info.signing_key_cert; + char *ed_cert_line = NULL; + + base16_encode(identity, sizeof(identity), + extrainfo->cache_info.identity_digest, DIGEST_LEN); + format_iso_time(published, extrainfo->cache_info.published_on); + bandwidth_usage = rep_hist_get_bandwidth_lines(); + if (emit_ed_sigs) { + if (!extrainfo->cache_info.signing_key_cert->signing_key_included || + !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key, + &signing_keypair->pubkey)) { + log_warn(LD_BUG, "Tried to sign a extrainfo descriptor with a " + "mismatched ed25519 key chain %d", + extrainfo->cache_info.signing_key_cert->signing_key_included); + goto err; + } + char ed_cert_base64[256]; + if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64), + (const char*)extrainfo->cache_info.signing_key_cert->encoded, + extrainfo->cache_info.signing_key_cert->encoded_len, + BASE64_ENCODE_MULTILINE) < 0) { + log_err(LD_BUG,"Couldn't base64-encode signing key certificate!"); + goto err; + } + tor_asprintf(&ed_cert_line, "identity-ed25519\n" + "-----BEGIN ED25519 CERT-----\n" + "%s" + "-----END ED25519 CERT-----\n", ed_cert_base64); + } else { + ed_cert_line = tor_strdup(""); + } + + tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s", + extrainfo->nickname, identity, + ed_cert_line, + published, bandwidth_usage); + smartlist_add(chunks, pre); + + if (geoip_is_loaded(AF_INET)) + smartlist_add_asprintf(chunks, "geoip-db-digest %s\n", + geoip_db_digest(AF_INET)); + if (geoip_is_loaded(AF_INET6)) + smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n", + geoip_db_digest(AF_INET6)); + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->HiddenServiceStatistics && + load_stats_file("stats"PATH_SEPARATOR"hidserv-stats", + "hidserv-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->ConnDirectionStatistics && + load_stats_file("stats"PATH_SEPARATOR"conn-stats", + "conn-bi-direct", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + } + + if (options->PaddingStatistics) { + contents = rep_hist_get_padding_count_lines(); + if (contents) + smartlist_add(chunks, contents); + } + + /* Add information about the pluggable transports we support. */ + if (options->ServerTransportPlugin) { + char *pluggable_transports = pt_get_extra_info_descriptor_string(); + if (pluggable_transports) + smartlist_add(chunks, pluggable_transports); + } + + if (should_record_bridge_info(options) && write_stats_to_extrainfo) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + smartlist_add_strdup(chunks, bridge_stats); + } + } + + if (emit_ed_sigs) { + char sha256_digest[DIGEST256_LEN]; + smartlist_add_strdup(chunks, "router-sig-ed25519 "); + crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN, + ED_DESC_SIGNATURE_PREFIX, + chunks, "", DIGEST_SHA256); + ed25519_signature_t ed_sig; + char buf[ED25519_SIG_BASE64_LEN+1]; + if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN, + signing_keypair) < 0) + goto err; + if (ed25519_signature_to_base64(buf, &ed_sig) < 0) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", buf); + } + + smartlist_add_strdup(chunks, "router-signature\n"); + s = smartlist_join_strings(chunks, "", 0, NULL); + + while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) { + /* So long as there are at least two chunks (one for the initial + * extra-info line and one for the router-signature), we can keep removing + * things. */ + if (smartlist_len(chunks) > 2) { + /* We remove the next-to-last element (remember, len-1 is the last + element), since we need to keep the router-signature element. */ + int idx = smartlist_len(chunks) - 2; + char *e = smartlist_get(chunks, idx); + smartlist_del_keeporder(chunks, idx); + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that exceeds the 50 KB " + "upload limit. Removing last added " + "statistics."); + tor_free(e); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + } else { + log_warn(LD_BUG, "We just generated an extra-info descriptors that " + "exceeds the 50 KB upload limit."); + goto err; + } + } + + memset(sig, 0, sizeof(sig)); + if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 || + router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, + ident_key) < 0) { + log_warn(LD_BUG, "Could not append signature to extra-info " + "descriptor."); + goto err; + } + smartlist_add_strdup(chunks, sig); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL, NULL); + if (!ei_tmp) { + if (write_stats_to_extrainfo) { + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that we can't parse. Not " + "adding statistics to this or any future " + "extra-info descriptors."); + write_stats_to_extrainfo = 0; + result = extrainfo_dump_to_string(s_out, extrainfo, ident_key, + signing_keypair); + goto done; + } else { + log_warn(LD_BUG, "We just generated an extrainfo descriptor we " + "can't parse."); + goto err; + } + } + + *s_out = s; + s = NULL; /* prevent free */ + result = 0; + goto done; + + err: + result = -1; + + done: + tor_free(s); + SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk)); + smartlist_free(chunks); + tor_free(s_dup); + tor_free(ed_cert_line); + extrainfo_free(ei_tmp); + tor_free(bandwidth_usage); + + return result; +} + +/** Return true iff <b>s</b> is a valid server nickname. (That is, a string + * containing between 1 and MAX_NICKNAME_LEN characters from + * LEGAL_NICKNAME_CHARACTERS.) */ +int +is_legal_nickname(const char *s) +{ + size_t len; + tor_assert(s); + len = strlen(s); + return len > 0 && len <= MAX_NICKNAME_LEN && + strspn(s,LEGAL_NICKNAME_CHARACTERS) == len; +} + +/** Return true iff <b>s</b> is a valid server nickname or + * hex-encoded identity-key digest. */ +int +is_legal_nickname_or_hexdigest(const char *s) +{ + if (*s!='$') + return is_legal_nickname(s); + else + return is_legal_hexdigest(s); +} + +/** Return true iff <b>s</b> is a valid hex-encoded identity-key + * digest. (That is, an optional $, followed by 40 hex characters, + * followed by either nothing, or = or ~ followed by a nickname, or + * a character other than =, ~, or a hex character.) + */ +int +is_legal_hexdigest(const char *s) +{ + size_t len; + tor_assert(s); + if (s[0] == '$') s++; + len = strlen(s); + if (len > HEX_DIGEST_LEN) { + if (s[HEX_DIGEST_LEN] == '=' || + s[HEX_DIGEST_LEN] == '~') { + if (!is_legal_nickname(s+HEX_DIGEST_LEN+1)) + return 0; + } else { + return 0; + } + } + return (len >= HEX_DIGEST_LEN && + strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN); +} + +/** + * Longest allowed output of format_node_description, plus 1 character for + * NUL. This allows space for: + * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" + * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" + * plus a terminating NUL. + */ +#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN) + +/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to + * hold a human-readable description of a node with identity digest + * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>, + * and address <b>addr</b> or <b>addr32h</b>. + * + * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to + * NULL. The <b>addr32h</b> field is optional and may be set to 0. + * + * Return a pointer to the front of <b>buf</b>. + */ +const char * +format_node_description(char *buf, + const char *id_digest, + int is_named, + const char *nickname, + const tor_addr_t *addr, + uint32_t addr32h) +{ + char *cp; + + if (!buf) + return "<NULL BUFFER>"; + + buf[0] = '$'; + base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN); + cp = buf+1+HEX_DIGEST_LEN; + if (nickname) { + buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~'; + strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1); + cp += strlen(cp); + } + if (addr32h || addr) { + memcpy(cp, " at ", 4); + cp += 4; + if (addr) { + tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0); + } else { + struct in_addr in; + in.s_addr = htonl(addr32h); + tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN); + } + } + return buf; +} + +/** Return a human-readable description of the routerinfo_t <b>ri</b>. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +const char * +router_describe(const routerinfo_t *ri) +{ + static char buf[NODE_DESC_BUF_LEN]; + + if (!ri) + return "<null>"; + return format_node_description(buf, + ri->cache_info.identity_digest, + 0, + ri->nickname, + NULL, + ri->addr); +} + +/** Return a human-readable description of the node_t <b>node</b>. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +const char * +node_describe(const node_t *node) +{ + static char buf[NODE_DESC_BUF_LEN]; + const char *nickname = NULL; + uint32_t addr32h = 0; + int is_named = 0; + + if (!node) + return "<null>"; + + if (node->rs) { + nickname = node->rs->nickname; + is_named = node->rs->is_named; + addr32h = node->rs->addr; + } else if (node->ri) { + nickname = node->ri->nickname; + addr32h = node->ri->addr; + } + + return format_node_description(buf, + node->identity, + is_named, + nickname, + NULL, + addr32h); +} + +/** Return a human-readable description of the routerstatus_t <b>rs</b>. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +const char * +routerstatus_describe(const routerstatus_t *rs) +{ + static char buf[NODE_DESC_BUF_LEN]; + + if (!rs) + return "<null>"; + return format_node_description(buf, + rs->identity_digest, + rs->is_named, + rs->nickname, + NULL, + rs->addr); +} + +/** Return a human-readable description of the extend_info_t <b>ei</b>. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +const char * +extend_info_describe(const extend_info_t *ei) +{ + static char buf[NODE_DESC_BUF_LEN]; + + if (!ei) + return "<null>"; + return format_node_description(buf, + ei->identity_digest, + 0, + ei->nickname, + &ei->addr, + 0); +} + +/** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the + * verbose representation of the identity of <b>router</b>. The format is: + * A dollar sign. + * The upper-case hexadecimal encoding of the SHA1 hash of router's identity. + * A "=" if the router is named (no longer implemented); a "~" if it is not. + * The router's nickname. + **/ +void +router_get_verbose_nickname(char *buf, const routerinfo_t *router) +{ + buf[0] = '$'; + base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest, + DIGEST_LEN); + buf[1+HEX_DIGEST_LEN] = '~'; + strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1); +} + +/** Forget that we have issued any router-related warnings, so that we'll + * warn again if we see the same errors. */ +void +router_reset_warnings(void) +{ + if (warned_nonexistent_family) { + SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); + smartlist_clear(warned_nonexistent_family); + } +} + +/** Given a router purpose, convert it to a string. Don't call this on + * ROUTER_PURPOSE_UNKNOWN: The whole point of that value is that we don't + * know its string representation. */ +const char * +router_purpose_to_string(uint8_t p) +{ + switch (p) + { + case ROUTER_PURPOSE_GENERAL: return "general"; + case ROUTER_PURPOSE_BRIDGE: return "bridge"; + case ROUTER_PURPOSE_CONTROLLER: return "controller"; + default: + tor_assert(0); + } + return NULL; +} + +/** Given a string, convert it to a router purpose. */ +uint8_t +router_purpose_from_string(const char *s) +{ + if (!strcmp(s, "general")) + return ROUTER_PURPOSE_GENERAL; + else if (!strcmp(s, "bridge")) + return ROUTER_PURPOSE_BRIDGE; + else if (!strcmp(s, "controller")) + return ROUTER_PURPOSE_CONTROLLER; + else + return ROUTER_PURPOSE_UNKNOWN; +} + +/** Release all static resources held in router.c */ +void +router_free_all(void) +{ + crypto_pk_free(onionkey); + crypto_pk_free(lastonionkey); + crypto_pk_free(server_identitykey); + crypto_pk_free(client_identitykey); + + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_pk_free(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_pk_free(legacy_signing_key); + authority_cert_free(legacy_key_certificate); + + memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); + memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); + + if (warned_nonexistent_family) { + SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); + smartlist_free(warned_nonexistent_family); + } +} + +/** Return a smartlist of tor_addr_port_t's with all the OR ports of + <b>ri</b>. Note that freeing of the items in the list as well as + the smartlist itself is the callers responsibility. */ +smartlist_t * +router_get_all_orports(const routerinfo_t *ri) +{ + tor_assert(ri); + node_t fake_node; + memset(&fake_node, 0, sizeof(fake_node)); + /* we don't modify ri, fake_node is passed as a const node_t * + */ + fake_node.ri = (routerinfo_t *)ri; + return node_get_all_orports(&fake_node); +} + +/* From the given RSA key object, convert it to ASN-1 encoded format and set + * the newly allocated object in onion_pkey_out. The length of the key is set + * in onion_pkey_len_out. */ +void +router_set_rsa_onion_pkey(const crypto_pk_t *pk, char **onion_pkey_out, + size_t *onion_pkey_len_out) +{ + int len; + char buf[1024]; + + tor_assert(pk); + tor_assert(onion_pkey_out); + tor_assert(onion_pkey_len_out); + + len = crypto_pk_asn1_encode(pk, buf, sizeof(buf)); + if (BUG(len < 0)) { + goto done; + } + + *onion_pkey_out = tor_memdup(buf, len); + *onion_pkey_len_out = len; + + done: + return; +} + +/* From an ASN-1 encoded onion pkey, return a newly allocated RSA key object. + * It is the caller responsability to free the returned object. + * + * Return NULL if the pkey is NULL, malformed or if the length is 0. */ +crypto_pk_t * +router_get_rsa_onion_pkey(const char *pkey, size_t pkey_len) +{ + if (!pkey || pkey_len == 0) { + return NULL; + } + return crypto_pk_asn1_decode(pkey, pkey_len); +} diff --cc src/feature/stats/rephist.c index 6bb680c5d,000000000..e24728ca7 mode 100644,000000..100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@@ -1,3220 -1,0 +1,3220 @@@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rephist.c + * \brief Basic history and performance-tracking functionality. + * + * Basic history and performance-tracking functionality to remember + * which servers have worked in the past, how much bandwidth we've + * been using, which ports we tend to want, and so on; further, + * exit port statistics, cell statistics, and connection statistics. + * + * The history and information tracked in this module could sensibly be + * divided into several categories: + * + * <ul><li>Statistics used by authorities to remember the uptime and + * stability information about various relays, including "uptime", + * "weighted fractional uptime" and "mean time between failures". + * + * <li>Bandwidth usage history, used by relays to self-report how much + * bandwidth they've used for different purposes over last day or so, + * in order to generate the {dirreq-,}{read,write}-history lines in + * that they publish. + * + * <li>Predicted ports, used by clients to remember how long it's been + * since they opened an exit connection to each given target + * port. Clients use this information in order to try to keep circuits + * open to exit nodes that can connect to the ports that they care + * about. (The predicted ports mechanism also handles predicted circuit + * usage that _isn't_ port-specific, such as resolves, internal circuits, + * and so on.) + * + * <li>Public key operation counters, for tracking how many times we've + * done each public key operation. (This is unmaintained and we should + * remove it.) + * + * <li>Exit statistics by port, used by exits to keep track of the + * number of streams and bytes they've served at each exit port, so they + * can generate their exit-kibibytes-{read,written} and + * exit-streams-opened statistics. + * + * <li>Circuit stats, used by relays instances to tract circuit + * queue fullness and delay over time, and generate cell-processed-cells, + * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile + * statistics. + * + * <li>Descriptor serving statistics, used by directory caches to track + * how many descriptors they've served. + * + * <li>Connection statistics, used by relays to track one-way and + * bidirectional connections. + * + * <li>Onion handshake statistics, used by relays to count how many + * TAP and ntor handshakes they've handled. + * + * <li>Hidden service statistics, used by relays to count rendezvous + * traffic and HSDir-stored descriptors. + * + * <li>Link protocol statistics, used by relays to count how many times + * each link protocol has been used. + * + * </ul> + * + * The entry points for this module are scattered throughout the + * codebase. Sending data, receiving data, connecting to a relay, + * losing a connection to a relay, and so on can all trigger a change in + * our current stats. Relays also invoke this module in order to + * extract their statistics when building routerinfo and extrainfo + * objects in router.c. + * + * TODO: This module should be broken up. + * + * (The "rephist" name originally stood for "reputation and history". ) + **/ + +#include "core/or/or.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "app/config/config.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/stats/rephist.h" +#include "feature/relay/router.h" +#include "feature/nodelist/routerlist.h" +#include "ht.h" +#include "core/or/channelpadding.h" +#include "core/or/connection_or.h" +#include "app/config/statefile.h" + +#include "feature/nodelist/networkstatus_st.h" +#include "core/or/or_circuit_st.h" +#include "app/config/or_state_st.h" + +#include "lib/container/bloomfilt.h" +#include "lib/container/order.h" +#include "lib/math/fp.h" +#include "lib/math/laplace.h" + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +static void bw_arrays_init(void); +static void predicted_ports_alloc(void); + +/** Total number of bytes currently allocated in fields used by rephist.c. */ +uint64_t rephist_total_alloc=0; +/** Number of or_history_t objects currently allocated. */ +uint32_t rephist_total_num=0; + +/** If the total weighted run count of all runs for a router ever falls + * below this amount, the router can be treated as having 0 MTBF. */ +#define STABILITY_EPSILON 0.0001 +/** Value by which to discount all old intervals for MTBF purposes. This + * is compounded every STABILITY_INTERVAL. */ +#define STABILITY_ALPHA 0.95 +/** Interval at which to discount all old intervals for MTBF purposes. */ +#define STABILITY_INTERVAL (12*60*60) +/* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an + * interval that just ended counts twice as much as one that ended a week ago, + * 20X as much as one that ended a month ago, and routers that have had no + * uptime data for about half a year will get forgotten.) */ + +/** History of an OR. */ +typedef struct or_history_t { + /** When did we start tracking this OR? */ + time_t since; + /** When did we most recently note a change to this OR? */ + time_t changed; + + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + + /* === For MTBF tracking: */ + /** Weighted sum total of all times that this router has been online. + */ + unsigned long weighted_run_length; + /** If the router is now online (according to stability-checking rules), + * when did it come online? */ + time_t start_of_run; + /** Sum of weights for runs in weighted_run_length. */ + double total_run_weights; + /* === For fractional uptime tracking: */ + time_t start_of_downtime; + unsigned long weighted_uptime; + unsigned long total_weighted_time; +} or_history_t; + +/** + * This structure holds accounting needed to calculate the padding overhead. + */ +typedef struct padding_counts_t { + /** Total number of cells we have received, including padding */ + uint64_t read_cell_count; + /** Total number of cells we have sent, including padding */ + uint64_t write_cell_count; + /** Total number of CELL_PADDING cells we have received */ + uint64_t read_pad_cell_count; + /** Total number of CELL_PADDING cells we have sent */ + uint64_t write_pad_cell_count; + /** Total number of read cells on padding-enabled conns */ + uint64_t enabled_read_cell_count; + /** Total number of sent cells on padding-enabled conns */ + uint64_t enabled_write_cell_count; + /** Total number of read CELL_PADDING cells on padding-enabled cons */ + uint64_t enabled_read_pad_cell_count; + /** Total number of sent CELL_PADDING cells on padding-enabled cons */ + uint64_t enabled_write_pad_cell_count; + /** Total number of RELAY_DROP cells we have received */ + uint64_t read_drop_cell_count; + /** Total number of RELAY_DROP cells we have sent */ + uint64_t write_drop_cell_count; + /** The maximum number of padding timers we've seen in 24 hours */ + uint64_t maximum_chanpad_timers; + /** When did we first copy padding_current into padding_published? */ + char first_published_at[ISO_TIME_LEN+1]; +} padding_counts_t; + +/** Holds the current values of our padding statistics. + * It is not published until it is transferred to padding_published. */ +static padding_counts_t padding_current; + +/** Remains fixed for a 24 hour period, and then is replaced + * by a redacted copy of padding_current */ +static padding_counts_t padding_published; + +/** When did we last multiply all routers' weighted_run_length and + * total_run_weights by STABILITY_ALPHA? */ +static time_t stability_last_downrated = 0; + +/** */ +static time_t started_tracking_stability = 0; + +/** Map from hex OR identity digest to or_history_t. */ +static digestmap_t *history_map = NULL; + +/** Return the or_history_t for the OR with identity digest <b>id</b>, + * creating it if necessary. */ +static or_history_t * +get_or_history(const char* id) +{ + or_history_t *hist; + + if (tor_digest_is_zero(id)) + return NULL; + + hist = digestmap_get(history_map, id); + if (!hist) { + hist = tor_malloc_zero(sizeof(or_history_t)); + rephist_total_alloc += sizeof(or_history_t); + rephist_total_num++; + hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); + digestmap_set(history_map, id, hist); + } + return hist; +} + +/** Helper: free storage held by a single OR history entry. */ +static void +free_or_history(void *_hist) +{ + or_history_t *hist = _hist; + rephist_total_alloc -= sizeof(or_history_t); + rephist_total_num--; + tor_free(hist); +} + +/** Initialize the static data structures for tracking history. */ +void +rep_hist_init(void) +{ + history_map = digestmap_new(); + bw_arrays_init(); + predicted_ports_alloc(); +} + +/** We have just decided that this router with identity digest <b>id</b> is + * reachable, meaning we will give it a "Running" flag for the next while. */ +void +rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) +{ + or_history_t *hist = get_or_history(id); + int was_in_run = 1; + char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; + + tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && hist->last_reached_port && + at_port != hist->last_reached_port; + + if (!started_tracking_stability) + started_tracking_stability = time(NULL); + if (!hist->start_of_run) { + hist->start_of_run = when; + was_in_run = 0; + } + if (hist->start_of_downtime) { + long down_length; + + format_local_iso_time(tbuf, hist->start_of_downtime); + log_info(LD_HIST, "Router %s is now Running; it had been down since %s.", + hex_str(id, DIGEST_LEN), tbuf); + if (was_in_run) + log_info(LD_HIST, " (Paradoxically, it was already Running too.)"); + + down_length = when - hist->start_of_downtime; + hist->total_weighted_time += down_length; + hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); + } else { + format_local_iso_time(tbuf, hist->start_of_run); + if (was_in_run) + log_debug(LD_HIST, "Router %s is still Running; it has been Running " + "since %s", hex_str(id, DIGEST_LEN), tbuf); + else + log_info(LD_HIST,"Router %s is now Running; it was previously untracked", + hex_str(id, DIGEST_LEN)); + } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; +} + +/** We have just decided that this router is unreachable, meaning + * we are taking away its "Running" flag. */ +void +rep_hist_note_router_unreachable(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + char tbuf[ISO_TIME_LEN+1]; + int was_running = 0; + if (!started_tracking_stability) + started_tracking_stability = time(NULL); + + tor_assert(hist); + if (hist->start_of_run) { + /*XXXX We could treat failed connections differently from failed + * connect attempts. */ + long run_length = when - hist->start_of_run; + format_local_iso_time(tbuf, hist->start_of_run); + + hist->total_run_weights += 1.0; + hist->start_of_run = 0; + if (run_length < 0) { + unsigned long penalty = -run_length; +#define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } + was_running = 1; + log_info(LD_HIST, "Router %s is now non-Running: it had previously been " + "Running since %s. Its total weighted uptime is %lu/%lu.", + hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime, + hist->total_weighted_time); + } + if (!hist->start_of_downtime) { + hist->start_of_downtime = when; + + if (!was_running) + log_info(LD_HIST, "Router %s is now non-Running; it was previously " + "untracked.", hex_str(id, DIGEST_LEN)); + } else { + if (!was_running) { + format_local_iso_time(tbuf, hist->start_of_downtime); + + log_info(LD_HIST, "Router %s is still non-Running; it has been " + "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf); + } + } +} + +/** Mark a router with ID <b>id</b> as non-Running, and retroactively declare + * that it has never been running: give it no stability and no WFU. */ +void +rep_hist_make_router_pessimal(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + tor_assert(hist); + + rep_hist_note_router_unreachable(id, when); + + hist->weighted_run_length = 0; + hist->weighted_uptime = 0; +} + +/** Helper: Discount all old MTBF data, if it is time to do so. Return + * the time at which we should next discount MTBF data. */ +time_t +rep_hist_downrate_old_runs(time_t now) +{ + digestmap_iter_t *orhist_it; + const char *digest1; + or_history_t *hist; + void *hist_p; + double alpha = 1.0; + + if (!history_map) + history_map = digestmap_new(); + if (!stability_last_downrated) + stability_last_downrated = now; + if (stability_last_downrated + STABILITY_INTERVAL > now) + return stability_last_downrated + STABILITY_INTERVAL; + + /* Okay, we should downrate the data. By how much? */ + while (stability_last_downrated + STABILITY_INTERVAL < now) { + stability_last_downrated += STABILITY_INTERVAL; + alpha *= STABILITY_ALPHA; + } + + log_info(LD_HIST, "Discounting all old stability info by a factor of %f", + alpha); + + /* Multiply every w_r_l, t_r_w pair by alpha. */ + for (orhist_it = digestmap_iter_init(history_map); + !digestmap_iter_done(orhist_it); + orhist_it = digestmap_iter_next(history_map,orhist_it)) { + digestmap_iter_get(orhist_it, &digest1, &hist_p); + hist = hist_p; + + hist->weighted_run_length = + (unsigned long)(hist->weighted_run_length * alpha); + hist->total_run_weights *= alpha; + + hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha); + hist->total_weighted_time = (unsigned long) + (hist->total_weighted_time * alpha); + } + + return stability_last_downrated + STABILITY_INTERVAL; +} + +/** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */ +static double +get_stability(or_history_t *hist, time_t when) +{ + long total = hist->weighted_run_length; + double total_weights = hist->total_run_weights; + + if (hist->start_of_run) { + /* We're currently in a run. Let total and total_weights hold the values + * they would hold if the current run were to end now. */ + total += (when-hist->start_of_run); + total_weights += 1.0; + } + if (total_weights < STABILITY_EPSILON) { + /* Round down to zero, and avoid divide-by-zero. */ + return 0.0; + } + + return total / total_weights; +} + +/** Return the total amount of time we've been observing, with each run of + * time downrated by the appropriate factor. */ +static long +get_total_weighted_time(or_history_t *hist, time_t when) +{ + long total = hist->total_weighted_time; + if (hist->start_of_run) { + total += (when - hist->start_of_run); + } else if (hist->start_of_downtime) { + total += (when - hist->start_of_downtime); + } + return total; +} + +/** Helper: Return the weighted percent-of-time-online of the router with + * history <b>hist</b>. */ +static double +get_weighted_fractional_uptime(or_history_t *hist, time_t when) +{ + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; + + if (hist->start_of_run) { + long run_length = (when - hist->start_of_run); + up += run_length; + total += run_length; + } else if (hist->start_of_downtime) { + total += (when - hist->start_of_downtime); + } + + if (!total) { + /* Avoid calling anybody's uptime infinity (which should be impossible if + * the code is working), or NaN (which can happen for any router we haven't + * observed up or down yet). */ + return 0.0; + } + + return ((double) up) / total; +} + +/** Return how long the router whose identity digest is <b>id</b> has + * been reachable. Return 0 if the router is unknown or currently deemed + * unreachable. */ +long +rep_hist_get_uptime(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0; + if (!hist->start_of_run || when < hist->start_of_run) + return 0; + return when - hist->start_of_run; +} + +/** Return an estimated MTBF for the router whose identity digest is + * <b>id</b>. Return 0 if the router is unknown. */ +double +rep_hist_get_stability(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0.0; + + return get_stability(hist, when); +} + +/** Return an estimated percent-of-time-online for the router whose identity + * digest is <b>id</b>. Return 0 if the router is unknown. */ +double +rep_hist_get_weighted_fractional_uptime(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0.0; + + return get_weighted_fractional_uptime(hist, when); +} + +/** Return a number representing how long we've known about the router whose + * digest is <b>id</b>. Return 0 if the router is unknown. + * + * Be careful: this measure increases monotonically as we know the router for + * longer and longer, but it doesn't increase linearly. + */ +long +rep_hist_get_weighted_time_known(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0; + + return get_total_weighted_time(hist, when); +} + +/** Return true if we've been measuring MTBFs for long enough to + * pronounce on Stability. */ +int +rep_hist_have_measured_enough_stability(void) +{ + /* XXXX++ This doesn't do so well when we change our opinion + * as to whether we're tracking router stability. */ + return started_tracking_stability < time(NULL) - 4*60*60; +} + +/** Log all the reliability data we have remembered, with the chosen + * severity. + */ +void +rep_hist_dump_stats(time_t now, int severity) +{ + digestmap_iter_t *orhist_it; + const char *name1, *digest1; + char hexdigest1[HEX_DIGEST_LEN+1]; + or_history_t *or_history; + void *or_history_p; + const node_t *node; + + rep_history_clean(now - get_options()->RephistTrackTime); + + tor_log(severity, LD_HIST, "--------------- Dumping history information:"); + + for (orhist_it = digestmap_iter_init(history_map); + !digestmap_iter_done(orhist_it); + orhist_it = digestmap_iter_next(history_map,orhist_it)) { + double s; + long stability; + digestmap_iter_get(orhist_it, &digest1, &or_history_p); + or_history = (or_history_t*) or_history_p; + + if ((node = node_get_by_id(digest1)) && node_get_nickname(node)) + name1 = node_get_nickname(node); + else + name1 = "(unknown)"; + base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN); + s = get_stability(or_history, now); + stability = (long)s; + tor_log(severity, LD_HIST, + "OR %s [%s]: wmtbf %lu:%02lu:%02lu", + name1, hexdigest1, + stability/3600, (stability/60)%60, stability%60); + } +} + +/** Remove history info for routers/links that haven't changed since + * <b>before</b>. + */ +void +rep_history_clean(time_t before) +{ + int authority = authdir_mode(get_options()); + or_history_t *or_history; + void *or_history_p; + digestmap_iter_t *orhist_it; + const char *d1; + + orhist_it = digestmap_iter_init(history_map); + while (!digestmap_iter_done(orhist_it)) { + int should_remove; + digestmap_iter_get(orhist_it, &d1, &or_history_p); + or_history = or_history_p; + + should_remove = authority ? + (or_history->total_run_weights < STABILITY_EPSILON && + !or_history->start_of_run) + : (or_history->changed < before); + if (should_remove) { + orhist_it = digestmap_iter_next_rmv(history_map, orhist_it); + free_or_history(or_history); + continue; + } + orhist_it = digestmap_iter_next(history_map, orhist_it); + } +} + +/** Write MTBF data to disk. Return 0 on success, negative on failure. + * + * If <b>missing_means_down</b>, then if we're about to write an entry + * that is still considered up but isn't in our routerlist, consider it + * to be down. */ +int +rep_hist_record_mtbf_data(time_t now, int missing_means_down) +{ + char time_buf[ISO_TIME_LEN+1]; + + digestmap_iter_t *orhist_it; + const char *digest; + void *or_history_p; + or_history_t *hist; + open_file_t *open_file = NULL; + FILE *f; + + { + char *filename = get_datadir_fname("router-stability"); + f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600, + &open_file); + tor_free(filename); + if (!f) + return -1; + } + + /* File format is: + * FormatLine *KeywordLine Data + * + * FormatLine = "format 1" NL + * KeywordLine = Keyword SP Arguments NL + * Data = "data" NL *RouterMTBFLine "." NL + * RouterMTBFLine = Fingerprint SP WeightedRunLen SP + * TotalRunWeights [SP S=StartRunTime] NL + */ +#define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END +#define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END + + PUT("format 2\n"); + + format_iso_time(time_buf, time(NULL)); + PRINTF((f, "stored-at %s\n", time_buf)); + + if (started_tracking_stability) { + format_iso_time(time_buf, started_tracking_stability); + PRINTF((f, "tracked-since %s\n", time_buf)); + } + if (stability_last_downrated) { + format_iso_time(time_buf, stability_last_downrated); + PRINTF((f, "last-downrated %s\n", time_buf)); + } + + PUT("data\n"); + + /* XXX Nick: now bridge auths record this for all routers too. + * Should we make them record it only for bridge routers? -RD + * Not for 0.2.0. -NM */ + for (orhist_it = digestmap_iter_init(history_map); + !digestmap_iter_done(orhist_it); + orhist_it = digestmap_iter_next(history_map,orhist_it)) { + char dbuf[HEX_DIGEST_LEN+1]; + const char *t = NULL; + digestmap_iter_get(orhist_it, &digest, &or_history_p); + hist = (or_history_t*) or_history_p; + + base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN); + + if (missing_means_down && hist->start_of_run && + !connection_or_digest_is_known_relay(digest)) { + /* We think this relay is running, but it's not listed in our + * consensus. Somehow it fell out without telling us it went + * down. Complain and also correct it. */ + log_info(LD_HIST, + "Relay '%s' is listed as up in rephist, but it's not in " + "our routerlist. Correcting.", dbuf); + rep_hist_note_router_unreachable(digest, now); + } + + PRINTF((f, "R %s\n", dbuf)); + if (hist->start_of_run > 0) { + format_iso_time(time_buf, hist->start_of_run); + t = time_buf; + } + PRINTF((f, "+MTBF %lu %.5f%s%s\n", + hist->weighted_run_length, hist->total_run_weights, + t ? " S=" : "", t ? t : "")); + t = NULL; + if (hist->start_of_downtime > 0) { + format_iso_time(time_buf, hist->start_of_downtime); + t = time_buf; + } + PRINTF((f, "+WFU %lu %lu%s%s\n", + hist->weighted_uptime, hist->total_weighted_time, + t ? " S=" : "", t ? t : "")); + } + + PUT(".\n"); + +#undef PUT +#undef PRINTF + + return finish_writing_to_file(open_file); + err: + abort_writing_to_file(open_file); + return -1; +} + +/** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and + * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no + * such line exists. */ +static int +find_next_with(smartlist_t *sl, int i, const char *prefix) +{ + for ( ; i < smartlist_len(sl); ++i) { + const char *line = smartlist_get(sl, i); + if (!strcmpstart(line, prefix)) + return i; + if (!strcmpstart(line, "R ")) + return -1; + } + return -1; +} + +/** How many bad times has parse_possibly_bad_iso_time() parsed? */ +static int n_bogus_times = 0; +/** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but + * round any pre-1970 date to Jan 1, 1970. */ +static int +parse_possibly_bad_iso_time(const char *s, time_t *time_out) +{ + int year; + char b[5]; + strlcpy(b, s, sizeof(b)); + b[4] = '\0'; + year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL); + if (year < 1970) { + *time_out = 0; + ++n_bogus_times; + return 0; + } else + return parse_iso_time(s, time_out); +} + +/** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which + * says we started measuring at <b>started_measuring</b>. Return a new number + * that's about as much before <b>now</b> as <b>t</b> was before + * <b>stored_at</b>. + */ +static inline time_t +correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring) +{ + if (t < started_measuring - 24*60*60*365) + return 0; + else if (t < started_measuring) + return started_measuring; + else if (t > stored_at) + return 0; + else { + long run_length = stored_at - t; + t = (time_t)(now - run_length); + if (t < started_measuring) + t = started_measuring; + return t; + } +} + +/** Load MTBF data from disk. Returns 0 on success or recoverable error, -1 + * on failure. */ +int +rep_hist_load_mtbf_data(time_t now) +{ + /* XXXX won't handle being called while history is already populated. */ + smartlist_t *lines; + const char *line = NULL; + int r=0, i; + time_t last_downrated = 0, stored_at = 0, tracked_since = 0; + time_t latest_possible_start = now; + long format = -1; + + { + char *filename = get_datadir_fname("router-stability"); + char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + tor_free(filename); + if (!d) + return -1; + lines = smartlist_new(); + smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0); + tor_free(d); + } + + { + const char *firstline; + if (smartlist_len(lines)>4) { + firstline = smartlist_get(lines, 0); + if (!strcmpstart(firstline, "format ")) + format = tor_parse_long(firstline+strlen("format "), + 10, -1, LONG_MAX, NULL, NULL); + } + } + if (format != 1 && format != 2) { + log_warn(LD_HIST, + "Unrecognized format in mtbf history file. Skipping."); + goto err; + } + for (i = 1; i < smartlist_len(lines); ++i) { + line = smartlist_get(lines, i); + if (!strcmp(line, "data")) + break; + if (!strcmpstart(line, "last-downrated ")) { + if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0) + log_warn(LD_HIST,"Couldn't parse downrate time in mtbf " + "history file."); + } + if (!strcmpstart(line, "stored-at ")) { + if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0) + log_warn(LD_HIST,"Couldn't parse stored time in mtbf " + "history file."); + } + if (!strcmpstart(line, "tracked-since ")) { + if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0) + log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf " + "history file."); + } + } + if (last_downrated > now) + last_downrated = now; + if (tracked_since > now) + tracked_since = now; + + if (!stored_at) { + log_warn(LD_HIST, "No stored time recorded."); + goto err; + } + + if (line && !strcmp(line, "data")) + ++i; + + n_bogus_times = 0; + + for (; i < smartlist_len(lines); ++i) { + char digest[DIGEST_LEN]; + char hexbuf[HEX_DIGEST_LEN+1]; + char mtbf_timebuf[ISO_TIME_LEN+1]; + char wfu_timebuf[ISO_TIME_LEN+1]; + time_t start_of_run = 0; + time_t start_of_downtime = 0; + int have_mtbf = 0, have_wfu = 0; + long wrl = 0; + double trw = 0; + long wt_uptime = 0, total_wt_time = 0; + int n; + or_history_t *hist; + line = smartlist_get(lines, i); + if (!strcmp(line, ".")) + break; + + mtbf_timebuf[0] = '\0'; + wfu_timebuf[0] = '\0'; + + if (format == 1) { + n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s", + hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11); + if (n != 3 && n != 5) { + log_warn(LD_HIST, "Couldn't scan line %s", escaped(line)); + continue; + } + have_mtbf = 1; + } else { + // format == 2. + int mtbf_idx, wfu_idx; + if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN) + continue; + strlcpy(hexbuf, line+2, sizeof(hexbuf)); + mtbf_idx = find_next_with(lines, i+1, "+MTBF "); + wfu_idx = find_next_with(lines, i+1, "+WFU "); + if (mtbf_idx >= 0) { + const char *mtbfline = smartlist_get(lines, mtbf_idx); + n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s", + &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11); + if (n == 2 || n == 4) { + have_mtbf = 1; + } else { + log_warn(LD_HIST, "Couldn't scan +MTBF line %s", + escaped(mtbfline)); + } + } + if (wfu_idx >= 0) { + const char *wfuline = smartlist_get(lines, wfu_idx); + n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s", + &wt_uptime, &total_wt_time, + wfu_timebuf, wfu_timebuf+11); + if (n == 2 || n == 4) { + have_wfu = 1; + } else { + log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline)); + } + } + if (wfu_idx > i) + i = wfu_idx; + if (mtbf_idx > i) + i = mtbf_idx; + } + if (base16_decode(digest, DIGEST_LEN, + hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) { + log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf)); + continue; + } + hist = get_or_history(digest); + if (!hist) + continue; + + if (have_mtbf) { + if (mtbf_timebuf[0]) { + mtbf_timebuf[10] = ' '; + if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0) + log_warn(LD_HIST, "Couldn't parse time %s", + escaped(mtbf_timebuf)); + } + hist->start_of_run = correct_time(start_of_run, now, stored_at, + tracked_since); + if (hist->start_of_run < latest_possible_start + wrl) + latest_possible_start = (time_t)(hist->start_of_run - wrl); + + hist->weighted_run_length = wrl; + hist->total_run_weights = trw; + } + if (have_wfu) { + if (wfu_timebuf[0]) { + wfu_timebuf[10] = ' '; + if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0) + log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf)); + } + } + hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at, + tracked_since); + hist->weighted_uptime = wt_uptime; + hist->total_weighted_time = total_wt_time; + } + if (strcmp(line, ".")) + log_warn(LD_HIST, "Truncated MTBF file."); + + if (tracked_since < 86400*365) /* Recover from insanely early value. */ + tracked_since = latest_possible_start; + + stability_last_downrated = last_downrated; + started_tracking_stability = tracked_since; + + goto done; + err: + r = -1; + done: + SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); + smartlist_free(lines); + return r; +} + +/** For how many seconds do we keep track of individual per-second bandwidth + * totals? */ +#define NUM_SECS_ROLLING_MEASURE 10 +/** How large are the intervals for which we track and report bandwidth use? */ +#define NUM_SECS_BW_SUM_INTERVAL (24*60*60) +/** How far in the past do we remember and publish bandwidth use? */ +#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60) +/** How many bandwidth usage intervals do we remember? (derived) */ +#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL) + +/** Structure to track bandwidth use, and remember the maxima for a given + * time period. + */ +typedef struct bw_array_t { + /** Observation array: Total number of bytes transferred in each of the last + * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */ + uint64_t obs[NUM_SECS_ROLLING_MEASURE]; + int cur_obs_idx; /**< Current position in obs. */ + time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */ + uint64_t total_obs; /**< Total for all members of obs except + * obs[cur_obs_idx] */ + uint64_t max_total; /**< Largest value that total_obs has taken on in the + * current period. */ + uint64_t total_in_period; /**< Total bytes transferred in the current + * period. */ + + /** When does the next period begin? */ + time_t next_period; + /** Where in 'maxima' should the maximum bandwidth usage for the current + * period be stored? */ + int next_max_idx; + /** How many values in maxima/totals have been set ever? */ + int num_maxes_set; + /** Circular array of the maximum + * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last + * NUM_TOTALS periods */ + uint64_t maxima[NUM_TOTALS]; + /** Circular array of the total bandwidth usage for the last NUM_TOTALS + * periods */ + uint64_t totals[NUM_TOTALS]; +} bw_array_t; + +/** Shift the current period of b forward by one. */ +static void +commit_max(bw_array_t *b) +{ + /* Store total from current period. */ + b->totals[b->next_max_idx] = b->total_in_period; + /* Store maximum from current period. */ + b->maxima[b->next_max_idx++] = b->max_total; + /* Advance next_period and next_max_idx */ + b->next_period += NUM_SECS_BW_SUM_INTERVAL; + if (b->next_max_idx == NUM_TOTALS) + b->next_max_idx = 0; + if (b->num_maxes_set < NUM_TOTALS) + ++b->num_maxes_set; + /* Reset max_total. */ + b->max_total = 0; + /* Reset total_in_period. */ + b->total_in_period = 0; +} + +/** Shift the current observation time of <b>b</b> forward by one second. */ +static inline void +advance_obs(bw_array_t *b) +{ + int nextidx; + uint64_t total; + + /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE + * seconds; adjust max_total as needed.*/ + total = b->total_obs + b->obs[b->cur_obs_idx]; + if (total > b->max_total) + b->max_total = total; + + nextidx = b->cur_obs_idx+1; + if (nextidx == NUM_SECS_ROLLING_MEASURE) + nextidx = 0; + + b->total_obs = total - b->obs[nextidx]; + b->obs[nextidx]=0; + b->cur_obs_idx = nextidx; + + if (++b->cur_obs_time >= b->next_period) + commit_max(b); +} + +/** Add <b>n</b> bytes to the number of bytes in <b>b</b> for second + * <b>when</b>. */ +static inline void +add_obs(bw_array_t *b, time_t when, uint64_t n) +{ + if (when < b->cur_obs_time) + return; /* Don't record data in the past. */ + + /* If we're currently adding observations for an earlier second than + * 'when', advance b->cur_obs_time and b->cur_obs_idx by an + * appropriate number of seconds, and do all the other housekeeping. */ + while (when > b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ + advance_obs(b); + } + + b->obs[b->cur_obs_idx] += n; + b->total_in_period += n; +} + +/** Allocate, initialize, and return a new bw_array. */ +static bw_array_t * +bw_array_new(void) +{ + bw_array_t *b; + time_t start; + b = tor_malloc_zero(sizeof(bw_array_t)); + rephist_total_alloc += sizeof(bw_array_t); + start = time(NULL); + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + return b; +} + +#define bw_array_free(val) \ + FREE_AND_NULL(bw_array_t, bw_array_free_, (val)) + +/** Free storage held by bandwidth array <b>b</b>. */ +static void +bw_array_free_(bw_array_t *b) +{ + if (!b) { + return; + } + + rephist_total_alloc -= sizeof(bw_array_t); + tor_free(b); +} + +/** Recent history of bandwidth observations for read operations. */ +static bw_array_t *read_array = NULL; +/** Recent history of bandwidth observations for write operations. */ +static bw_array_t *write_array = NULL; +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ +static void +bw_arrays_init(void) +{ + bw_array_free(read_array); + bw_array_free(write_array); + bw_array_free(dir_read_array); + bw_array_free(dir_write_array); + + read_array = bw_array_new(); + write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); +} + +/** Remember that we read <b>num_bytes</b> bytes in second <b>when</b>. + * + * Add num_bytes to the current running total for <b>when</b>. + * + * <b>when</b> can go back to time, but it's safe to ignore calls + * earlier than the latest <b>when</b> you've heard of. + */ +void +rep_hist_note_bytes_written(size_t num_bytes, time_t when) +{ +/* Maybe a circular array for recent seconds, and step to a new point + * every time a new second shows up. Or simpler is to just to have + * a normal array and push down each item every second; it's short. + */ +/* When a new second has rolled over, compute the sum of the bytes we've + * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it + * somewhere. See rep_hist_bandwidth_assess() below. + */ + add_obs(write_array, when, num_bytes); +} + +/** Remember that we wrote <b>num_bytes</b> bytes in second <b>when</b>. + * (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_bytes_read(size_t num_bytes, time_t when) +{ +/* if we're smart, we can make this func and the one above share code */ + add_obs(read_array, when, num_bytes); +} + +/** Remember that we wrote <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when) +{ + add_obs(dir_write_array, when, num_bytes); +} + +/** Remember that we read <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when) +{ + add_obs(dir_read_array, when, num_bytes); +} + +/** Helper: Return the largest value in b->maxima. (This is equal to the + * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last + * NUM_SECS_BW_SUM_IS_VALID seconds.) + */ +static uint64_t +find_largest_max(bw_array_t *b) +{ + int i; + uint64_t max; + max=0; + for (i=0; i<NUM_TOTALS; ++i) { + if (b->maxima[i]>max) + max = b->maxima[i]; + } + return max; +} + +/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly) + * seconds. Find one sum for reading and one for writing. They don't have + * to be at the same time. + * + * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE. + */ - int - rep_hist_bandwidth_assess(void) ++MOCK_IMPL(int, ++rep_hist_bandwidth_assess,(void)) +{ + uint64_t w,r; + r = find_largest_max(read_array); + w = find_largest_max(write_array); + if (r>w) + return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); + else + return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE); +} + +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. + * + * It returns the number of bytes written. + */ +static size_t +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) +{ + char *cp = buf; + int i, n; + const or_options_t *options = get_options(); + uint64_t cutoff; + + if (b->num_maxes_set <= b->next_max_idx) { + /* We haven't been through the circular array yet; time starts at i=0.*/ + i = 0; + } else { + /* We've been around the array at least once. The next i to be + overwritten is the oldest. */ + i = b->next_max_idx; + } + + if (options->RelayBandwidthRate) { + /* We don't want to report that we used more bandwidth than the max we're + * willing to relay; otherwise everybody will know how much traffic + * we used ourself. */ + cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL; + } else { + cutoff = UINT64_MAX; + } + + for (n=0; n<b->num_maxes_set; ++n,++i) { + uint64_t total; + if (i >= NUM_TOTALS) + i -= NUM_TOTALS; + tor_assert(i < NUM_TOTALS); + /* Round the bandwidth used down to the nearest 1k. */ + total = b->totals[i] & ~0x3ff; + if (total > cutoff) + total = cutoff; + + if (n==(b->num_maxes_set-1)) + tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total)); + else + tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total)); + cp += strlen(cp); + } + return cp-buf; +} + +/** Allocate and return lines for representing this server's bandwidth + * history in its descriptor. We publish these lines in our extra-info + * descriptor. + */ +char * +rep_hist_get_bandwidth_lines(void) +{ + char *buf, *cp; + char t[ISO_TIME_LEN+1]; + int r; + bw_array_t *b = NULL; + const char *desc = NULL; + size_t len; + + /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ +/* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS) + len = (67+MAX_HIST_VALUE_LEN)*4; + buf = tor_malloc_zero(len); + cp = buf; + for (r=0;r<4;++r) { + char tmp[MAX_HIST_VALUE_LEN]; + size_t slen; + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } + tor_assert(b); + slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + continue; + format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); + tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", + desc, t, NUM_SECS_BW_SUM_INTERVAL); + cp += strlen(cp); + strlcat(cp, tmp, len-(cp-buf)); + cp += slen; + strlcat(cp, "\n", len-(cp-buf)); + ++cp; + } + return buf; +} + +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. Done before writing out a new state file. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + int i,j; + uint64_t maxval; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_new(); + *s_maxima = smartlist_new(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_new(); + *s_maxima = smartlist_new(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + if (i >= NUM_TOTALS) + i = 0; + smartlist_add_asprintf(*s_values, "%"PRIu64, + (b->totals[i] & ~0x3ff)); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + smartlist_add_asprintf(*s_maxima, "%"PRIu64, + (maxval & ~0x3ff)); + } + smartlist_add_asprintf(*s_values, "%"PRIu64, + (b->total_in_period & ~0x3ff)); + maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; + smartlist_add_asprintf(*s_maxima, "%"PRIu64, + (maxval & ~0x3ff)); +} + +/** Update <b>state</b> with the newest bandwidth history. Done before + * writing out a new state file. */ +void +rep_hist_update_state(or_state_t *state) +{ +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) + + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); + + if (server_mode(get_options())) { + or_state_mark_dirty(state, time(NULL)+(2*3600)); + } +#undef UPDATE +} + +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. Done while reading the state file. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) +{ + time_t now = time(NULL); + int retval = 0; + time_t start; + + uint64_t v, mv; + int i,ok,ok_m = 0; + int have_maxima = s_maxima && s_values && + (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; + v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; + } + if (!ok) { + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); + } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + + if (start < now) { + time_t cur_start = start; + time_t actual_interval_len = s_interval; + uint64_t cur_val = 0; + /* Calculate the average per second. This is the best we can do + * because our state file doesn't have per-second resolution. */ + if (start + s_interval > now) + actual_interval_len = now - start; + cur_val = v / actual_interval_len; + /* This is potentially inefficient, but since we don't do it very + * often it should be ok. */ + while (cur_start < start + actual_interval_len) { + add_obs(b, cur_start, cur_val); + ++cur_start; + } + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += actual_interval_len; + } + } SMARTLIST_FOREACH_END(cp); + } + + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; + } + b->total_obs = 0; + + return retval; +} + +/** Set bandwidth history from the state file we just loaded. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD + if (!all_ok) { + *err = tor_strdup("Parsing of bandwidth history values failed"); + /* and create fresh arrays */ + bw_arrays_init(); + return -1; + } + return 0; +} + +/*********************************************************************/ + +/** A single predicted port: used to remember which ports we've made + * connections to, so that we can try to keep making circuits that can handle + * those ports. */ +typedef struct predicted_port_t { + /** The port we connected to */ + uint16_t port; + /** The time at which we last used it */ + time_t time; +} predicted_port_t; + +/** A list of port numbers that have been used recently. */ +static smartlist_t *predicted_ports_list=NULL; +/** How long do we keep predicting circuits? */ +static int prediction_timeout=0; +/** When was the last time we added a prediction entry (HS or port) */ +static time_t last_prediction_add_time=0; + +/** + * How much time left until we stop predicting circuits? + */ +int +predicted_ports_prediction_time_remaining(time_t now) +{ + time_t idle_delta; + + /* Protect against overflow of return value. This can happen if the clock + * jumps backwards in time. Update the last prediction time (aka last + * active time) to prevent it. This update is preferable to using monotonic + * time because it prevents clock jumps into the past from simply causing + * very long idle timeouts while the monotonic time stands still. */ + if (last_prediction_add_time > now) { + last_prediction_add_time = now; + idle_delta = 0; + } else { + idle_delta = now - last_prediction_add_time; + } + + /* Protect against underflow of the return value. This can happen for very + * large periods of inactivity/system sleep. */ + if (idle_delta > prediction_timeout) + return 0; + + if (BUG((prediction_timeout - idle_delta) > INT_MAX)) { + return INT_MAX; + } + + return (int)(prediction_timeout - idle_delta); +} + +/** We just got an application request for a connection with + * port <b>port</b>. Remember it for the future, so we can keep + * some circuits open that will exit to this port. + */ +static void +add_predicted_port(time_t now, uint16_t port) +{ + predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t)); + + // If the list is empty, re-randomize predicted ports lifetime + if (!any_predicted_circuits(now)) { + prediction_timeout = channelpadding_get_circuits_available_timeout(); + } + + last_prediction_add_time = now; + + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ building " + "for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + + pp->port = port; + pp->time = now; + rephist_total_alloc += sizeof(*pp); + smartlist_add(predicted_ports_list, pp); +} + +/** + * Allocate whatever memory and structs are needed for predicting + * which ports will be used. Also seed it with port 80, so we'll build + * circuits on start-up. + */ +static void +predicted_ports_alloc(void) +{ + predicted_ports_list = smartlist_new(); +} + +void +predicted_ports_init(void) +{ + add_predicted_port(time(NULL), 443); // Add a port to get us started +} + +/** Free whatever memory is needed for predicting which ports will + * be used. + */ +static void +predicted_ports_free_all(void) +{ + rephist_total_alloc -= + smartlist_len(predicted_ports_list)*sizeof(predicted_port_t); + SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *, + pp, tor_free(pp)); + smartlist_free(predicted_ports_list); +} + +/** Remember that <b>port</b> has been asked for as of time <b>now</b>. + * This is used for predicting what sorts of streams we'll make in the + * future and making exit circuits to anticipate that. + */ +void +rep_hist_note_used_port(time_t now, uint16_t port) +{ + tor_assert(predicted_ports_list); + + if (!port) /* record nothing */ + return; + + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (pp->port == port) { + pp->time = now; + + last_prediction_add_time = now; + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ " + "building for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + return; + } + } SMARTLIST_FOREACH_END(pp); + /* it's not there yet; we need to add it */ + add_predicted_port(now, port); +} + +/** Return a newly allocated pointer to a list of uint16_t * for ports that + * are likely to be asked for in the near future. + */ +smartlist_t * +rep_hist_get_predicted_ports(time_t now) +{ + int predicted_circs_relevance_time; + smartlist_t *out = smartlist_new(); + tor_assert(predicted_ports_list); + + predicted_circs_relevance_time = prediction_timeout; + + /* clean out obsolete entries */ + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (pp->time + predicted_circs_relevance_time < now) { + log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); + + rephist_total_alloc -= sizeof(predicted_port_t); + tor_free(pp); + SMARTLIST_DEL_CURRENT(predicted_ports_list, pp); + } else { + smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t))); + } + } SMARTLIST_FOREACH_END(pp); + return out; +} + +/** + * Take a list of uint16_t *, and remove every port in the list from the + * current list of predicted ports. + */ +void +rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports) +{ + /* Let's do this on O(N), not O(N^2). */ + bitarray_t *remove_ports = bitarray_init_zero(UINT16_MAX); + SMARTLIST_FOREACH(rmv_ports, const uint16_t *, p, + bitarray_set(remove_ports, *p)); + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (bitarray_is_set(remove_ports, pp->port)) { + tor_free(pp); + rephist_total_alloc -= sizeof(*pp); + SMARTLIST_DEL_CURRENT(predicted_ports_list, pp); + } + } SMARTLIST_FOREACH_END(pp); + bitarray_free(remove_ports); +} + +/** The user asked us to do a resolve. Rather than keeping track of + * timings and such of resolves, we fake it for now by treating + * it the same way as a connection to port 80. This way we will continue + * to have circuits lying around if the user only uses Tor for resolves. + */ +void +rep_hist_note_used_resolve(time_t now) +{ + rep_hist_note_used_port(now, 80); +} + +/** The last time at which we needed an internal circ. */ +static time_t predicted_internal_time = 0; +/** The last time we needed an internal circ with good uptime. */ +static time_t predicted_internal_uptime_time = 0; +/** The last time we needed an internal circ with good capacity. */ +static time_t predicted_internal_capacity_time = 0; + +/** Remember that we used an internal circ at time <b>now</b>. */ +void +rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity) +{ + // If the list is empty, re-randomize predicted ports lifetime + if (!any_predicted_circuits(now)) { + prediction_timeout = channelpadding_get_circuits_available_timeout(); + } + + last_prediction_add_time = now; + + log_info(LD_CIRC, + "New port prediction added. Will continue predictive circ building " + "for %d more seconds.", + predicted_ports_prediction_time_remaining(now)); + + predicted_internal_time = now; + if (need_uptime) + predicted_internal_uptime_time = now; + if (need_capacity) + predicted_internal_capacity_time = now; +} + +/** Return 1 if we've used an internal circ recently; else return 0. */ +int +rep_hist_get_predicted_internal(time_t now, int *need_uptime, + int *need_capacity) +{ + int predicted_circs_relevance_time; + + predicted_circs_relevance_time = prediction_timeout; + + if (!predicted_internal_time) { /* initialize it */ + predicted_internal_time = now; + predicted_internal_uptime_time = now; + predicted_internal_capacity_time = now; + } + if (predicted_internal_time + predicted_circs_relevance_time < now) + return 0; /* too long ago */ + if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now) + *need_uptime = 1; + // Always predict that we need capacity. + *need_capacity = 1; + return 1; +} + +/** Any ports used lately? These are pre-seeded if we just started + * up or if we're running a hidden service. */ +int +any_predicted_circuits(time_t now) +{ + int predicted_circs_relevance_time; + predicted_circs_relevance_time = prediction_timeout; + + return smartlist_len(predicted_ports_list) || + predicted_internal_time + predicted_circs_relevance_time >= now; +} + +/** Return 1 if we have no need for circuits currently, else return 0. */ +int +rep_hist_circbuilding_dormant(time_t now) +{ + const or_options_t *options = get_options(); + + if (any_predicted_circuits(now)) + return 0; + + /* see if we'll still need to build testing circuits */ + if (server_mode(options) && + (!check_whether_orport_reachable(options) || + !circuit_enough_testing_circs())) + return 0; + if (!check_whether_dirport_reachable(options)) + return 0; + + return 1; +} + +/*** Exit port statistics ***/ + +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** Start time of exit stats or 0 if we're not collecting exit stats. */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ +void +rep_hist_exit_stats_init(time_t now) +{ + start_of_exit_stats_interval = now; + exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t)); + exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t)); + exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t)); +} + +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) +{ + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); +} + +/** Stop collecting exit port stats in a way that we can re-start doing + * so in rep_hist_exit_stats_init(). */ +void +rep_hist_exit_stats_term(void) +{ + start_of_exit_stats_interval = 0; + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); +} + +/** Helper for qsort: compare two ints. Does not handle overflow properly, + * but works fine for sorting an array of port numbers, which is what we use + * it for. */ +static int +compare_int_(const void *x, const void *y) +{ + return (*(int*)x - *(int*)y); +} + +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. Caller + * must ensure start_of_exit_stats_interval is in the past. */ +char * +rep_hist_format_exit_stats(time_t now) +{ + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; + char t[ISO_TIME_LEN+1]; + char *result; + + if (!start_of_exit_stats_interval) + return NULL; /* Not initialized. */ + + tor_assert(now >= start_of_exit_stats_interval); + + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } + } + + /* Add observations of top ports to smartlists. */ + written_strings = smartlist_new(); + read_strings = smartlist_new(); + streams_strings = smartlist_new(); + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), compare_int_); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + smartlist_add_asprintf(written_strings, "%d=%"PRIu64, + cur_port, (num)); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + smartlist_add_asprintf(read_strings, "%d=%"PRIu64, + cur_port, (num)); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num); + other_streams -= exit_streams[cur_port]; + } + } + + /* Add observations of other ports in a single element. */ + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + smartlist_add_asprintf(written_strings, "other=%"PRIu64, + (other_written)); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + smartlist_add_asprintf(read_strings, "other=%"PRIu64, + (other_read)); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + smartlist_add_asprintf(streams_strings, "other=%u", other_streams); + + /* Join all observations in single strings. */ + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; +} + +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + log_info(LD_HIST, "Writing exit port statistics to disk."); + + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); + + /* Reset counters. */ + rep_hist_reset_exit_stats(now); + + /* Try to write to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "exit-stats", str, "exit port statistics"); + } + + done: + tor_free(str); + return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ +void +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) +{ + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); +} + +/** Note that we opened an exit stream to <b>port</b>. */ +void +rep_hist_note_exit_stream_opened(uint16_t port) +{ + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); +} + +/*** cell statistics ***/ + +/** Start of the current buffer stats interval or 0 if we're not + * collecting buffer statistics. */ +static time_t start_of_buffer_stats_interval; + +/** Initialize buffer stats. */ +void +rep_hist_buffer_stats_init(time_t now) +{ + start_of_buffer_stats_interval = now; +} + +/** Statistics from a single circuit. Collected when the circuit closes, or + * when we flush statistics to disk. */ +typedef struct circ_buffer_stats_t { + /** Average number of cells in the circuit's queue */ + double mean_num_cells_in_queue; + /** Average time a cell waits in the queue. */ + double mean_time_cells_in_queue; + /** Total number of cells sent over this circuit */ + uint32_t processed_cells; +} circ_buffer_stats_t; + +/** List of circ_buffer_stats_t. */ +static smartlist_t *circuits_for_buffer_stats = NULL; + +/** Remember cell statistics <b>mean_num_cells_in_queue</b>, + * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a + * circuit. */ +void +rep_hist_add_buffer_stats(double mean_num_cells_in_queue, + double mean_time_cells_in_queue, uint32_t processed_cells) +{ + circ_buffer_stats_t *stats; + if (!start_of_buffer_stats_interval) + return; /* Not initialized. */ + stats = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stats->mean_num_cells_in_queue = mean_num_cells_in_queue; + stats->mean_time_cells_in_queue = mean_time_cells_in_queue; + stats->processed_cells = processed_cells; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_new(); + smartlist_add(circuits_for_buffer_stats, stats); +} + +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ +void +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) +{ + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + double mean_num_cells_in_queue, mean_time_cells_in_queue; + uint32_t processed_cells; + if (CIRCUIT_IS_ORIGIN(circ)) + return; + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) + return; + start_of_interval = (circ->timestamp_created.tv_sec > + start_of_buffer_stats_interval) ? + (time_t)circ->timestamp_created.tv_sec : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + if (interval_length <= 0) + return; + processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; + rep_hist_add_buffer_stats(mean_num_cells_in_queue, + mean_time_cells_in_queue, + processed_cells); +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +buffer_stats_compare_entries_(const void **_a, const void **_b) +{ + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; +} + +/** Stop collecting cell stats in a way that we can re-start doing so in + * rep_hist_buffer_stats_init(). */ +void +rep_hist_buffer_stats_term(void) +{ + rep_hist_reset_buffer_stats(0); +} + +/** Clear history of circuit statistics and set the measurement interval + * start to <b>now</b>. */ +void +rep_hist_reset_buffer_stats(time_t now) +{ + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_new(); + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stats, tor_free(stats)); + smartlist_clear(circuits_for_buffer_stats); + start_of_buffer_stats_interval = now; +} + +/** Return a newly allocated string containing the buffer statistics until + * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must + * ensure start_of_buffer_stats_interval is in the past. */ +char * +rep_hist_format_buffer_stats(time_t now) +{ +#define SHARES 10 + uint64_t processed_cells[SHARES]; + uint32_t circs_in_share[SHARES]; + int number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *processed_cells_strings, *queued_cells_strings, + *time_in_queue_strings; + char *processed_cells_string, *queued_cells_string, + *time_in_queue_string; + char t[ISO_TIME_LEN+1]; + char *result; + + if (!start_of_buffer_stats_interval) + return NULL; /* Not initialized. */ + + tor_assert(now >= start_of_buffer_stats_interval); + + /* Calculate deciles if we saw at least one circuit. */ + memset(processed_cells, 0, SHARES * sizeof(uint64_t)); + memset(circs_in_share, 0, SHARES * sizeof(uint32_t)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_new(); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits > 0) { + smartlist_sort(circuits_for_buffer_stats, + buffer_stats_compare_entries_); + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stats) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stats->processed_cells; + queued_cells[share] += stats->mean_num_cells_in_queue; + time_in_queue[share] += stats->mean_time_cells_in_queue; + circs_in_share[share]++; + } + SMARTLIST_FOREACH_END(stats); + } + + /* Write deciles to strings. */ + processed_cells_strings = smartlist_new(); + queued_cells_strings = smartlist_new(); + time_in_queue_strings = smartlist_new(); + for (i = 0; i < SHARES; i++) { + smartlist_add_asprintf(processed_cells_strings, + "%"PRIu64, !circs_in_share[i] ? 0 : + (processed_cells[i] / + circs_in_share[i])); + } + for (i = 0; i < SHARES; i++) { + smartlist_add_asprintf(queued_cells_strings, "%.2f", + circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + } + for (i = 0; i < SHARES; i++) { + smartlist_add_asprintf(time_in_queue_strings, "%.0f", + circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + } + + /* Join all observations in single strings. */ + processed_cells_string = smartlist_join_strings(processed_cells_strings, + ",", 0, NULL); + queued_cells_string = smartlist_join_strings(queued_cells_strings, + ",", 0, NULL); + time_in_queue_string = smartlist_join_strings(time_in_queue_strings, + ",", 0, NULL); + SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp)); + smartlist_free(processed_cells_strings); + smartlist_free(queued_cells_strings); + smartlist_free(time_in_queue_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "cell-stats-end %s (%d s)\n" + "cell-processed-cells %s\n" + "cell-queued-cells %s\n" + "cell-time-in-queue %s\n" + "cell-circuits-per-decile %d\n", + t, (unsigned) (now - start_of_buffer_stats_interval), + processed_cells_string, + queued_cells_string, + time_in_queue_string, + CEIL_DIV(number_of_circuits, SHARES)); + tor_free(processed_cells_string); + tor_free(queued_cells_string); + tor_free(time_in_queue_string); + return result; +#undef SHARES +} + +/** If 24 hours have passed since the beginning of the current buffer + * stats period, write buffer stats to $DATADIR/stats/buffer-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write buffer stats or 0 if we never want to + * write. */ +time_t +rep_hist_buffer_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* Add open circuits to the history. */ + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + rep_hist_buffer_stats_add_circ(circ, now); + } + SMARTLIST_FOREACH_END(circ); + + /* Generate history string. */ + str = rep_hist_format_buffer_stats(now); + + /* Reset both buffer history and counters of open circuits. */ + rep_hist_reset_buffer_stats(now); + + /* Try to write to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics"); + } + + done: + tor_free(str); + return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; +} + +/*** Descriptor serving statistics ***/ + +/** Digestmap to track which descriptors were downloaded this stats + * collection interval. It maps descriptor digest to pointers to 1, + * effectively turning this into a list. */ +static digestmap_t *served_descs = NULL; + +/** Number of how many descriptors were downloaded in total during this + * interval. */ +static unsigned long total_descriptor_downloads; + +/** Start time of served descs stats or 0 if we're not collecting those. */ +static time_t start_of_served_descs_stats_interval; + +/** Initialize descriptor stats. */ +void +rep_hist_desc_stats_init(time_t now) +{ + if (served_descs) { + log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were " + "already initialized. This is probably harmless."); + return; // Already initialized + } + served_descs = digestmap_new(); + total_descriptor_downloads = 0; + start_of_served_descs_stats_interval = now; +} + +/** Reset served descs stats to empty, starting a new interval <b>now</b>. */ +static void +rep_hist_reset_desc_stats(time_t now) +{ + rep_hist_desc_stats_term(); + rep_hist_desc_stats_init(now); +} + +/** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is + * safe to be called again. */ +void +rep_hist_desc_stats_term(void) +{ + digestmap_free(served_descs, NULL); + served_descs = NULL; + start_of_served_descs_stats_interval = 0; + total_descriptor_downloads = 0; +} + +/** Helper for rep_hist_desc_stats_write(). Return a newly allocated string + * containing the served desc statistics until now, or NULL if we're not + * collecting served desc stats. Caller must ensure that now is not before + * start_of_served_descs_stats_interval. */ +static char * +rep_hist_format_desc_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + char *result; + + digestmap_iter_t *iter; + const char *key; + void *val; + unsigned size; + int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0; + int n = 0; + + if (!start_of_served_descs_stats_interval) + return NULL; + + size = digestmap_size(served_descs); + if (size > 0) { + vals = tor_calloc(size, sizeof(int)); + for (iter = digestmap_iter_init(served_descs); + !digestmap_iter_done(iter); + iter = digestmap_iter_next(served_descs, iter)) { + uintptr_t count; + digestmap_iter_get(iter, &key, &val); + count = (uintptr_t)val; + vals[n++] = (int)count; + (void)key; + } + max = find_nth_int(vals, size, size-1); + q3 = find_nth_int(vals, size, (3*size-1)/4); + md = find_nth_int(vals, size, (size-1)/2); + q1 = find_nth_int(vals, size, (size-1)/4); + min = find_nth_int(vals, size, 0); + tor_free(vals); + } + + format_iso_time(t, now); + + tor_asprintf(&result, + "served-descs-stats-end %s (%d s) total=%lu unique=%u " + "max=%d q3=%d md=%d q1=%d min=%d\n", + t, + (unsigned) (now - start_of_served_descs_stats_interval), + total_descriptor_downloads, + size, max, q3, md, q1, min); + + return result; +} + +/** If WRITE_STATS_INTERVAL seconds have passed since the beginning of + * the current served desc stats interval, write the stats to + * $DATADIR/stats/served-desc-stats (possibly appending to an existing file) + * and reset the state for the next interval. Return when we would next want + * to write served desc stats or 0 if we won't want to write. */ +time_t +rep_hist_desc_stats_write(time_t now) +{ + char *filename = NULL, *str = NULL; + + if (!start_of_served_descs_stats_interval) + return 0; /* We're not collecting stats. */ + if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now) + return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL; + + str = rep_hist_format_desc_stats(now); + tor_assert(str != NULL); + + if (check_or_create_data_subdir("stats") < 0) { + goto done; + } + filename = get_datadir_fname2("stats", "served-desc-stats"); + if (append_bytes_to_file(filename, str, strlen(str), 0) < 0) + log_warn(LD_HIST, "Unable to write served descs statistics to disk!"); + + rep_hist_reset_desc_stats(now); + + done: + tor_free(filename); + tor_free(str); + return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Called to note that we've served a given descriptor (by + * digest). Increments the count of descriptors served, and the number + * of times we've served this descriptor. */ +void +rep_hist_note_desc_served(const char * desc) +{ + void *val; + uintptr_t count; + if (!served_descs) + return; // We're not collecting stats + val = digestmap_get(served_descs, desc); + count = (uintptr_t)val; + if (count != INT_MAX) + ++count; + digestmap_set(served_descs, desc, (void*)count); + total_descriptor_downloads++; +} + +/*** Connection statistics ***/ + +/** Start of the current connection stats interval or 0 if we're not + * collecting connection statistics. */ +static time_t start_of_conn_stats_interval; + +/** Initialize connection stats. */ +void +rep_hist_conn_stats_init(time_t now) +{ + start_of_conn_stats_interval = now; +} + +/* Count connections that we read and wrote less than these many bytes + * from/to as below threshold. */ +#define BIDI_THRESHOLD 20480 + +/* Count connections that we read or wrote at least this factor as many + * bytes from/to than we wrote or read to/from as mostly reading or + * writing. */ +#define BIDI_FACTOR 10 + +/* Interval length in seconds for considering read and written bytes for + * connection stats. */ +#define BIDI_INTERVAL 10 + +/** Start of next BIDI_INTERVAL second interval. */ +static time_t bidi_next_interval = 0; + +/** Number of connections that we read and wrote less than BIDI_THRESHOLD + * bytes from/to in BIDI_INTERVAL seconds. */ +static uint32_t below_threshold = 0; + +/** Number of connections that we read at least BIDI_FACTOR times more + * bytes from than we wrote to in BIDI_INTERVAL seconds. */ +static uint32_t mostly_read = 0; + +/** Number of connections that we wrote at least BIDI_FACTOR times more + * bytes to than we read from in BIDI_INTERVAL seconds. */ +static uint32_t mostly_written = 0; + +/** Number of connections that we read and wrote at least BIDI_THRESHOLD + * bytes from/to, but not BIDI_FACTOR times more in either direction in + * BIDI_INTERVAL seconds. */ +static uint32_t both_read_and_written = 0; + +/** Entry in a map from connection ID to the number of read and written + * bytes on this connection in a BIDI_INTERVAL second interval. */ +typedef struct bidi_map_entry_t { + HT_ENTRY(bidi_map_entry_t) node; + uint64_t conn_id; /**< Connection ID */ + size_t read; /**< Number of read bytes */ + size_t written; /**< Number of written bytes */ +} bidi_map_entry_t; + +/** Map of OR connections together with the number of read and written + * bytes in the current BIDI_INTERVAL second interval. */ +static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map = + HT_INITIALIZER(); + +static int +bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b) +{ + return a->conn_id == b->conn_id; +} + +/* DOCDOC bidi_map_ent_hash */ +static unsigned +bidi_map_ent_hash(const bidi_map_entry_t *entry) +{ + return (unsigned) entry->conn_id; +} + +HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq) +HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) + +/* DOCDOC bidi_map_free */ +static void +bidi_map_free_all(void) +{ + bidi_map_entry_t **ptr, **next, *ent; + for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) { + ent = *ptr; + next = HT_NEXT_RMV(bidimap, &bidi_map, ptr); + tor_free(ent); + } + HT_CLEAR(bidimap, &bidi_map); +} + +/** Reset counters for conn statistics. */ +void +rep_hist_reset_conn_stats(time_t now) +{ + start_of_conn_stats_interval = now; + below_threshold = 0; + mostly_read = 0; + mostly_written = 0; + both_read_and_written = 0; + bidi_map_free_all(); +} + +/** Stop collecting connection stats in a way that we can re-start doing + * so in rep_hist_conn_stats_init(). */ +void +rep_hist_conn_stats_term(void) +{ + rep_hist_reset_conn_stats(0); +} + +/** We read <b>num_read</b> bytes and wrote <b>num_written</b> from/to OR + * connection <b>conn_id</b> in second <b>when</b>. If this is the first + * observation in a new interval, sum up the last observations. Add bytes + * for this connection. */ +void +rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read, + size_t num_written, time_t when) +{ + if (!start_of_conn_stats_interval) + return; + /* Initialize */ + if (bidi_next_interval == 0) + bidi_next_interval = when + BIDI_INTERVAL; + /* Sum up last period's statistics */ + if (when >= bidi_next_interval) { + bidi_map_entry_t **ptr, **next, *ent; + for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) { + ent = *ptr; + if (ent->read + ent->written < BIDI_THRESHOLD) + below_threshold++; + else if (ent->read >= ent->written * BIDI_FACTOR) + mostly_read++; + else if (ent->written >= ent->read * BIDI_FACTOR) + mostly_written++; + else + both_read_and_written++; + next = HT_NEXT_RMV(bidimap, &bidi_map, ptr); + tor_free(ent); + } + while (when >= bidi_next_interval) + bidi_next_interval += BIDI_INTERVAL; + log_info(LD_GENERAL, "%d below threshold, %d mostly read, " + "%d mostly written, %d both read and written.", + below_threshold, mostly_read, mostly_written, + both_read_and_written); + } + /* Add this connection's bytes. */ + if (num_read > 0 || num_written > 0) { + bidi_map_entry_t *entry, lookup; + lookup.conn_id = conn_id; + entry = HT_FIND(bidimap, &bidi_map, &lookup); + if (entry) { + entry->written += num_written; + entry->read += num_read; + } else { + entry = tor_malloc_zero(sizeof(bidi_map_entry_t)); + entry->conn_id = conn_id; + entry->written = num_written; + entry->read = num_read; + HT_INSERT(bidimap, &bidi_map, entry); + } + } +} + +/** Return a newly allocated string containing the connection statistics + * until <b>now</b>, or NULL if we're not collecting conn stats. Caller must + * ensure start_of_conn_stats_interval is in the past. */ +char * +rep_hist_format_conn_stats(time_t now) +{ + char *result, written[ISO_TIME_LEN+1]; + + if (!start_of_conn_stats_interval) + return NULL; /* Not initialized. */ + + tor_assert(now >= start_of_conn_stats_interval); + + format_iso_time(written, now); + tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n", + written, + (unsigned) (now - start_of_conn_stats_interval), + below_threshold, + mostly_read, + mostly_written, + both_read_and_written); + return result; +} + +/** If 24 hours have passed since the beginning of the current conn stats + * period, write conn stats to $DATADIR/stats/conn-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write conn stats or 0 if we never want to write. */ +time_t +rep_hist_conn_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_conn_stats_interval) + return 0; /* Not initialized. */ + if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* Generate history string. */ + str = rep_hist_format_conn_stats(now); + + /* Reset counters. */ + rep_hist_reset_conn_stats(now); + + /* Try to write to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "conn-stats", str, "connection statistics"); + } + + done: + tor_free(str); + return start_of_conn_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Internal statistics to track how many requests of each type of + * handshake we've received, and how many we've assigned to cpuworkers. + * Useful for seeing trends in cpu load. + * @{ */ +STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +/**@}*/ + +/** A new onionskin (using the <b>type</b> handshake) has arrived. */ +void +rep_hist_note_circuit_handshake_requested(uint16_t type) +{ + if (type <= MAX_ONION_HANDSHAKE_TYPE) + onion_handshakes_requested[type]++; +} + +/** We've sent an onionskin (using the <b>type</b> handshake) to a + * cpuworker. */ +void +rep_hist_note_circuit_handshake_assigned(uint16_t type) +{ + if (type <= MAX_ONION_HANDSHAKE_TYPE) + onion_handshakes_assigned[type]++; +} + +/** Log our onionskin statistics since the last time we were called. */ +void +rep_hist_log_circuit_handshake_stats(time_t now) +{ + (void)now; + log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: " + "%d/%d TAP, %d/%d NTor.", + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP], + onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR], + onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]); + memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned)); + memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested)); +} + +/* Hidden service statistics section */ + +/** Start of the current hidden service stats interval or 0 if we're + * not collecting hidden service statistics. */ +static time_t start_of_hs_stats_interval; + +/** Carries the various hidden service statistics, and any other + * information needed. */ +typedef struct hs_stats_t { + /** How many relay cells have we seen as rendezvous points? */ + uint64_t rp_relay_cells_seen; + + /** Set of unique public key digests we've seen this stat period + * (could also be implemented as sorted smartlist). */ + digestmap_t *onions_seen_this_period; +} hs_stats_t; + +/** Our statistics structure singleton. */ +static hs_stats_t *hs_stats = NULL; + +/** Allocate, initialize and return an hs_stats_t structure. */ +static hs_stats_t * +hs_stats_new(void) +{ + hs_stats_t *new_hs_stats = tor_malloc_zero(sizeof(hs_stats_t)); + new_hs_stats->onions_seen_this_period = digestmap_new(); + + return new_hs_stats; +} + +#define hs_stats_free(val) \ + FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val)) + +/** Free an hs_stats_t structure. */ +static void +hs_stats_free_(hs_stats_t *victim_hs_stats) +{ + if (!victim_hs_stats) { + return; + } + + digestmap_free(victim_hs_stats->onions_seen_this_period, NULL); + tor_free(victim_hs_stats); +} + +/** Initialize hidden service statistics. */ +void +rep_hist_hs_stats_init(time_t now) +{ + if (!hs_stats) { + hs_stats = hs_stats_new(); + } + + start_of_hs_stats_interval = now; +} + +/** Clear history of hidden service statistics and set the measurement + * interval start to <b>now</b>. */ +static void +rep_hist_reset_hs_stats(time_t now) +{ + if (!hs_stats) { + hs_stats = hs_stats_new(); + } + + hs_stats->rp_relay_cells_seen = 0; + + digestmap_free(hs_stats->onions_seen_this_period, NULL); + hs_stats->onions_seen_this_period = digestmap_new(); + + start_of_hs_stats_interval = now; +} + +/** Stop collecting hidden service stats in a way that we can re-start + * doing so in rep_hist_buffer_stats_init(). */ +void +rep_hist_hs_stats_term(void) +{ + rep_hist_reset_hs_stats(0); +} + +/** We saw a new HS relay cell, Count it! */ +void +rep_hist_seen_new_rp_cell(void) +{ + if (!hs_stats) { + return; // We're not collecting stats + } + + hs_stats->rp_relay_cells_seen++; +} + +/** As HSDirs, we saw another hidden service with public key + * <b>pubkey</b>. Check whether we have counted it before, if not + * count it now! */ +void +rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey) +{ + char pubkey_hash[DIGEST_LEN]; + + if (!hs_stats) { + return; // We're not collecting stats + } + + /* Get the digest of the pubkey which will be used to detect whether + we've seen this hidden service before or not. */ + if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) { + /* This fail should not happen; key has been validated by + descriptor parsing code first. */ + return; + } + + /* Check if this is the first time we've seen this hidden + service. If it is, count it as new. */ + if (!digestmap_get(hs_stats->onions_seen_this_period, + pubkey_hash)) { + digestmap_set(hs_stats->onions_seen_this_period, + pubkey_hash, (void*)(uintptr_t)1); + } +} + +/* The number of cells that are supposed to be hidden from the adversary + * by adding noise from the Laplace distribution. This value, divided by + * EPSILON, is Laplace parameter b. It must be greather than 0. */ +#define REND_CELLS_DELTA_F 2048 +/* Security parameter for obfuscating number of cells with a value between + * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same + * time make statistics less usable. */ +#define REND_CELLS_EPSILON 0.3 +/* The number of cells that are supposed to be hidden from the adversary + * by rounding up to the next multiple of this number. */ +#define REND_CELLS_BIN_SIZE 1024 +/* The number of service identities that are supposed to be hidden from the + * adversary by adding noise from the Laplace distribution. This value, + * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */ +#define ONIONS_SEEN_DELTA_F 8 +/* Security parameter for obfuscating number of service identities with a + * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but + * at the same time make statistics less usable. */ +#define ONIONS_SEEN_EPSILON 0.3 +/* The number of service identities that are supposed to be hidden from + * the adversary by rounding up to the next multiple of this number. */ +#define ONIONS_SEEN_BIN_SIZE 8 + +/** Allocate and return a string containing hidden service stats that + * are meant to be placed in the extra-info descriptor. */ +static char * +rep_hist_format_hs_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + char *hs_stats_string; + int64_t obfuscated_cells_seen; + int64_t obfuscated_onions_seen; + + uint64_t rounded_cells_seen + = round_uint64_to_next_multiple_of(hs_stats->rp_relay_cells_seen, + REND_CELLS_BIN_SIZE); + rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX); + obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen, + crypto_rand_double(), + REND_CELLS_DELTA_F, REND_CELLS_EPSILON); + + uint64_t rounded_onions_seen = + round_uint64_to_next_multiple_of((size_t)digestmap_size( + hs_stats->onions_seen_this_period), + ONIONS_SEEN_BIN_SIZE); + rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX); + obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen, + crypto_rand_double(), ONIONS_SEEN_DELTA_F, + ONIONS_SEEN_EPSILON); + + format_iso_time(t, now); + tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n" + "hidserv-rend-relayed-cells %"PRId64" delta_f=%d " + "epsilon=%.2f bin_size=%d\n" + "hidserv-dir-onions-seen %"PRId64" delta_f=%d " + "epsilon=%.2f bin_size=%d\n", + t, (unsigned) (now - start_of_hs_stats_interval), + (obfuscated_cells_seen), REND_CELLS_DELTA_F, + REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE, + (obfuscated_onions_seen), + ONIONS_SEEN_DELTA_F, + ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE); + + return hs_stats_string; +} + +/** If 24 hours have passed since the beginning of the current HS + * stats period, write buffer stats to $DATADIR/stats/hidserv-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write buffer stats or 0 if we never want to + * write. */ +time_t +rep_hist_hs_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_hs_stats_interval) { + return 0; /* Not initialized. */ + } + + if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) { + goto done; /* Not ready to write */ + } + + /* Generate history string. */ + str = rep_hist_format_hs_stats(now); + + /* Reset HS history. */ + rep_hist_reset_hs_stats(now); + + /* Try to write to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "hidserv-stats", str, + "hidden service stats"); + } + + done: + tor_free(str); + return start_of_hs_stats_interval + WRITE_STATS_INTERVAL; +} + +static uint64_t link_proto_count[MAX_LINK_PROTO+1][2]; + +/** Note that we negotiated link protocol version <b>link_proto</b>, on + * a connection that started here iff <b>started_here</b> is true. + */ +void +rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) +{ + started_here = !!started_here; /* force to 0 or 1 */ + if (link_proto > MAX_LINK_PROTO) { + log_warn(LD_BUG, "Can't log link protocol %u", link_proto); + return; + } + + link_proto_count[link_proto][started_here]++; +} + +/** + * Update the maximum count of total pending channel padding timers + * in this period. + */ +void +rep_hist_padding_count_timers(uint64_t num_timers) +{ + if (num_timers > padding_current.maximum_chanpad_timers) { + padding_current.maximum_chanpad_timers = num_timers; + } +} + +/** + * Count a cell that we sent for padding overhead statistics. + * + * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be + * counted for PADDING_TYPE_TOTAL. + */ +void +rep_hist_padding_count_write(padding_type_t type) +{ + switch (type) { + case PADDING_TYPE_DROP: + padding_current.write_drop_cell_count++; + break; + case PADDING_TYPE_CELL: + padding_current.write_pad_cell_count++; + break; + case PADDING_TYPE_TOTAL: + padding_current.write_cell_count++; + break; + case PADDING_TYPE_ENABLED_TOTAL: + padding_current.enabled_write_cell_count++; + break; + case PADDING_TYPE_ENABLED_CELL: + padding_current.enabled_write_pad_cell_count++; + break; + } +} + +/** + * Count a cell that we've received for padding overhead statistics. + * + * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be + * counted for PADDING_TYPE_TOTAL. + */ +void +rep_hist_padding_count_read(padding_type_t type) +{ + switch (type) { + case PADDING_TYPE_DROP: + padding_current.read_drop_cell_count++; + break; + case PADDING_TYPE_CELL: + padding_current.read_pad_cell_count++; + break; + case PADDING_TYPE_TOTAL: + padding_current.read_cell_count++; + break; + case PADDING_TYPE_ENABLED_TOTAL: + padding_current.enabled_read_cell_count++; + break; + case PADDING_TYPE_ENABLED_CELL: + padding_current.enabled_read_pad_cell_count++; + break; + } +} + +/** + * Reset our current padding statistics. Called once every 24 hours. + */ +void +rep_hist_reset_padding_counts(void) +{ + memset(&padding_current, 0, sizeof(padding_current)); +} + +/** + * Copy our current cell counts into a structure for listing in our + * extra-info descriptor. Also perform appropriate rounding and redaction. + * + * This function is called once every 24 hours. + */ +#define MIN_CELL_COUNTS_TO_PUBLISH 1 +#define ROUND_CELL_COUNTS_TO 10000 +void +rep_hist_prep_published_padding_counts(time_t now) +{ + memcpy(&padding_published, &padding_current, sizeof(padding_published)); + + if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH || + padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) { + memset(&padding_published, 0, sizeof(padding_published)); + return; + } + + format_iso_time(padding_published.first_published_at, now); +#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \ + ROUND_CELL_COUNTS_TO) + ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count); + ROUND_AND_SET_COUNT(padding_published.write_cell_count); + ROUND_AND_SET_COUNT(padding_published.read_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count); + ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count); +#undef ROUND_AND_SET_COUNT +} + +/** + * Returns an allocated string for extra-info documents for publishing + * padding statistics from the last 24 hour interval. + */ +char * +rep_hist_get_padding_count_lines(void) +{ + char *result = NULL; + + if (!padding_published.read_cell_count || + !padding_published.write_cell_count) { + return NULL; + } + + tor_asprintf(&result, "padding-counts %s (%d s)" + " bin-size=%"PRIu64 + " write-drop=%"PRIu64 + " write-pad=%"PRIu64 + " write-total=%"PRIu64 + " read-drop=%"PRIu64 + " read-pad=%"PRIu64 + " read-total=%"PRIu64 + " enabled-read-pad=%"PRIu64 + " enabled-read-total=%"PRIu64 + " enabled-write-pad=%"PRIu64 + " enabled-write-total=%"PRIu64 + " max-chanpad-timers=%"PRIu64 + "\n", + padding_published.first_published_at, + REPHIST_CELL_PADDING_COUNTS_INTERVAL, + (uint64_t)ROUND_CELL_COUNTS_TO, + (padding_published.write_drop_cell_count), + (padding_published.write_pad_cell_count), + (padding_published.write_cell_count), + (padding_published.read_drop_cell_count), + (padding_published.read_pad_cell_count), + (padding_published.read_cell_count), + (padding_published.enabled_read_pad_cell_count), + (padding_published.enabled_read_cell_count), + (padding_published.enabled_write_pad_cell_count), + (padding_published.enabled_write_cell_count), + (padding_published.maximum_chanpad_timers) + ); + + return result; +} + +/** Log a heartbeat message explaining how many connections of each link + * protocol version we have used. + */ +void +rep_hist_log_link_protocol_counts(void) +{ + log_notice(LD_HEARTBEAT, + "Since startup, we have initiated " + "%"PRIu64" v1 connections, " + "%"PRIu64" v2 connections, " + "%"PRIu64" v3 connections, and " + "%"PRIu64" v4 connections; and received " + "%"PRIu64" v1 connections, " + "%"PRIu64" v2 connections, " + "%"PRIu64" v3 connections, and " + "%"PRIu64" v4 connections.", + (link_proto_count[1][1]), + (link_proto_count[2][1]), + (link_proto_count[3][1]), + (link_proto_count[4][1]), + (link_proto_count[1][0]), + (link_proto_count[2][0]), + (link_proto_count[3][0]), + (link_proto_count[4][0])); +} + +/** Free all storage held by the OR/link history caches, by the + * bandwidth history arrays, by the port history, or by statistics . */ +void +rep_hist_free_all(void) +{ + hs_stats_free(hs_stats); + digestmap_free(history_map, free_or_history); + + bw_array_free(read_array); + read_array = NULL; + + bw_array_free(write_array); + write_array = NULL; + + bw_array_free(dir_read_array); + dir_read_array = NULL; + + bw_array_free(dir_write_array); + dir_write_array = NULL; + + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); + predicted_ports_free_all(); + bidi_map_free_all(); + + if (circuits_for_buffer_stats) { + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s, + tor_free(s)); + smartlist_free(circuits_for_buffer_stats); + circuits_for_buffer_stats = NULL; + } + rep_hist_desc_stats_term(); + total_descriptor_downloads = 0; + + tor_assert_nonfatal(rephist_total_alloc == 0); + tor_assert_nonfatal_once(rephist_total_num == 0); +} diff --cc src/feature/stats/rephist.h index 06a5e4821,000000000..67a015a4c mode 100644,000000..100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@@ -1,140 -1,0 +1,140 @@@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rephist.h + * \brief Header file for rephist.c. + **/ + +#ifndef TOR_REPHIST_H +#define TOR_REPHIST_H + +void rep_hist_init(void); +void rep_hist_dump_stats(time_t now, int severity); +void rep_hist_note_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_bytes_written(size_t num_bytes, time_t when); + +void rep_hist_make_router_pessimal(const char *id, time_t when); + +void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when); + - int rep_hist_bandwidth_assess(void); ++MOCK_DECL(int, rep_hist_bandwidth_assess, (void)); +char *rep_hist_get_bandwidth_lines(void); +void rep_hist_update_state(or_state_t *state); +int rep_hist_load_state(or_state_t *state, char **err); +void rep_history_clean(time_t before); + +void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when); +void rep_hist_note_router_unreachable(const char *id, time_t when); +int rep_hist_record_mtbf_data(time_t now, int missing_means_down); +int rep_hist_load_mtbf_data(time_t now); + +time_t rep_hist_downrate_old_runs(time_t now); +long rep_hist_get_uptime(const char *id, time_t when); +double rep_hist_get_stability(const char *id, time_t when); +double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); +long rep_hist_get_weighted_time_known(const char *id, time_t when); +int rep_hist_have_measured_enough_stability(void); + +void predicted_ports_init(void); +void rep_hist_note_used_port(time_t now, uint16_t port); +smartlist_t *rep_hist_get_predicted_ports(time_t now); +void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports); +void rep_hist_note_used_resolve(time_t now); +void rep_hist_note_used_internal(time_t now, int need_uptime, + int need_capacity); +int rep_hist_get_predicted_internal(time_t now, int *need_uptime, + int *need_capacity); + +int any_predicted_circuits(time_t now); +int rep_hist_circbuilding_dormant(time_t now); +int predicted_ports_prediction_time_remaining(time_t now); + +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); + +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +time_t rep_hist_buffer_stats_write(time_t now); +void rep_hist_buffer_stats_term(void); +void rep_hist_add_buffer_stats(double mean_num_cells_in_queue, + double mean_time_cells_in_queue, uint32_t processed_cells); +char *rep_hist_format_buffer_stats(time_t now); +void rep_hist_reset_buffer_stats(time_t now); + +void rep_hist_desc_stats_init(time_t now); +void rep_hist_note_desc_served(const char * desc); +void rep_hist_desc_stats_term(void); +time_t rep_hist_desc_stats_write(time_t now); + +void rep_hist_conn_stats_init(time_t now); +void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read, + size_t num_written, time_t when); +void rep_hist_reset_conn_stats(time_t now); +char *rep_hist_format_conn_stats(time_t now); +time_t rep_hist_conn_stats_write(time_t now); +void rep_hist_conn_stats_term(void); + +void rep_hist_note_circuit_handshake_requested(uint16_t type); +void rep_hist_note_circuit_handshake_assigned(uint16_t type); +void rep_hist_log_circuit_handshake_stats(time_t now); + +void rep_hist_hs_stats_init(time_t now); +void rep_hist_hs_stats_term(void); +time_t rep_hist_hs_stats_write(time_t now); +char *rep_hist_get_hs_stats_string(void); +void rep_hist_seen_new_rp_cell(void); +void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey); + +void rep_hist_free_all(void); + +void rep_hist_note_negotiated_link_proto(unsigned link_proto, + int started_here); +void rep_hist_log_link_protocol_counts(void); + +extern uint64_t rephist_total_alloc; +extern uint32_t rephist_total_num; +#ifdef TOR_UNIT_TESTS +extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; +extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; +#endif + +/** + * Represents the type of a cell for padding accounting + */ +typedef enum padding_type_t { + /** A RELAY_DROP cell */ + PADDING_TYPE_DROP, + /** A CELL_PADDING cell */ + PADDING_TYPE_CELL, + /** Total counts of padding and non-padding together */ + PADDING_TYPE_TOTAL, + /** Total cell counts for all padding-enabled channels */ + PADDING_TYPE_ENABLED_TOTAL, + /** CELL_PADDING counts for all padding-enabled channels */ + PADDING_TYPE_ENABLED_CELL +} padding_type_t; + +/** The amount of time over which the padding cell counts were counted */ +#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60) +void rep_hist_padding_count_read(padding_type_t type); +void rep_hist_padding_count_write(padding_type_t type); +char *rep_hist_get_padding_count_lines(void); +void rep_hist_reset_padding_counts(void); +void rep_hist_prep_published_padding_counts(time_t now); +void rep_hist_padding_count_timers(uint64_t num_timers); + +#endif /* !defined(TOR_REPHIST_H) */ + diff --cc src/test/test_router.c index 613ec0402,51055a336..6e64131fc --- a/src/test/test_router.c +++ b/src/test/test_router.c @@@ -4,114 -3,140 +4,236 @@@

/** * \file test_router.c - * \brief Unittests for code in src/or/router.c + * \brief Unittests for code in router.c **/

-#include "or.h" -#include "hibernate.h" -#include "log_test_helpers.h" -#include "main.h" -#include "rephist.h" -#include "router.h" -#include "test.h" +#include "core/or/or.h" +#include "app/config/config.h" ++#include "core/mainloop/main.h" ++#include "feature/hibernate/hibernate.h" ++#include "feature/nodelist/routerinfo_st.h" ++#include "feature/nodelist/routerlist.h" ++#include "feature/relay/router.h" ++#include "feature/stats/rephist.h" +#include "lib/crypt_ops/crypto_curve25519.h" +#include "lib/crypt_ops/crypto_ed25519.h" - #include "feature/relay/router.h" - #include "feature/nodelist/routerlist.h" - - #include "feature/nodelist/routerinfo_st.h" + +/* Test suite stuff */ +#include "test/test.h" ++#include "test/log_test_helpers.h" + +NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); + +static routerinfo_t* mock_routerinfo; + +static const routerinfo_t* +NS(router_get_my_routerinfo)(void) +{ + crypto_pk_t* ident_key; + crypto_pk_t* tap_key; + time_t now; + + if (!mock_routerinfo) { + /* Mock the published timestamp, otherwise router_dump_router_to_string() + * will poop its pants. */ + time(&now); + + /* We'll need keys, or router_dump_router_to_string() would return NULL. */ + ident_key = pk_generate(0); + tap_key = pk_generate(0); + + tor_assert(ident_key != NULL); + tor_assert(tap_key != NULL); + + mock_routerinfo = tor_malloc_zero(sizeof(routerinfo_t)); + mock_routerinfo->nickname = tor_strdup("ConlonNancarrow"); + mock_routerinfo->addr = 123456789; + mock_routerinfo->or_port = 443; + mock_routerinfo->platform = tor_strdup("unittest"); + mock_routerinfo->cache_info.published_on = now; + mock_routerinfo->identity_pkey = crypto_pk_dup_key(ident_key); + router_set_rsa_onion_pkey(tap_key, &mock_routerinfo->onion_pkey, + &mock_routerinfo->onion_pkey_len); + mock_routerinfo->bandwidthrate = 9001; + mock_routerinfo->bandwidthburst = 9002; + } + + return mock_routerinfo; +} + +/* If no distribution option was set, then check_bridge_distribution_setting() + * should have set it to "any". */ +static void +test_router_dump_router_to_string_no_bridge_distribution_method(void *arg) +{ + const char* needle = "bridge-distribution-request any"; + or_options_t* options = get_options_mutable(); + routerinfo_t* router = NULL; + curve25519_keypair_t ntor_keypair; + ed25519_keypair_t signing_keypair; + char* desc = NULL; + char* found = NULL; + (void)arg; + + NS_MOCK(router_get_my_routerinfo); + + options->ORPort_set = 1; + options->BridgeRelay = 1; + + /* Generate keys which router_dump_router_to_string() expects to exist. */ + tt_int_op(0, ==, curve25519_keypair_generate(&ntor_keypair, 0)); + tt_int_op(0, ==, ed25519_keypair_generate(&signing_keypair, 0)); + + /* Set up part of our routerinfo_t so that we don't trigger any other + * assertions in router_dump_router_to_string(). */ + router = (routerinfo_t*)router_get_my_routerinfo(); + tt_ptr_op(router, !=, NULL); + + router->onion_curve25519_pkey = &ntor_keypair.pubkey; + + /* Generate our server descriptor and ensure that the substring + * "bridge-distribution-request any" occurs somewhere within it. */ + crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(router->onion_pkey, + router->onion_pkey_len); + desc = router_dump_router_to_string(router, + router->identity_pkey, + onion_pkey, + &ntor_keypair, + &signing_keypair); + crypto_pk_free(onion_pkey); + tt_ptr_op(desc, !=, NULL); + found = strstr(desc, needle); + tt_ptr_op(found, !=, NULL); + + done: + NS_UNMOCK(router_get_my_routerinfo); + + tor_free(desc); +}

+ static routerinfo_t *mock_router_get_my_routerinfo_result = NULL; + + static const routerinfo_t * + mock_router_get_my_routerinfo(void) + { + return mock_router_get_my_routerinfo_result; + } + + static long + mock_get_uptime_3h(void) + { + return 3*60*60; + } + + static long + mock_get_uptime_1d(void) + { + return 24*60*60; + } + + static int + mock_rep_hist_bandwidth_assess(void) + { + return 20001; + } + + static int + mock_we_are_not_hibernating(void) + { + return 0; + } + + static int + mock_we_are_hibernating(void) + { + return 0; + } + + static void + test_router_check_descriptor_bandwidth_changed(void *arg) + { + (void)arg; + routerinfo_t routerinfo; + memset(&routerinfo, 0, sizeof(routerinfo)); + mock_router_get_my_routerinfo_result = NULL; + + MOCK(we_are_hibernating, mock_we_are_not_hibernating); + MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo); + mock_router_get_my_routerinfo_result = &routerinfo; + + /* When uptime is less than 24h, no previous bandwidth, no last_changed + * Uptime: 10800, last_changed: 0, Previous bw: 0, Current bw: 0 */ + routerinfo.bandwidthcapacity = 0; + MOCK(get_uptime, mock_get_uptime_3h); + setup_full_capture_of_logs(LOG_INFO); + check_descriptor_bandwidth_changed(time(NULL)); + expect_log_msg_not_containing( + "Measured bandwidth has changed; rebuilding descriptor."); + teardown_capture_of_logs(); + + /* When uptime is less than 24h, previous bandwidth, + * last_changed more than 3h ago + * Uptime: 10800, last_changed: 0, Previous bw: 10000, Current bw: 0 */ + routerinfo.bandwidthcapacity = 10000; + setup_full_capture_of_logs(LOG_INFO); + check_descriptor_bandwidth_changed(time(NULL)); + expect_log_msg_containing( + "Measured bandwidth has changed; rebuilding descriptor."); + teardown_capture_of_logs(); + + /* When uptime is less than 24h, previous bandwidth, + * last_changed more than 3h ago, and hibernating + * Uptime: 10800, last_changed: 0, Previous bw: 10000, Current bw: 0 */ + + UNMOCK(we_are_hibernating); + MOCK(we_are_hibernating, mock_we_are_hibernating); + routerinfo.bandwidthcapacity = 10000; + setup_full_capture_of_logs(LOG_INFO); + check_descriptor_bandwidth_changed(time(NULL)); + expect_log_msg_not_containing( + "Measured bandwidth has changed; rebuilding descriptor."); + teardown_capture_of_logs(); + UNMOCK(we_are_hibernating); + MOCK(we_are_hibernating, mock_we_are_not_hibernating); + + /* When uptime is less than 24h, last_changed is not more than 3h ago + * Uptime: 10800, last_changed: x, Previous bw: 10000, Current bw: 0 */ + setup_full_capture_of_logs(LOG_INFO); + check_descriptor_bandwidth_changed(time(NULL)); + expect_log_msg_not_containing( + "Measured bandwidth has changed; rebuilding descriptor."); + teardown_capture_of_logs(); + + /* When uptime is less than 24h and bandwidthcapacity does not change + * Uptime: 10800, last_changed: x, Previous bw: 10000, Current bw: 20001 */ + MOCK(rep_hist_bandwidth_assess, mock_rep_hist_bandwidth_assess); + setup_full_capture_of_logs(LOG_INFO); + check_descriptor_bandwidth_changed(time(NULL) + 6*60*60 + 1); + expect_log_msg_containing( + "Measured bandwidth has changed; rebuilding descriptor."); + UNMOCK(get_uptime); + UNMOCK(rep_hist_bandwidth_assess); + teardown_capture_of_logs(); + + /* When uptime is more than 24h */ + MOCK(get_uptime, mock_get_uptime_1d); + setup_full_capture_of_logs(LOG_INFO); + check_descriptor_bandwidth_changed(time(NULL)); + expect_log_msg_not_containing( + "Measured bandwidth has changed; rebuilding descriptor."); + teardown_capture_of_logs(); + + done: + UNMOCK(get_uptime); + UNMOCK(router_get_my_routerinfo); + UNMOCK(we_are_hibernating); + } + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL }

struct testcase_t router_tests[] = { + ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), + ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), END_OF_TESTCASES }; -