commit 7aef3ec0fde0b320343ecb3aa7080b6e1d9a2e62 Author: George Kadianakis desnacked@riseup.net Date: Thu Jun 1 13:37:11 2017 +0300

prop224: Add client-side HS descriptor cache.

Signed-off-by: David Goulet dgoulet@torproject.org --- src/or/hs_cache.c | 259 +++++++++++++++++++++++++++++++++++++++++++++++-- src/or/hs_cache.h | 25 +++++ src/or/hs_client.c | 39 ++++++++ src/or/hs_client.h | 7 ++ src/or/hs_descriptor.c | 31 ++++++ src/or/hs_descriptor.h | 1 + src/or/main.c | 2 +- 7 files changed, 357 insertions(+), 7 deletions(-)

diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c index 30215d868..6f7bf1abd 100644 --- a/src/or/hs_cache.c +++ b/src/or/hs_cache.c @@ -9,15 +9,19 @@ /* For unit tests.*/ #define HS_CACHE_PRIVATE

-#include "hs_cache.h" - #include "or.h" #include "config.h" +#include "hs_ident.h" #include "hs_common.h" +#include "hs_client.h" #include "hs_descriptor.h" #include "networkstatus.h" #include "rendcache.h"

+#include "hs_cache.h" + +/********************** Directory HS cache ******************/ + /* Directory descriptor cache. Map indexed by blinded key. */ static digest256map_t *hs_cache_v3_dir;

@@ -98,7 +102,7 @@ cache_dir_desc_new(const char *desc)

/* Return the size of a cache entry in bytes. */ static size_t -cache_get_entry_size(const hs_cache_dir_descriptor_t *entry) +cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry) { return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data) + strlen(entry->encoded_desc)); @@ -134,7 +138,7 @@ cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc) * remove the entry we currently have from our cache so we can then * store the new one. */ remove_v3_desc_as_dir(cache_entry); - rend_cache_decrement_allocation(cache_get_entry_size(cache_entry)); + rend_cache_decrement_allocation(cache_get_dir_entry_size(cache_entry)); cache_dir_desc_free(cache_entry); } /* Store the descriptor we just got. We are sure here that either we @@ -144,7 +148,7 @@ cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)

/* Update our total cache size with this entry for the OOM. This uses the * old HS protocol cache subsystem for which we are tied with. */ - rend_cache_increment_allocation(cache_get_entry_size(desc)); + rend_cache_increment_allocation(cache_get_dir_entry_size(desc));

/* XXX: Update HS statistics. We should have specific stats for v3. */

@@ -221,7 +225,7 @@ cache_clean_v3_as_dir(time_t now, time_t global_cutoff) } /* Here, our entry has expired, remove and free. */ MAP_DEL_CURRENT(key); - entry_size = cache_get_entry_size(entry); + entry_size = cache_get_dir_entry_size(entry); bytes_removed += entry_size; /* Entry is not in the cache anymore, destroy it. */ cache_dir_desc_free(entry); @@ -315,6 +319,243 @@ hs_cache_clean_as_dir(time_t now) cache_clean_v3_as_dir(now, 0); }

+/********************** Client-side HS cache ******************/ + +/* Client-side HS descriptor cache. Map indexed by service identity key. */ +static digest256map_t *hs_cache_v3_client; + +/* Remove a given descriptor from our cache. */ +static void +remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc) +{ + tor_assert(desc); + digest256map_remove(hs_cache_v3_client, desc->key.pubkey); +} + +/* Store a given descriptor in our cache. */ +static void +store_v3_desc_as_client(hs_cache_client_descriptor_t *desc) +{ + tor_assert(desc); + digest256map_set(hs_cache_v3_client, desc->key.pubkey, desc); +} + +/* Query our cache and return the entry or NULL if not found. */ +STATIC hs_cache_client_descriptor_t * +lookup_v3_desc_as_client(const uint8_t *key) +{ + tor_assert(key); + return digest256map_get(hs_cache_v3_client, key); +} + +/* Return the size of a client cache entry in bytes. */ +static size_t +cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry) +{ + return sizeof(*entry) + + strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc); +} + +/* Parse the encoded descriptor in <b>desc_str</b> using + * <b>service_identity_pk<b> to decrypt it first. + * + * If everything goes well, allocate and return a new + * hs_cache_client_descriptor_t object. In case of error, return NULL. */ +static hs_cache_client_descriptor_t * +cache_client_desc_new(const char *desc_str, + const ed25519_public_key_t *service_identity_pk) +{ + hs_descriptor_t *desc = NULL; + hs_cache_client_descriptor_t *client_desc = NULL; + + tor_assert(desc_str); + tor_assert(service_identity_pk); + + /* Decode the descriptor we just fetched. */ + if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) { + goto end; + } + tor_assert(desc); + + /* All is good: make a cache object for this descriptor */ + client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t)); + ed25519_pubkey_copy(&client_desc->key, service_identity_pk); + client_desc->created_ts = approx_time(); + client_desc->desc = desc; + client_desc->encoded_desc = tor_strdup(desc_str); + + end: + return client_desc; +} + +/** Free memory allocated by <b>desc</b>. */ +static void +cache_client_desc_free(hs_cache_client_descriptor_t *desc) +{ + if (desc == NULL) { + return; + } + hs_descriptor_free(desc->desc); + memwipe(&desc->key, 0, sizeof(desc->key)); + memwipe(desc->encoded_desc, 0, strlen(desc->encoded_desc)); + tor_free(desc->encoded_desc); + tor_free(desc); +} + +/** Helper function: Use by the free all function to clear the client cache */ +static void +cache_client_desc_free_(void *ptr) +{ + hs_cache_client_descriptor_t *desc = ptr; + cache_client_desc_free(desc); +} + +/** Check whether <b>client_desc</b> is useful for us, and store it in the + * client-side HS cache if so. The client_desc is freed if we already have a + * fresher (higher revision counter count) in the cache. */ +static int +cache_store_as_client(hs_cache_client_descriptor_t *client_desc) +{ + hs_cache_client_descriptor_t *cache_entry; + + /* TODO: Heavy code duplication with cache_store_as_dir(). Consider + * refactoring and uniting! */ + + tor_assert(client_desc); + + /* Check if we already have a descriptor from this HS in cache. If we do, + * check if this descriptor is newer than the cached one */ + cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey); + if (cache_entry != NULL) { + /* If we have an entry in our cache that has a revision counter greater + * than the one we just fetched, discard the one we fetched. */ + if (cache_entry->desc->plaintext_data.revision_counter > + client_desc->desc->plaintext_data.revision_counter) { + log_info(LD_REND, "We already have fresher descriptor. Ignoring."); + cache_client_desc_free(client_desc); + goto done; + } + /* Remove old entry. Make space for the new one! */ + remove_v3_desc_as_client(cache_entry); + rend_cache_decrement_allocation(cache_get_client_entry_size(cache_entry)); + cache_client_desc_free(cache_entry); + } + + /* Store descriptor in cache */ + store_v3_desc_as_client(client_desc); + + /* Update cache size with this entry for the OOM handler. */ + rend_cache_increment_allocation(cache_get_client_entry_size(client_desc)); + + done: + return 0; +} + +/* Clean the client cache using now as the current time. Return the total size + * of removed bytes from the cache. */ +static size_t +cache_clean_v3_as_client(time_t now) +{ + size_t bytes_removed = 0; + + if (!hs_cache_v3_client) { /* No cache to clean. Just return. */ + return 0; + } + + DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key, + hs_cache_client_descriptor_t *, entry) { + size_t entry_size; + time_t cutoff = now - rend_cache_max_entry_lifetime(); + + /* If the entry has been created _after_ the cutoff, not expired so + * continue to the next entry in our v3 cache. */ + if (entry->created_ts > cutoff) { + continue; + } + /* Here, our entry has expired, remove and free. */ + MAP_DEL_CURRENT(key); + entry_size = cache_get_client_entry_size(entry); + bytes_removed += entry_size; + /* Entry is not in the cache anymore, destroy it. */ + cache_client_desc_free(entry); + /* Update our cache entry allocation size for the OOM. */ + rend_cache_decrement_allocation(entry_size); + /* Logging. */ + { + char key_b64[BASE64_DIGEST256_LEN + 1]; + base64_encode(key_b64, sizeof(key_b64), (const char *) key, + DIGEST256_LEN, 0); + log_info(LD_REND, "Removing hidden service v3 descriptor '%s' " + "from client cache", + safe_str_client(key_b64)); + } + } DIGEST256MAP_FOREACH_END; + + return bytes_removed; +} + +/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return + * its HS descriptor if it's stored in our cache, or NULL if not. */ +const hs_descriptor_t * +hs_cache_lookup_as_client(const ed25519_public_key_t *key) +{ + hs_cache_client_descriptor_t *cached_desc = NULL; + + tor_assert(key); + + cached_desc = lookup_v3_desc_as_client(key->pubkey); + if (cached_desc) { + tor_assert(cached_desc->desc); + return cached_desc->desc; + } + + return NULL; +} + +/** Public API: Given an encoded descriptor, store it in the client HS + * cache. Return -1 on error, 0 on success .*/ +int +hs_cache_store_as_client(const char *desc_str, + const ed25519_public_key_t *identity_pk) +{ + hs_cache_client_descriptor_t *client_desc = NULL; + + tor_assert(desc_str); + tor_assert(identity_pk); + + /* Create client cache descriptor object */ + client_desc = cache_client_desc_new(desc_str, identity_pk); + if (!client_desc) { + log_warn(LD_GENERAL, "Failed to parse received descriptor %s.", + escaped(desc_str)); + goto err; + } + + /* Push it to the cache */ + if (cache_store_as_client(client_desc) < 0) { + goto err; + } + + return 0; + + err: + cache_client_desc_free(client_desc); + return -1; +} + +/* Clean all client caches using the current time now. */ +void +hs_cache_clean_as_client(time_t now) +{ + /* Start with v2 cache cleaning. */ + rend_cache_clean(now, REND_CACHE_TYPE_CLIENT); + /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function + * to compute the cutoff by itself using the lifetime value. */ + cache_clean_v3_as_client(now); +} + +/**************** Generics *********************************/ + /* Do a round of OOM cleanup on all directory caches. Return the amount of * removed bytes. It is possible that the returned value is lower than * min_remove_bytes if the caches get emptied out so the caller should be @@ -388,6 +629,9 @@ hs_cache_init(void) /* Calling this twice is very wrong code flow. */ tor_assert(!hs_cache_v3_dir); hs_cache_v3_dir = digest256map_new(); + + tor_assert(!hs_cache_v3_client); + hs_cache_v3_client = digest256map_new(); }

/* Cleanup the hidden service cache subsystem. */ @@ -396,5 +640,8 @@ hs_cache_free_all(void) { digest256map_free(hs_cache_v3_dir, cache_dir_desc_free_); hs_cache_v3_dir = NULL; + + digest256map_free(hs_cache_v3_client, cache_client_desc_free_); + hs_cache_v3_client = NULL; }

diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h index ed0042423..79456f69c 100644 --- a/src/or/hs_cache.h +++ b/src/or/hs_cache.h @@ -53,10 +53,35 @@ int hs_cache_store_as_dir(const char *desc); int hs_cache_lookup_as_dir(uint32_t version, const char *query, const char **desc_out);

+const hs_descriptor_t * +hs_cache_lookup_as_client(const ed25519_public_key_t *key); +int hs_cache_store_as_client(const char *desc_str, + const ed25519_public_key_t *identity_pk); +void hs_cache_clean_as_client(time_t now); + #ifdef HS_CACHE_PRIVATE

+/** Represents a locally cached HS descriptor on a hidden service client. */ +typedef struct hs_cache_client_descriptor_t { + /* This object is indexed using the service identity public key */ + ed25519_public_key_t key; + + /* When was this entry created. Used to expire entries. */ + time_t created_ts; + + /* The cached descriptor, this object is the owner. It can't be NULL. A + * cache object without a valid descriptor is not possible. */ + hs_descriptor_t *desc; + + /* Encoded descriptor in string form. Can't be NULL. */ + char *encoded_desc; +} hs_cache_client_descriptor_t; + STATIC size_t cache_clean_v3_as_dir(time_t now, time_t global_cutoff);

+STATIC hs_cache_client_descriptor_t * +lookup_v3_desc_as_client(const uint8_t *key); + #endif /* HS_CACHE_PRIVATE */

#endif /* TOR_HS_CACHE_H */ diff --git a/src/or/hs_client.c b/src/or/hs_client.c index 051490aaa..37981c8b6 100644 --- a/src/or/hs_client.c +++ b/src/or/hs_client.c @@ -46,3 +46,42 @@ hs_client_note_connection_attempt_succeeded(const edge_connection_t *conn) } }

+/* With the given encoded descriptor in desc_str and the service key in + * service_identity_pk, decode the descriptor and set the desc pointer with a + * newly allocated descriptor object. + * + * Return 0 on success else a negative value and desc is set to NULL. */ +int +hs_client_decode_descriptor(const char *desc_str, + const ed25519_public_key_t *service_identity_pk, + hs_descriptor_t **desc) +{ + int ret; + uint8_t subcredential[DIGEST256_LEN]; + + tor_assert(desc_str); + tor_assert(service_identity_pk); + tor_assert(desc); + + /* Create subcredential for this HS so that we can decrypt */ + { + ed25519_public_key_t blinded_pubkey; + uint64_t current_time_period = hs_get_time_period_num(approx_time()); + hs_build_blinded_pubkey(service_identity_pk, NULL, 0, current_time_period, + &blinded_pubkey); + hs_get_subcredential(service_identity_pk, &blinded_pubkey, subcredential); + } + + /* Parse descriptor */ + ret = hs_desc_decode_descriptor(desc_str, subcredential, desc); + memwipe(subcredential, 0, sizeof(subcredential)); + if (ret < 0) { + log_warn(LD_GENERAL, "Could not parse received descriptor as client"); + goto err; + } + + return 0; + err: + return -1; +} + diff --git a/src/or/hs_client.h b/src/or/hs_client.h index 4f28937b0..348724ab3 100644 --- a/src/or/hs_client.h +++ b/src/or/hs_client.h @@ -9,8 +9,15 @@ #ifndef TOR_HS_CLIENT_H #define TOR_HS_CLIENT_H

+#include "hs_descriptor.h" + void hs_client_note_connection_attempt_succeeded( const edge_connection_t *conn);

+int hs_client_decode_descriptor( + const char *desc_str, + const ed25519_public_key_t *service_identity_pk, + hs_descriptor_t **desc); + #endif /* TOR_HS_CLIENT_H */

diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c index 3cdc023f7..1e595f3a3 100644 --- a/src/or/hs_descriptor.c +++ b/src/or/hs_descriptor.c @@ -2437,6 +2437,37 @@ hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data) data->superencrypted_blob_size); }

+/* Return the size in bytes of the given encrypted data object. Used by OOM + * subsystem. */ +static size_t +hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data) +{ + tor_assert(data); + size_t intro_size = 0; + if (data->intro_auth_types) { + intro_size += + smartlist_len(data->intro_auth_types) * sizeof(intro_auth_types); + } + if (data->intro_points) { + /* XXX could follow pointers here and get more accurate size */ + intro_size += + smartlist_len(data->intro_points) * sizeof(hs_desc_intro_point_t); + } + + return sizeof(*data) + intro_size; +} + +/* Return the size in bytes of the given descriptor object. Used by OOM + * subsystem. */ + size_t +hs_desc_obj_size(const hs_descriptor_t *data) +{ + tor_assert(data); + return (hs_desc_plaintext_obj_size(&data->plaintext_data) + + hs_desc_encrypted_obj_size(&data->encrypted_data) + + sizeof(data->subcredential)); +} + /* Return a newly allocated descriptor intro point. */ hs_desc_intro_point_t * hs_desc_intro_point_new(void) diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h index 19e52333b..256a04dd6 100644 --- a/src/or/hs_descriptor.h +++ b/src/or/hs_descriptor.h @@ -224,6 +224,7 @@ int hs_desc_decode_plaintext(const char *encoded, int hs_desc_decode_encrypted(const hs_descriptor_t *desc, hs_desc_encrypted_data_t *desc_out);

+size_t hs_desc_obj_size(const hs_descriptor_t *data); size_t hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data);

hs_desc_intro_point_t *hs_desc_intro_point_new(void); diff --git a/src/or/main.c b/src/or/main.c index 86fdb9328..3db0b5e90 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1827,8 +1827,8 @@ clean_caches_callback(time_t now, const or_options_t *options) { /* Remove old information from rephist and the rend cache. */ rep_history_clean(now - options->RephistTrackTime); - rend_cache_clean(now, REND_CACHE_TYPE_CLIENT); rend_cache_clean(now, REND_CACHE_TYPE_SERVICE); + hs_cache_clean_as_client(now); hs_cache_clean_as_dir(now); microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60)