[or-cvs] [tor/master 5/6] Download microdescriptors if you're a cache
nickm at torproject.org
nickm at torproject.org
Tue Sep 28 04:44:16 UTC 2010
Author: Nick Mathewson <nickm at torproject.org>
Date: Tue, 11 May 2010 17:20:33 -0400
Subject: Download microdescriptors if you're a cache
Commit: 3a492d31d5c50ef3b766881ae1d765c296d55797
This commit adds some functions to see what microdescriptors we're missing,
and adds fetch-microdesc/store-microdesc logic to the directory code.
---
src/or/directory.c | 106 ++++++++++++++++++++++++++++++++++++---
src/or/main.c | 6 ++-
src/or/microdesc.c | 115 ++++++++++++++++++++++++++++++++++++++++---
src/or/microdesc.h | 11 ++++-
src/or/networkstatus.c | 27 ++++++----
src/or/networkstatus.h | 3 +-
src/or/or.h | 19 ++++++-
src/or/routerlist.c | 129 +++++++++++++++++++++++++++++++++++------------
src/or/routerlist.h | 6 ++
9 files changed, 359 insertions(+), 63 deletions(-)
diff --git a/src/or/directory.c b/src/or/directory.c
index 4411278..52a0f9f 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -15,6 +15,7 @@
#include "dirvote.h"
#include "geoip.h"
#include "main.h"
+#include "microdesc.h"
#include "networkstatus.h"
#include "policies.h"
#include "rendclient.h"
@@ -78,6 +79,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed,
int router_purpose,
int was_extrainfo,
int was_descriptor_digests);
+static void dir_microdesc_download_failed(smartlist_t *failed,
+ int status_code);
static void note_client_request(int purpose, int compressed, size_t bytes);
static int client_likes_consensus(networkstatus_t *v, const char *want_url);
@@ -137,7 +140,8 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
- dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
+ dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+ dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
return 0;
return 1;
}
@@ -201,6 +205,8 @@ dir_conn_purpose_to_string(int purpose)
return "hidden-service v2 descriptor fetch";
case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
return "hidden-service v2 descriptor upload";
+ case DIR_PURPOSE_FETCH_MICRODESC:
+ return "microdescriptor fetch";
}
log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
@@ -355,6 +361,9 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
case DIR_PURPOSE_FETCH_CERTIFICATE:
type = V3_AUTHORITY;
break;
+ case DIR_PURPOSE_FETCH_MICRODESC:
+ type = V3_AUTHORITY;
+ break;
default:
log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
return;
@@ -410,7 +419,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
if (prefer_authority || type == BRIDGE_AUTHORITY) {
/* only ask authdirservers, and don't ask myself */
rs = router_pick_trusteddirserver(type, pds_flags);
- if (rs == NULL && (pds_flags & PDS_NO_EXISTING_SERVERDESC_FETCH)) {
+ if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+ PDS_NO_EXISTING_MICRODESC_FETCH))) {
/* We don't want to fetch from any authorities that we're currently
* fetching server descriptors from, and we got no match. Did we
* get no match because all the authorities have connections
@@ -418,7 +428,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
* return,) or because all the authorities are down or on fire or
* unreachable or something (in which case we should go on with
* our fallback code)? */
- pds_flags &= ~PDS_NO_EXISTING_SERVERDESC_FETCH;
+ pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
+ PDS_NO_EXISTING_MICRODESC_FETCH);
rs = router_pick_trusteddirserver(type, pds_flags);
if (rs) {
log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
@@ -607,7 +618,8 @@ connection_dir_request_failed(dir_connection_t *conn)
connection_dir_download_networkstatus_failed(conn, -1);
} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
- log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
+ log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
+ "directory server at '%s'; retrying",
conn->_base.address);
if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
connection_dir_bridge_routerdesc_failed(conn);
@@ -617,7 +629,8 @@ connection_dir_request_failed(dir_connection_t *conn)
conn->requested_resource ? conn->requested_resource : "ns";
networkstatus_consensus_download_failed(0, flavname);
} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
- log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
+ log_info(LD_DIR, "Giving up on certificate fetch from directory server "
+ "at '%s'; retrying",
conn->_base.address);
connection_dir_download_cert_failed(conn, 0);
} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
@@ -626,6 +639,10 @@ connection_dir_request_failed(dir_connection_t *conn)
} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
log_info(LD_DIR, "Giving up downloading votes from '%s'",
conn->_base.address);
+ } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+ log_info(LD_DIR, "Giving up on downloading microdescriptors from "
+ " directory server at '%s'; will retry", conn->_base.address);
+ connection_dir_download_routerdesc_failed(conn);
}
}
@@ -696,7 +713,8 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn)
/* No need to relaunch descriptor downloads here: we already do it
* every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
- conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+ conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+ conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
(void) conn;
}
@@ -1136,6 +1154,11 @@ directory_send_command(dir_connection_t *conn,
url = tor_malloc(len);
tor_snprintf(url, len, "/tor/extra/%s", resource);
break;
+ case DIR_PURPOSE_FETCH_MICRODESC:
+ tor_assert(resource);
+ httpcommand = "GET";
+ tor_asprintf(&url, "/tor/micro/%s.z", resource);
+ break;
case DIR_PURPOSE_UPLOAD_DIR:
tor_assert(!resource);
tor_assert(payload);
@@ -1411,6 +1434,9 @@ body_is_plausible(const char *body, size_t len, int purpose)
return 1; /* empty bodies don't need decompression */
if (len < 32)
return 0;
+ if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+ return (!strcmpstart(body,"onion-key"));
+ }
if (purpose != DIR_PURPOSE_FETCH_RENDDESC) {
if (!strcmpstart(body,"router") ||
!strcmpstart(body,"signed-directory") ||
@@ -1487,7 +1513,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
int plausible;
int skewed=0;
int allow_partial = (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
- conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+ conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+ conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
int was_compressed=0;
time_t now = time(NULL);
@@ -1886,6 +1913,41 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
if (directory_conn_is_self_reachability_test(conn))
router_dirport_found_reachable();
}
+ if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+ smartlist_t *which = NULL;
+ log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
+ "size %d) from server '%s:%d'",
+ status_code, (int)body_len, conn->_base.address, conn->_base.port);
+ tor_assert(conn->requested_resource &&
+ !strcmpstart(conn->requested_resource, "d/"));
+ which = smartlist_create();
+ dir_split_resource_into_fingerprints(conn->requested_resource+2,
+ which, NULL,
+ DSR_DIGEST256|DSR_BASE64);
+ if (status_code != 200) {
+ log_info(LD_DIR, "Received status code %d (%s) from server "
+ "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again "
+ "soon.",
+ status_code, escaped(reason), conn->_base.address,
+ (int)conn->_base.port, conn->requested_resource);
+ dir_microdesc_download_failed(which, status_code);
+ SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+ smartlist_free(which);
+ tor_free(body); tor_free(headers); tor_free(reason);
+ return 0;
+ } else {
+ smartlist_t *mds;
+ mds = microdescs_add_to_cache(get_microdesc_cache(),
+ body, body+body_len, SAVED_NOWHERE, 0,
+ now, which);
+ if (smartlist_len(which)) {
+ /* Mark remaining ones as failed. */
+ dir_microdesc_download_failed(which, status_code);
+ }
+ SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+ smartlist_free(which);
+ }
+ }
if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_DIR) {
switch (status_code) {
@@ -3612,6 +3674,36 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
* every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
}
+/* DOCDOC NM */
+static void
+dir_microdesc_download_failed(smartlist_t *failed,
+ int status_code)
+{
+ networkstatus_t *consensus
+ = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+ routerstatus_t *rs;
+ download_status_t *dls;
+ time_t now = time(NULL);
+ int server = directory_fetches_from_authorities(get_options());
+
+ if (! consensus)
+ return;
+ SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
+ rs = router_get_consensus_status_by_descriptor_digest(consensus, d);
+ if (!rs)
+ continue;
+ dls = &rs->dl_status;
+ if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES)
+ continue;
+ {
+ char buf[BASE64_DIGEST256_LEN+1];
+ digest256_to_base64(buf, d);
+ download_status_increment_failure(dls, status_code, buf,
+ server, now);
+ }
+ } SMARTLIST_FOREACH_END(d);
+}
+
/** Helper. Compare two fp_pair_t objects, and return -1, 0, or 1 as
* appropriate. */
static int
diff --git a/src/or/main.c b/src/or/main.c
index adc2d3d..8ea1728 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -851,10 +851,13 @@ directory_info_has_arrived(time_t now, int from_cache)
"I learned some more directory information, but not enough to "
"build a circuit: %s", get_dir_info_status_string());
update_router_descriptor_downloads(now);
+ update_microdesc_downloads(now);
return;
} else {
- if (directory_fetches_from_authorities(options))
+ if (directory_fetches_from_authorities(options)) {
update_router_descriptor_downloads(now);
+ update_microdesc_downloads(now);
+ }
/* if we have enough dir info, then update our guard status with
* whatever we just learned. */
@@ -1062,6 +1065,7 @@ run_scheduled_events(time_t now)
if (time_to_try_getting_descriptors < now) {
update_router_descriptor_downloads(now);
update_extrainfo_downloads(now);
+ update_microdesc_downloads(now);
if (options->UseBridges)
fetch_bridge_descriptors(now);
if (router_have_minimum_dir_info())
diff --git a/src/or/microdesc.c b/src/or/microdesc.c
index e8f3e7c..e42d55a 100644
--- a/src/or/microdesc.c
+++ b/src/or/microdesc.c
@@ -3,8 +3,12 @@
#include "or.h"
#include "config.h"
+#include "directory.h"
#include "microdesc.h"
#include "routerparse.h"
+#include "networkstatus.h"
+#include "routerlist.h"
+#include "dirserv.h"
/** A data structure to hold a bunch of cached microdescriptors. There are
* two active files in the cache: a "cache file" that we mmap, and a "journal
@@ -119,15 +123,19 @@ get_microdesc_cache(void)
* ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>,
* mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
* leave their bodies as pointers to the mmap'd cache. If where is
- * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added
- * microdescriptors. */
+ * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is positive,
+ * set the last_listed field of every microdesc to listed_at. If
+ * requested_digests is non-null, then it contains a list of digests we mean
+ * to allow, so we should reject any non-requested microdesc with a different
+ * digest, and alter the list to contain only the digests of those microdescs we
+ * didn't find.
+ * Return a list of the added microdescriptors. */
smartlist_t *
microdescs_add_to_cache(microdesc_cache_t *cache,
const char *s, const char *eos, saved_location_t where,
- int no_save)
+ int no_save, time_t listed_at,
+ smartlist_t *requested_digests256)
{
- /*XXXX need an argument that sets last_listed as appropriate. */
-
smartlist_t *descriptors, *added;
const int allow_annotations = (where != SAVED_NOWHERE);
const int copy_body = (where != SAVED_IN_CACHE);
@@ -135,6 +143,33 @@ microdescs_add_to_cache(microdesc_cache_t *cache,
descriptors = microdescs_parse_from_string(s, eos,
allow_annotations,
copy_body);
+ if (listed_at > 0) {
+ SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
+ md->last_listed = listed_at);
+ }
+ if (requested_digests256) {
+ digestmap_t *requested; /* XXXX actuqlly we should just use a
+ digest256map */
+ requested = digestmap_new();
+ SMARTLIST_FOREACH(requested_digests256, const char *, cp,
+ digestmap_set(requested, cp, (void*)1));
+ SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+ if (digestmap_get(requested, md->digest)) {
+ digestmap_set(requested, md->digest, (void*)2);
+ } else {
+ log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microcdesc");
+ microdesc_free(md);
+ SMARTLIST_DEL_CURRENT(descriptors, md);
+ }
+ } SMARTLIST_FOREACH_END(md);
+ SMARTLIST_FOREACH_BEGIN(requested_digests256, char *, cp) {
+ if (digestmap_get(requested, cp) == (void*)2) {
+ tor_free(cp);
+ SMARTLIST_DEL_CURRENT(requested_digests256, cp);
+ }
+ } SMARTLIST_FOREACH_END(cp);
+ digestmap_free(requested, NULL);
+ }
added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
smartlist_free(descriptors);
@@ -251,7 +286,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
if (mm) {
added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
- SAVED_IN_CACHE, 0);
+ SAVED_IN_CACHE, 0, -1, NULL);
if (added) {
total += smartlist_len(added);
smartlist_free(added);
@@ -263,7 +298,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
if (journal_content) {
added = microdescs_add_to_cache(cache, journal_content,
journal_content+st.st_size,
- SAVED_IN_JOURNAL, 0);
+ SAVED_IN_JOURNAL, 0, -1, NULL);
if (added) {
total += smartlist_len(added);
smartlist_free(added);
@@ -412,3 +447,69 @@ microdesc_average_size(microdesc_cache_t *cache)
return (size_t)(cache->total_len_seen / cache->n_seen);
}
+/** Return a smartlist of all the sha256 digest of the microdescriptors that
+ * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
+ * to internals of <b>ns</b>; you should not free the members of the resulting
+ * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
+smartlist_t *
+microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
+ int downloadable_only, digestmap_t *skip)
+{
+ smartlist_t *result = smartlist_create();
+ time_t now = time(NULL);
+ tor_assert(ns->flavor == FLAV_MICRODESC);
+ SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+ if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
+ continue;
+ if (downloadable_only &&
+ !download_status_is_ready(&rs->dl_status, now,
+ MAX_MICRODESC_DOWNLOAD_FAILURES))
+ continue;
+ if (skip && digestmap_get(skip, rs->descriptor_digest))
+ continue;
+ /* XXXX Also skip if we're a noncache and wouldn't use this router.
+ * XXXX NM Microdesc
+ */
+ smartlist_add(result, rs->descriptor_digest);
+ } SMARTLIST_FOREACH_END(rs);
+ return result;
+}
+
+/** DOCDOC */
+void
+update_microdesc_downloads(time_t now)
+{
+ or_options_t *options = get_options();
+ networkstatus_t *consensus;
+ smartlist_t *missing;
+ digestmap_t *pending;
+
+ if (should_delay_dir_fetches(options))
+ return;
+ if (directory_too_idle_to_fetch_descriptors(options, now))
+ return;
+
+ consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+ if (!consensus)
+ return;
+
+ if (!directory_caches_dir_info(options)) {
+ /* Right now, only caches fetch microdescriptors.
+ * XXXX NM Microdescs */
+ return;
+ }
+
+ pending = digestmap_new();
+ list_pending_microdesc_downloads(pending);
+
+ missing = microdesc_list_missing_digest256(consensus,
+ get_microdesc_cache(),
+ 1,
+ pending);
+ digestmap_free(pending, NULL);
+
+ launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
+ missing, NULL, now);
+
+ smartlist_free(missing);
+}
diff --git a/src/or/microdesc.h b/src/or/microdesc.h
index 2d1a60a..7cba3a3 100644
--- a/src/or/microdesc.h
+++ b/src/or/microdesc.h
@@ -16,7 +16,8 @@ microdesc_cache_t *get_microdesc_cache(void);
smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
const char *s, const char *eos, saved_location_t where,
- int no_save);
+ int no_save, time_t listed_at,
+ smartlist_t *requested_digests256);
smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
smartlist_t *descriptors, saved_location_t where,
int no_save);
@@ -30,8 +31,16 @@ microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
size_t microdesc_average_size(microdesc_cache_t *cache);
+smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
+ microdesc_cache_t *cache,
+ int downloadable_only,
+ digestmap_t *skip);
+
void microdesc_free(microdesc_t *md);
void microdesc_free_all(void);
+void update_microdesc_downloads(time_t now);
+
+
#endif
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index 4ccd1ae..704e665 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -978,21 +978,25 @@ networkstatus_get_v2_list(void)
}
/** Return the consensus view of the status of the router whose current
- * <i>descriptor</i> digest is <b>digest</b>, or NULL if no such router is
- * known. */
+ * <i>descriptor</i> digest in <b>consensus</b> is <b>digest</b>, or NULL if
+ * no such router is known. */
routerstatus_t *
-router_get_consensus_status_by_descriptor_digest(const char *digest)
+router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus,
+ const char *digest)
{
- if (!current_consensus) return NULL;
- if (!current_consensus->desc_digest_map) {
- digestmap_t * m = current_consensus->desc_digest_map = digestmap_new();
- SMARTLIST_FOREACH(current_consensus->routerstatus_list,
+ if (!consensus)
+ consensus = current_consensus;
+ if (!consensus)
+ return NULL;
+ if (!consensus->desc_digest_map) {
+ digestmap_t *m = consensus->desc_digest_map = digestmap_new();
+ SMARTLIST_FOREACH(consensus->routerstatus_list,
routerstatus_t *, rs,
{
digestmap_set(m, rs->descriptor_digest, rs);
});
}
- return digestmap_get(current_consensus->desc_digest_map, digest);
+ return digestmap_get(consensus->desc_digest_map, digest);
}
/** Given the digest of a router descriptor, return its current download
@@ -1001,7 +1005,10 @@ download_status_t *
router_get_dl_status_by_descriptor_digest(const char *d)
{
routerstatus_t *rs;
- if ((rs = router_get_consensus_status_by_descriptor_digest(d)))
+ if (!current_ns_consensus)
+ return NULL;
+ if ((rs = router_get_consensus_status_by_descriptor_digest(
+ current_ns_consensus, d)))
return &rs->dl_status;
if (v2_download_status_map)
return digestmap_get(v2_download_status_map, d);
@@ -2118,7 +2125,7 @@ signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
char dummy[DIGEST_LEN];
/* instantiates the digest map. */
memset(dummy, 0, sizeof(dummy));
- router_get_consensus_status_by_descriptor_digest(dummy);
+ router_get_consensus_status_by_descriptor_digest(ns, dummy);
}
SMARTLIST_FOREACH(descs, signed_descriptor_t *, d,
{
diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h
index 566171c..adaddc4 100644
--- a/src/or/networkstatus.h
+++ b/src/or/networkstatus.h
@@ -48,7 +48,8 @@ const smartlist_t *networkstatus_get_v2_list(void);
download_status_t *router_get_dl_status_by_descriptor_digest(const char *d);
routerstatus_t *router_get_consensus_status_by_id(const char *digest);
routerstatus_t *router_get_consensus_status_by_descriptor_digest(
- const char *digest);
+ networkstatus_t *consensus,
+ const char *digest);
routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname,
int warn_if_unnamed);
const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
diff --git a/src/or/or.h b/src/or/or.h
index cd54279..4741cc3 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -391,7 +391,9 @@ typedef enum {
/** A connection to a hidden service directory server: download a v2 rendezvous
* descriptor. */
#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
-#define _DIR_PURPOSE_MAX 18
+/** A connection to a directory server: download a microdescriptor. */
+#define DIR_PURPOSE_FETCH_MICRODESC 19
+#define _DIR_PURPOSE_MAX 19
/** True iff <b>p</b> is a purpose corresponding to uploading data to a
* directory server. */
@@ -1191,8 +1193,13 @@ typedef struct edge_connection_t {
typedef struct dir_connection_t {
connection_t _base;
- char *requested_resource; /**< Which 'resource' did we ask the directory
- * for? */
+ /** Which 'resource' did we ask the directory for? This is typically the part
+ * of the URL string that defines, relative to the directory conn purpose,
+ * what thing we want. For example, in router descriptor downloads by
+ * descriptor digest, it contains "d/", then one ore more +-separated
+ * fingerprints.
+ **/
+ char *requested_resource;
unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
/* Used only for server sides of some dir connections, to implement
@@ -1690,6 +1697,10 @@ typedef struct microdesc_t {
* up? */
#define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8
+/** How many times will we try to download a microdescriptor before giving
+ * up? */
+#define MAX_MICRODESC_DOWNLOAD_FAILURES 8
+
/** Contents of a v2 (non-consensus, non-vote) network status object. */
typedef struct networkstatus_v2_t {
/** When did we receive the network-status document? */
@@ -3517,6 +3528,8 @@ typedef struct trusted_dir_server_t {
* fetches to _any_ single directory server.]
*/
#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
#define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16)
/** Possible ways to weight routers when choosing one randomly. See
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index 27b870d..e55973f 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -22,6 +22,7 @@
#include "geoip.h"
#include "hibernate.h"
#include "main.h"
+#include "microdesc.h"
#include "networkstatus.h"
#include "policies.h"
#include "reasons.h"
@@ -44,9 +45,6 @@ static routerstatus_t *router_pick_trusteddirserver_impl(
static void mark_all_trusteddirservers_up(void);
static int router_nickname_matches(routerinfo_t *router, const char *nickname);
static void trusted_dir_server_free(trusted_dir_server_t *ds);
-static void launch_router_descriptor_downloads(smartlist_t *downloadable,
- routerstatus_t *source,
- time_t now);
static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
static void update_router_have_minimum_dir_info(void);
static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc,
@@ -1047,7 +1045,8 @@ router_pick_trusteddirserver(authority_type_t type, int flags)
/* If the reason that we got no server is that servers are "busy",
* we must be excluding good servers because we already have serverdesc
* fetches with them. Do not mark down servers up because of this. */
- tor_assert((flags & PDS_NO_EXISTING_SERVERDESC_FETCH));
+ tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+ PDS_NO_EXISTING_MICRODESC_FETCH)));
return NULL;
}
@@ -1174,6 +1173,7 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags,
const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
const int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS);
const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+ const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
int n_busy = 0;
if (!trusted_dir_servers)
@@ -1212,6 +1212,13 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags,
continue;
}
}
+ if (no_microdesc_fetching) {
+ if (connection_get_by_type_addr_port_purpose(
+ CONN_TYPE_DIR, &addr, d->dir_port, DIR_PURPOSE_FETCH_MICRODESC)) {
+ ++n_busy;
+ continue;
+ }
+ }
if (prefer_tunnel &&
d->or_port &&
@@ -3874,6 +3881,7 @@ routerlist_retry_directory_downloads(time_t now)
router_reset_descriptor_download_failures();
update_networkstatus_downloads(now);
update_router_descriptor_downloads(now);
+ update_microdesc_downloads(now);
}
/** Return 1 if all running sufficiently-stable routers will reject
@@ -4035,7 +4043,9 @@ any_trusted_dir_is_v1_authority(void)
/** For every current directory connection whose purpose is <b>purpose</b>,
* and where the resource being downloaded begins with <b>prefix</b>, split
* rest of the resource into base16 fingerprints, decode them, and set the
- * corresponding elements of <b>result</b> to a nonzero value. */
+ * corresponding elements of <b>result</b> to a nonzero value.
+ * DOCDOC purpose==microdesc
+ */
static void
list_pending_downloads(digestmap_t *result,
int purpose, const char *prefix)
@@ -4043,20 +4053,23 @@ list_pending_downloads(digestmap_t *result,
const size_t p_len = strlen(prefix);
smartlist_t *tmp = smartlist_create();
smartlist_t *conns = get_connection_array();
+ int flags = DSR_HEX;
+ if (purpose == DIR_PURPOSE_FETCH_MICRODESC)
+ flags = DSR_DIGEST256|DSR_BASE64;
tor_assert(result);
- SMARTLIST_FOREACH(conns, connection_t *, conn,
- {
+ SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
if (conn->type == CONN_TYPE_DIR &&
conn->purpose == purpose &&
!conn->marked_for_close) {
const char *resource = TO_DIR_CONN(conn)->requested_resource;
if (!strcmpstart(resource, prefix))
dir_split_resource_into_fingerprints(resource + p_len,
- tmp, NULL, DSR_HEX);
+ tmp, NULL, flags);
}
- });
+ } SMARTLIST_FOREACH_END(conn);
+
SMARTLIST_FOREACH(tmp, char *, d,
{
digestmap_set(result, d, (void*)1);
@@ -4076,10 +4089,18 @@ list_pending_descriptor_downloads(digestmap_t *result, int extrainfo)
list_pending_downloads(result, purpose, "d/");
}
-/** Launch downloads for all the descriptors whose digests are listed
- * as digests[i] for lo <= i < hi. (Lo and hi may be out of range.)
- * If <b>source</b> is given, download from <b>source</b>; otherwise,
- * download from an appropriate random directory server.
+/** DOCDOC */
+/*XXXX NM should use digest256, if one comes into being. */
+void
+list_pending_microdesc_downloads(digestmap_t *result)
+{
+ list_pending_downloads(result, DIR_PURPOSE_FETCH_MICRODESC, "d/");
+}
+
+/** Launch downloads for all the descriptors whose digests or digests256
+ * are listed as digests[i] for lo <= i < hi. (Lo and hi may be out of
+ * range.) If <b>source</b> is given, download from <b>source</b>;
+ * otherwise, download from an appropriate random directory server.
*/
static void
initiate_descriptor_downloads(routerstatus_t *source,
@@ -4090,6 +4111,20 @@ initiate_descriptor_downloads(routerstatus_t *source,
int i, n = hi-lo;
char *resource, *cp;
size_t r_len;
+
+ int digest_len = DIGEST_LEN, enc_digest_len = HEX_DIGEST_LEN;
+ char sep = '+';
+ int b64_256 = 0;
+
+ if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+ /* Microdescriptors are downloaded by "-"-separated base64-encoded
+ * 256-bit digests. */
+ digest_len = DIGEST256_LEN;
+ enc_digest_len = BASE64_DIGEST256_LEN;
+ sep = '-';
+ b64_256 = 1;
+ }
+
if (n <= 0)
return;
if (lo < 0)
@@ -4097,15 +4132,19 @@ initiate_descriptor_downloads(routerstatus_t *source,
if (hi > smartlist_len(digests))
hi = smartlist_len(digests);
- r_len = 8 + (HEX_DIGEST_LEN+1)*n;
+ r_len = 8 + (enc_digest_len+1)*n;
cp = resource = tor_malloc(r_len);
memcpy(cp, "d/", 2);
cp += 2;
for (i = lo; i < hi; ++i) {
- base16_encode(cp, r_len-(cp-resource),
- smartlist_get(digests,i), DIGEST_LEN);
- cp += HEX_DIGEST_LEN;
- *cp++ = '+';
+ if (b64_256) {
+ digest256_to_base64(cp, smartlist_get(digests, i));
+ } else {
+ base16_encode(cp, r_len-(cp-resource),
+ smartlist_get(digests,i), digest_len);
+ }
+ cp += enc_digest_len;
+ *cp++ = sep;
}
memcpy(cp-1, ".z", 3);
@@ -4152,6 +4191,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options)
* So use 96 because it's a nice number.
*/
#define MAX_DL_PER_REQUEST 96
+#define MAX_MICRODESC_DL_PER_REQUEST 92
/** Don't split our requests so finely that we are requesting fewer than
* this number per server. */
#define MIN_DL_PER_REQUEST 4
@@ -4166,21 +4206,33 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options)
* them until they have more, or until this amount of time has passed. */
#define MAX_CLIENT_INTERVAL_WITHOUT_REQUEST (10*60)
-/** Given a list of router descriptor digests in <b>downloadable</b>, decide
- * whether to delay fetching until we have more. If we don't want to delay,
- * launch one or more requests to the appropriate directory authorities. */
-static void
-launch_router_descriptor_downloads(smartlist_t *downloadable,
- routerstatus_t *source, time_t now)
+/** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of
+ * router descriptor digests or microdescriptor digest256s in
+ * <b>downloadable</b>, decide whether to delay fetching until we have more.
+ * If we don't want to delay, launch one or more requests to the appropriate
+ * directory authorities.
+ */
+void
+launch_descriptor_downloads(int purpose,
+ smartlist_t *downloadable,
+ routerstatus_t *source, time_t now)
{
int should_delay = 0, n_downloadable;
or_options_t *options = get_options();
+ const char *descname;
+
+ tor_assert(purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+ purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+ descname = (purpose == DIR_PURPOSE_FETCH_SERVERDESC) ?
+ "routerdesc" : "microdesc";
n_downloadable = smartlist_len(downloadable);
if (!directory_fetches_dir_info_early(options)) {
if (n_downloadable >= MAX_DL_TO_DELAY) {
log_debug(LD_DIR,
- "There are enough downloadable routerdescs to launch requests.");
+ "There are enough downloadable %ss to launch requests.",
+ descname);
should_delay = 0;
} else {
should_delay = (last_routerdesc_download_attempted +
@@ -4188,13 +4240,15 @@ launch_router_descriptor_downloads(smartlist_t *downloadable,
if (!should_delay && n_downloadable) {
if (last_routerdesc_download_attempted) {
log_info(LD_DIR,
- "There are not many downloadable routerdescs, but we've "
+ "There are not many downloadable %ss, but we've "
"been waiting long enough (%d seconds). Downloading.",
+ descname,
(int)(now-last_routerdesc_download_attempted));
} else {
log_info(LD_DIR,
- "There are not many downloadable routerdescs, but we haven't "
- "tried downloading descriptors recently. Downloading.");
+ "There are not many downloadable %ss, but we haven't "
+ "tried downloading descriptors recently. Downloading.",
+ descname);
}
}
}
@@ -4221,12 +4275,20 @@ launch_router_descriptor_downloads(smartlist_t *downloadable,
* update_router_descriptor_downloads() later on, once the connections
* have succeeded or failed.
*/
- pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH;
+ pds_flags |= (purpose == DIR_PURPOSE_FETCH_MICRODESC) ?
+ PDS_NO_EXISTING_MICRODESC_FETCH :
+ PDS_NO_EXISTING_SERVERDESC_FETCH;
+
}
n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS);
- if (n_per_request > MAX_DL_PER_REQUEST)
- n_per_request = MAX_DL_PER_REQUEST;
+ if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+ if (n_per_request > MAX_MICRODESC_DL_PER_REQUEST)
+ n_per_request = MAX_MICRODESC_DL_PER_REQUEST;
+ } else {
+ if (n_per_request > MAX_DL_PER_REQUEST)
+ n_per_request = MAX_DL_PER_REQUEST;
+ }
if (n_per_request < MIN_DL_PER_REQUEST)
n_per_request = MIN_DL_PER_REQUEST;
@@ -4241,7 +4303,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable,
req_plural, n_downloadable, rtr_plural, n_per_request);
smartlist_sort_digests(downloadable);
for (i=0; i < n_downloadable; i += n_per_request) {
- initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC,
+ initiate_descriptor_downloads(source, purpose,
downloadable, i, i+n_per_request,
pds_flags);
}
@@ -4514,7 +4576,8 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters,
n_would_reject, n_wouldnt_use, n_inprogress);
- launch_router_descriptor_downloads(downloadable, source, now);
+ launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC,
+ downloadable, source, now);
digestmap_free(map, NULL);
done:
diff --git a/src/or/routerlist.h b/src/or/routerlist.h
index 574bce7..804be2a 100644
--- a/src/or/routerlist.h
+++ b/src/or/routerlist.h
@@ -192,5 +192,11 @@ int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
int hid_serv_acting_as_directory(void);
int hid_serv_responsible_for_desc_id(const char *id);
+void list_pending_microdesc_downloads(digestmap_t *result);
+void launch_descriptor_downloads(int purpose,
+ smartlist_t *downloadable,
+ routerstatus_t *source,
+ time_t now);
+
#endif
--
1.7.1
More information about the tor-commits
mailing list