[or-cvs] [tor/master 2/6] Code to download, parse, and store microdesc consensuses
nickm at torproject.org
nickm at torproject.org
Tue Sep 28 04:44:15 UTC 2010
Author: Nick Mathewson <nickm at torproject.org>
Date: Thu, 16 Sep 2010 22:12:03 -0400
Subject: Code to download, parse, and store microdesc consensuses
Commit: 4e0f7f4ffc1a0c624b8d4b3e23bd962fc488a4f7
---
changes/microdesc_dl | 4 +
src/or/directory.c | 61 ++++++++++++------
src/or/dirserv.c | 3 +-
src/or/networkstatus.c | 163 +++++++++++++++++++++++++++++++++++------------
src/or/networkstatus.h | 5 +-
5 files changed, 173 insertions(+), 63 deletions(-)
create mode 100644 changes/microdesc_dl
diff --git a/changes/microdesc_dl b/changes/microdesc_dl
new file mode 100644
index 0000000..aca634c
--- /dev/null
+++ b/changes/microdesc_dl
@@ -0,0 +1,4 @@
+ o Major features:
+ - Caches now download and cache all the consensus flavors that
+ they know about. This allows them to assess which microdescriptors
+ they need to fetch.
diff --git a/src/or/directory.c b/src/or/directory.c
index ac6f205..4411278 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -361,9 +361,24 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
}
if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
- networkstatus_t *v = networkstatus_get_latest_consensus();
- if (v)
- if_modified_since = v->valid_after + 180;
+ int flav = FLAV_NS;
+ networkstatus_t *v;
+ if (resource)
+ flav = networkstatus_parse_flavor_name(resource);
+
+ if (flav != -1) {
+ /* IF we have a parsed consensus of this type, we can do an
+ * if-modified-time based on it. */
+ v = networkstatus_get_latest_consensus_by_flavor(flav);
+ if (v)
+ if_modified_since = v->valid_after + 180;
+ } else {
+ /* Otherwise it might be a consensus we don't parse, but which we
+ * do cache. Look at the cached copy, perhaps. */
+ cached_dir_t *cd = dirserv_get_consensus(resource ? resource : "ns");
+ if (cd)
+ if_modified_since = cd->published + 180;
+ }
}
if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
@@ -598,7 +613,9 @@ connection_dir_request_failed(dir_connection_t *conn)
connection_dir_bridge_routerdesc_failed(conn);
connection_dir_download_routerdesc_failed(conn);
} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
- networkstatus_consensus_download_failed(0);
+ const char *flavname =
+ conn->requested_resource ? conn->requested_resource : "ns";
+ networkstatus_consensus_download_failed(0, flavname);
} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
conn->_base.address);
@@ -955,12 +972,16 @@ _compare_strs(const void **a, const void **b)
* This url depends on whether or not the server we go to
* is sufficiently new to support conditional consensus downloading,
* i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
+ *
+ * If 'resource' is provided, it is the name of a consensus flavor to request.
*/
static char *
-directory_get_consensus_url(int supports_conditional_consensus)
+directory_get_consensus_url(int supports_conditional_consensus,
+ const char *resource)
{
- char *url;
- size_t len;
+ char *url = NULL;
+ const char *hyphen = resource ? "-" : "";
+ const char *flavor = resource ? resource : "";
if (supports_conditional_consensus) {
char *authority_id_list;
@@ -982,16 +1003,15 @@ directory_get_consensus_url(int supports_conditional_consensus)
authority_id_list = smartlist_join_strings(authority_digests,
"+", 0, NULL);
- len = strlen(authority_id_list)+64;
- url = tor_malloc(len);
- tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
- authority_id_list);
+ tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
+ hyphen, flavor, authority_id_list);
SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
smartlist_free(authority_digests);
tor_free(authority_id_list);
} else {
- url = tor_strdup("/tor/status-vote/current/consensus.z");
+ tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z",
+ hyphen, flavor);
}
return url;
}
@@ -1072,10 +1092,11 @@ directory_send_command(dir_connection_t *conn,
tor_snprintf(url, len, "/tor/status/%s", resource);
break;
case DIR_PURPOSE_FETCH_CONSENSUS:
- tor_assert(!resource);
+ /* resource is optional. If present, it's a flavor name */
tor_assert(!payload);
httpcommand = "GET";
- url = directory_get_consensus_url(supports_conditional_consensus);
+ url = directory_get_consensus_url(supports_conditional_consensus,
+ resource);
log_info(LD_DIR, "Downloading consensus from %s using %s",
hoststring, url);
break;
@@ -1690,6 +1711,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
int r;
+ const char *flavname =
+ conn->requested_resource ? conn->requested_resource : "ns";
if (status_code != 200) {
int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
log(severity, LD_DIR,
@@ -1698,18 +1721,18 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
status_code, escaped(reason), conn->_base.address,
conn->_base.port);
tor_free(body); tor_free(headers); tor_free(reason);
- networkstatus_consensus_download_failed(status_code);
+ networkstatus_consensus_download_failed(status_code, flavname);
return -1;
}
log_info(LD_DIR,"Received consensus directory (size %d) from server "
"'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
- if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
+ if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) {
log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
- "Unable to load consensus directory downloaded from "
+ "Unable to load %s consensus directory downloaded from "
"server '%s:%d'. I'll try again soon.",
- conn->_base.address, conn->_base.port);
+ flavname, conn->_base.address, conn->_base.port);
tor_free(body); tor_free(headers); tor_free(reason);
- networkstatus_consensus_download_failed(0);
+ networkstatus_consensus_download_failed(0, flavname);
return -1;
}
/* launches router downloads as needed */
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index 1f7722f..75e3e86 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -1263,7 +1263,8 @@ static cached_dir_t cached_runningrouters;
* cached_dir_t. */
static digestmap_t *cached_v2_networkstatus = NULL;
-/** Map from flavor name to the v3 consensuses that we're currently serving. */
+/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
+ * currently serving. */
static strmap_t *cached_consensuses = NULL;
/** Possibly replace the contents of <b>d</b> with the value of
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index 1d8a20b..8b2a102 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -44,8 +44,19 @@ static strmap_t *named_server_map = NULL;
* as unnamed for some server in the consensus. */
static strmap_t *unnamed_server_map = NULL;
-/** Most recently received and validated v3 consensus network status. */
-static networkstatus_t *current_consensus = NULL;
+/** Most recently received and validated v3 consensus network status,
+ * of whichever type we are using for our own circuits. This will be the same
+ * as one of current_ns_consensus or current_md_consensus.
+ */
+#define current_consensus current_ns_consensus
+
+/** Most recently received and validated v3 "ns"-flavored consensus network
+ * status. */
+static networkstatus_t *current_ns_consensus = NULL;
+
+/** Most recently received and validated v3 "microdec"-flavored consensus
+ * network status. */
+static networkstatus_t *current_md_consensus = NULL;
/** A v3 consensus networkstatus that we've received, but which we don't
* have enough certificates to be happy about. */
@@ -1199,6 +1210,24 @@ update_v2_networkstatus_cache_downloads(time_t now)
}
}
+static int
+we_want_to_fetch_flavor(or_options_t *options, int flavor)
+{
+ if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
+ /* This flavor is crazy; we don't want it */
+ /*XXXX handle unrecognized flavors later */
+ return 0;
+ }
+ if (authdir_mode_v3(options) || directory_caches_dir_info(options)) {
+ /* We want to serve all flavors to others, regardless if we would use
+ * it ourselves. */
+ return 1;
+ }
+ /* Otherwise, we want the flavor only if we want to use it to build
+ * circuits. */
+ return (flavor == USABLE_CONSENSUS_FLAVOR);
+}
+
/** How many times will we try to fetch a consensus before we give up? */
#define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8
/** How long will we hang onto a possibly live consensus for which we're
@@ -1211,48 +1240,65 @@ static void
update_consensus_networkstatus_downloads(time_t now)
{
int i;
+ or_options_t *options = get_options();
+
if (!networkstatus_get_live_consensus(now))
time_to_download_next_consensus = now; /* No live consensus? Get one now!*/
if (time_to_download_next_consensus > now)
return; /* Wait until the current consensus is older. */
- /* XXXXNM Microdescs: may need to download more types. */
- if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now,
- CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
- return; /* We failed downloading a consensus too recently. */
- if (connection_get_by_type_purpose(CONN_TYPE_DIR,
- DIR_PURPOSE_FETCH_CONSENSUS))
- return; /* There's an in-progress download.*/
for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
- consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+ /* XXXX need some way to download unknown flavors if we are caching. */
+ const char *resource;
+ consensus_waiting_for_certs_t *waiting;
+
+ if (! we_want_to_fetch_flavor(options, i))
+ continue;
+
+ resource = i==FLAV_NS ? NULL : networkstatus_get_flavor_name(i);
+
+ if (!download_status_is_ready(&consensus_dl_status[i], now,
+ CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
+ continue; /* We failed downloading a consensus too recently. */
+ if (connection_dir_get_by_purpose_and_resource(
+ DIR_PURPOSE_FETCH_CONSENSUS, resource))
+ continue; /* There's an in-progress download.*/
+
+ waiting = &consensus_waiting_for_certs[i];
if (waiting->consensus) {
/* XXXX make sure this doesn't delay sane downloads. */
- if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now)
- return; /* We're still getting certs for this one. */
- else {
+ if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) {
+ continue; /* We're still getting certs for this one. */
+ } else {
if (!waiting->dl_failed) {
- download_status_failed(&consensus_dl_status[FLAV_NS], 0);
+ download_status_failed(&consensus_dl_status[i], 0);
waiting->dl_failed=1;
}
}
}
- }
- log_info(LD_DIR, "Launching networkstatus consensus download.");
- directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
- ROUTER_PURPOSE_GENERAL, NULL,
- PDS_RETRY_IF_NO_SERVERS);
+ log_info(LD_DIR, "Launching %s networkstatus consensus download.",
+ networkstatus_get_flavor_name(i));
+
+ directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+ ROUTER_PURPOSE_GENERAL, resource,
+ PDS_RETRY_IF_NO_SERVERS);
+ }
}
/** Called when an attempt to download a consensus fails: note that the
* failure occurred, and possibly retry. */
void
-networkstatus_consensus_download_failed(int status_code)
+networkstatus_consensus_download_failed(int status_code, const char *flavname)
{
- /* XXXXNM Microdescs: may need to handle more types. */
- download_status_failed(&consensus_dl_status[FLAV_NS], status_code);
- /* Retry immediately, if appropriate. */
- update_consensus_networkstatus_downloads(time(NULL));
+ int flav = networkstatus_parse_flavor_name(flavname);
+ if (flav >= 0) {
+ tor_assert(flav < N_CONSENSUS_FLAVORS);
+ /* XXXX handle unrecognized flavors */
+ download_status_failed(&consensus_dl_status[flav], status_code);
+ /* Retry immediately, if appropriate. */
+ update_consensus_networkstatus_downloads(time(NULL));
+ }
}
/** How long do we (as a cache) wait after a consensus becomes non-fresh
@@ -1373,7 +1419,10 @@ update_certificate_downloads(time_t now)
now);
}
- authority_certs_fetch_missing(current_consensus, now);
+ if (current_ns_consensus)
+ authority_certs_fetch_missing(current_ns_consensus, now);
+ if (current_ns_consensus)
+ authority_certs_fetch_missing(current_md_consensus, now);
}
/** Return 1 if we have a consensus but we don't have enough certificates
@@ -1405,6 +1454,18 @@ networkstatus_get_latest_consensus(void)
return current_consensus;
}
+/** DOCDOC */
+networkstatus_t *
+networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+ if (f == FLAV_NS)
+ return current_ns_consensus;
+ else if (f == FLAV_MICRODESC)
+ return current_md_consensus;
+ else
+ tor_assert(0);
+}
+
/** Return the most recent consensus that we have downloaded, or NULL if it is
* no longer live. */
networkstatus_t *
@@ -1569,6 +1630,7 @@ networkstatus_set_current_consensus(const char *consensus,
const digests_t *current_digests = NULL;
consensus_waiting_for_certs_t *waiting = NULL;
time_t current_valid_after = 0;
+ int free_consensus = 1;
if (flav < 0) {
/* XXXX we don't handle unrecognized flavors yet. */
@@ -1614,9 +1676,16 @@ networkstatus_set_current_consensus(const char *consensus,
if (!strcmp(flavor, "ns")) {
consensus_fname = get_datadir_fname("cached-consensus");
unverified_fname = get_datadir_fname("unverified-consensus");
- if (current_consensus) {
- current_digests = ¤t_consensus->digests;
- current_valid_after = current_consensus->valid_after;
+ if (current_ns_consensus) {
+ current_digests = ¤t_ns_consensus->digests;
+ current_valid_after = current_ns_consensus->valid_after;
+ }
+ } else if (!strcmp(flavor, "microdesc")) {
+ consensus_fname = get_datadir_fname("cached-microdesc-consensus");
+ unverified_fname = get_datadir_fname("unverified-microdesc-consensus");
+ if (current_md_consensus) {
+ current_digests = ¤t_md_consensus->digests;
+ current_valid_after = current_md_consensus->valid_after;
}
} else {
cached_dir_t *cur;
@@ -1702,11 +1771,21 @@ networkstatus_set_current_consensus(const char *consensus,
if (flav == USABLE_CONSENSUS_FLAVOR) {
notify_control_networkstatus_changed(current_consensus, c);
-
- if (current_consensus) {
- networkstatus_copy_old_consensus_info(c, current_consensus);
- networkstatus_vote_free(current_consensus);
+ }
+ if (flav == FLAV_NS) {
+ if (current_ns_consensus) {
+ networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+ networkstatus_vote_free(current_ns_consensus);
}
+ current_ns_consensus = c;
+ free_consensus = 0; /* avoid free */
+ } else if (flav == FLAV_MICRODESC) {
+ if (current_md_consensus) {
+ networkstatus_copy_old_consensus_info(c, current_md_consensus);
+ networkstatus_vote_free(current_md_consensus);
+ }
+ current_md_consensus = c;
+ free_consensus = 0; /* avoid free */
}
waiting = &consensus_waiting_for_certs[flav];
@@ -1739,11 +1818,9 @@ networkstatus_set_current_consensus(const char *consensus,
}
if (flav == USABLE_CONSENSUS_FLAVOR) {
- current_consensus = c;
- c = NULL; /* Prevent free. */
-
/* XXXXNM Microdescs: needs a non-ns variant. */
update_consensus_networkstatus_fetch_time(now);
+
dirvote_recalculate_timing(options, now);
routerstatus_list_update_named_server_map();
cell_ewma_set_scale_factor(options, current_consensus);
@@ -1758,11 +1835,11 @@ networkstatus_set_current_consensus(const char *consensus,
write_str_to_file(consensus_fname, consensus, 0);
}
- if (ftime_definitely_before(now, current_consensus->valid_after)) {
+ if (ftime_definitely_before(now, c->valid_after)) {
char tbuf[ISO_TIME_LEN+1];
char dbuf[64];
- long delta = now - current_consensus->valid_after;
- format_iso_time(tbuf, current_consensus->valid_after);
+ long delta = now - c->valid_after;
+ format_iso_time(tbuf, c->valid_after);
format_time_interval(dbuf, sizeof(dbuf), delta);
log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
"consensus network status document (%s GMT). Tor needs an "
@@ -1776,7 +1853,8 @@ networkstatus_set_current_consensus(const char *consensus,
result = 0;
done:
- networkstatus_vote_free(c);
+ if (free_consensus)
+ networkstatus_vote_free(c);
tor_free(consensus_fname);
tor_free(unverified_fname);
return result;
@@ -2028,7 +2106,7 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
void
signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
{
- networkstatus_t *ns = current_consensus;
+ networkstatus_t *ns = current_ns_consensus;
if (!ns)
return;
@@ -2264,8 +2342,9 @@ networkstatus_free_all(void)
digestmap_free(v2_download_status_map, _tor_free);
v2_download_status_map = NULL;
- networkstatus_vote_free(current_consensus);
- current_consensus = NULL;
+ networkstatus_vote_free(current_ns_consensus);
+ networkstatus_vote_free(current_md_consensus);
+ current_md_consensus = current_ns_consensus = NULL;
for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h
index 32b71a9..ad91e9d 100644
--- a/src/or/networkstatus.h
+++ b/src/or/networkstatus.h
@@ -53,7 +53,8 @@ routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname,
int warn_if_unnamed);
const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
int networkstatus_nickname_is_unnamed(const char *nickname);
-void networkstatus_consensus_download_failed(int status_code);
+void networkstatus_consensus_download_failed(int status_code,
+ const char *flavname);
void update_consensus_networkstatus_fetch_time(time_t now);
int should_delay_dir_fetches(or_options_t *options);
void update_networkstatus_downloads(time_t now);
@@ -61,6 +62,8 @@ void update_certificate_downloads(time_t now);
int consensus_is_waiting_for_certs(void);
networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest);
networkstatus_t *networkstatus_get_latest_consensus(void);
+networkstatus_t *networkstatus_get_latest_consensus_by_flavor(
+ consensus_flavor_t f);
networkstatus_t *networkstatus_get_live_consensus(time_t now);
networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now);
#define NSSET_FROM_CACHE 1
--
1.7.1
More information about the tor-commits
mailing list