[or-cvs] [tor/master] Code to generate, store, and parse microdescriptors and consensuses.
Nick Mathewson
nickm at seul.org
Mon Oct 19 04:48:28 UTC 2009
Author: Nick Mathewson <nickm at torproject.org>
Date: Mon, 24 Aug 2009 12:51:33 -0400
Subject: Code to generate, store, and parse microdescriptors and consensuses.
Commit: e1ddee8bbe724e934fe9a4cb2d290719a7d6105c
The consensus documents are not signed properly, not served, and not
exchanged yet.
---
src/common/util.c | 7 ++
src/common/util.h | 1 +
src/or/Makefile.am | 1 +
src/or/dirserv.c | 35 ++++++-
src/or/dirvote.c | 200 +++++++++++++++++++++++++-----------
src/or/microdesc.c | 267 ++++++++++++++++++++++++++++++++++++++++++++++++
src/or/networkstatus.c | 6 +
src/or/or.h | 73 +++++++++++--
src/or/routerparse.c | 206 ++++++++++++++++++++++++++++++++++---
src/test/test_dir.c | 9 +-
10 files changed, 712 insertions(+), 93 deletions(-)
create mode 100644 src/or/microdesc.c
diff --git a/src/common/util.c b/src/common/util.c
index d05c308..139c1aa 100644
--- a/src/common/util.c
+++ b/src/common/util.c
@@ -684,6 +684,13 @@ tor_digest_is_zero(const char *digest)
return tor_mem_is_zero(digest, DIGEST_LEN);
}
+/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
+int
+tor_digest256_is_zero(const char *digest)
+{
+ return tor_mem_is_zero(digest, DIGEST256_LEN);
+}
+
/* Helper: common code to check whether the result of a strtol or strtoul or
* strtoll is correct. */
#define CHECK_STRTOX_RESULT() \
diff --git a/src/common/util.h b/src/common/util.h
index 28ea8a0..85234f5 100644
--- a/src/common/util.h
+++ b/src/common/util.h
@@ -195,6 +195,7 @@ const char *find_whitespace(const char *s) ATTR_PURE;
const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE;
int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE;
int tor_digest_is_zero(const char *digest) ATTR_PURE;
+int tor_digest256_is_zero(const char *digest) ATTR_PURE;
char *esc_for_log(const char *string) ATTR_MALLOC;
const char *escaped(const char *string);
struct smartlist_t;
diff --git a/src/or/Makefile.am b/src/or/Makefile.am
index 097e3e2..3dc1889 100644
--- a/src/or/Makefile.am
+++ b/src/or/Makefile.am
@@ -20,6 +20,7 @@ libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \
connection.c connection_edge.c connection_or.c control.c \
cpuworker.c directory.c dirserv.c dirvote.c \
dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \
+ microdesc.c \
networkstatus.c onion.c policies.c \
reasons.c relay.c rendcommon.c rendclient.c rendmid.c \
rendservice.c rephist.c router.c routerlist.c routerparse.c \
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index f12ef2f..e7a58ed 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -1901,10 +1901,11 @@ routerstatus_format_entry(char *buf, size_t buf_len,
tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr));
r = tor_snprintf(buf, buf_len,
- "r %s %s %s %s %s %d %d\n",
+ "r %s %s %s%s%s %s %d %d\n",
rs->nickname,
identity64,
- digest64,
+ (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
+ (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
published,
ipaddr,
(int)rs->or_port,
@@ -1918,7 +1919,7 @@ routerstatus_format_entry(char *buf, size_t buf_len,
* this here, instead of in the caller. Then we could use the
* networkstatus_type_t values, with an additional control port value
* added -MP */
- if (format == NS_V3_CONSENSUS)
+ if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
return 0;
cp = buf + strlen(buf);
@@ -2434,6 +2435,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
vote_timing_t timing;
digestmap_t *omit_as_sybil = NULL;
const int vote_on_reachability = running_long_enough_to_decide_unreachable();
+ smartlist_t *microdescriptors = NULL;
tor_assert(private_key);
tor_assert(cert);
@@ -2482,11 +2484,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
omit_as_sybil = get_possible_sybil_list(routers);
routerstatuses = smartlist_create();
+ microdescriptors = smartlist_create();
- SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+ SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
if (ri->cache_info.published_on >= cutoff) {
routerstatus_t *rs;
vote_routerstatus_t *vrs;
+ microdesc_t *md;
vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
rs = &vrs->status;
@@ -2501,9 +2505,30 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
rs->is_running = 0;
vrs->version = version_from_platform(ri->platform);
+ md = dirvote_create_microdescriptor(ri);
+ if (md) {
+ char buf[128];
+ vote_microdesc_hash_t *h;
+ dirvote_format_microdesc_vote_line(buf, sizeof(buf), md);
+ h = tor_malloc(sizeof(vote_microdesc_hash_t));
+ h->microdesc_hash_line = tor_strdup(buf);
+ h->next = NULL;
+ vrs->microdesc = h;
+ md->last_listed = now;
+ smartlist_add(microdescriptors, md);
+ }
+
smartlist_add(routerstatuses, vrs);
}
- });
+ } SMARTLIST_FOREACH_END(ri);
+
+ {
+ smartlist_t *added =
+ microdescs_add_list_to_cache(get_microdesc_cache(),
+ microdescriptors, SAVED_NOWHERE, 0);
+ smartlist_free(added);
+ smartlist_free(microdescriptors);
+ }
smartlist_free(routers);
digestmap_free(omit_as_sybil, NULL);
diff --git a/src/or/dirvote.c b/src/or/dirvote.c
index f0f92d2..d200344 100644
--- a/src/or/dirvote.c
+++ b/src/or/dirvote.c
@@ -24,14 +24,20 @@ static int dirvote_publish_consensus(void);
static char *make_consensus_method_list(int low, int high, const char *sep);
/** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 7
+#define MAX_SUPPORTED_CONSENSUS_METHOD 8
#define MIN_METHOD_FOR_PARAMS 7
+/** Lowest consensus method that generates microdescriptors */
+#define MIN_METHOD_FOR_MICRODESC 8
+
/* =====
* Voting
* =====*/
+/* Overestimated. */
+#define MICRODESC_LINE_LEN 80
+
/** Return a new string containing the string representation of the vote in
* <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>.
* For v3 authorities. */
@@ -89,7 +95,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
len = 8192;
len += strlen(version_lines);
- len += (RS_ENTRY_LEN)*smartlist_len(rl->routers);
+ len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers);
len += v3_ns->cert->cache_info.signed_descriptor_len;
status = tor_malloc(len);
@@ -158,15 +164,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
outp += cert->cache_info.signed_descriptor_len;
}
- SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs,
- {
+ SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *,
+ vrs) {
+ vote_microdesc_hash_t *h;
if (routerstatus_format_entry(outp, endp-outp, &vrs->status,
vrs->version, NS_V3_VOTE) < 0) {
log_warn(LD_BUG, "Unable to print router status.");
goto err;
}
outp += strlen(outp);
- });
+
+ for (h = vrs->microdesc; h; h = h->next) {
+ size_t mlen = strlen(h->microdesc_hash_line);
+ if (outp+mlen >= endp) {
+ log_warn(LD_BUG, "Can't fit microdesc line in vote.");
+ }
+ memcpy(outp, h->microdesc_hash_line, mlen+1);
+ outp += strlen(outp);
+ }
+ } SMARTLIST_FOREACH_END(vrs);
{
char signing_key_fingerprint[FINGERPRINT_LEN+1];
@@ -189,7 +205,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
outp += strlen(outp);
}
- if (router_get_networkstatus_v3_hash(status, digest)<0)
+ if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0)
goto err;
note_crypto_pk_op(SIGN_DIR);
if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN,
@@ -294,34 +310,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min)
/** Given a sorted list of strings <b>lst</b>, return the member that appears
* most. Break ties in favor of later-occurring members. */
-static const char *
-get_most_frequent_member(smartlist_t *lst)
-{
- const char *most_frequent = NULL;
- int most_frequent_count = 0;
-
- const char *cur = NULL;
- int count = 0;
-
- SMARTLIST_FOREACH(lst, const char *, s,
- {
- if (cur && !strcmp(s, cur)) {
- ++count;
- } else {
- if (count >= most_frequent_count) {
- most_frequent = cur;
- most_frequent_count = count;
- }
- cur = s;
- count = 1;
- }
- });
- if (count >= most_frequent_count) {
- most_frequent = cur;
- most_frequent_count = count;
- }
- return most_frequent;
-}
+#define get_most_frequent_member(lst) \
+ smartlist_get_most_frequent_string(lst)
/** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses
* that come from the same routerinfo, with the same derived elements.
@@ -363,7 +353,8 @@ _compare_vote_rs(const void **_a, const void **_b)
* in favor of smaller descriptor digest.
*/
static vote_routerstatus_t *
-compute_routerstatus_consensus(smartlist_t *votes)
+compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
+ char *microdesc_digest256_out)
{
vote_routerstatus_t *most = NULL, *cur = NULL;
int most_n = 0, cur_n = 0;
@@ -399,6 +390,26 @@ compute_routerstatus_consensus(smartlist_t *votes)
}
tor_assert(most);
+
+ if (consensus_method >= MIN_METHOD_FOR_MICRODESC &&
+ microdesc_digest256_out) {
+ smartlist_t *digests = smartlist_create();
+ const char *best_microdesc_digest;
+ SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, {
+ char d[DIGEST256_LEN];
+ if (compare_vote_rs(rs, most))
+ continue;
+ if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method))
+ smartlist_add(digests, tor_memdup(d, sizeof(d)));
+ });
+ smartlist_sort_digests256(digests);
+ best_microdesc_digest = smartlist_get_most_frequent_digest256(digests);
+ if (best_microdesc_digest)
+ memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN);
+ SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp));
+ smartlist_free(digests);
+ }
+
return most;
}
@@ -615,16 +626,20 @@ networkstatus_compute_consensus(smartlist_t *votes,
crypto_pk_env_t *identity_key,
crypto_pk_env_t *signing_key,
const char *legacy_id_key_digest,
- crypto_pk_env_t *legacy_signing_key)
+ crypto_pk_env_t *legacy_signing_key,
+ consensus_flavor_t flavor)
{
smartlist_t *chunks;
char *result = NULL;
int consensus_method;
-
time_t valid_after, fresh_until, valid_until;
int vote_seconds, dist_seconds;
char *client_versions = NULL, *server_versions = NULL;
smartlist_t *flags;
+ const routerstatus_format_type_t rs_format =
+ flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
+
+ tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC);
tor_assert(total_authorities >= smartlist_len(votes));
if (!smartlist_len(votes)) {
@@ -955,6 +970,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
int n_listing = 0;
int i;
char buf[256];
+ char microdesc_digest[DIGEST256_LEN];
/* Of the next-to-be-considered digest in each voter, which is first? */
SMARTLIST_FOREACH(votes, networkstatus_t *, v, {
@@ -1021,7 +1037,9 @@ networkstatus_compute_consensus(smartlist_t *votes,
/* Figure out the most popular opinion of what the most recent
* routerinfo and its contents are. */
- rs = compute_routerstatus_consensus(matching_descs);
+ memset(microdesc_digest, 0, sizeof(microdesc_digest));
+ rs = compute_routerstatus_consensus(matching_descs, consensus_method,
+ microdesc_digest);
/* Copy bits of that into rs_out. */
tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN));
memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN);
@@ -1182,9 +1200,19 @@ networkstatus_compute_consensus(smartlist_t *votes,
/* Okay!! Now we can write the descriptor... */
/* First line goes into "buf". */
routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL,
- NS_V3_CONSENSUS);
+ rs_format);
smartlist_add(chunks, tor_strdup(buf));
- /* Second line is all flags. The "\n" is missing. */
+ /* Now an m line, if applicable. */
+ if (flavor == FLAV_MICRODESC &&
+ !tor_digest256_is_zero(microdesc_digest)) {
+ char m[BASE64_DIGEST256_LEN+1], *cp;
+ const size_t mlen = BASE64_DIGEST256_LEN+5;
+ digest256_to_base64(m, microdesc_digest);
+ cp = tor_malloc(mlen);
+ tor_snprintf(cp, mlen, "m %s\n", m);
+ smartlist_add(chunks, cp);
+ }
+ /* Next line is all flags. The "\n" is missing. */
smartlist_add(chunks,
smartlist_join_strings(chosen_flags, " ", 0, NULL));
/* Now the version line. */
@@ -1206,7 +1234,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
};
/* Now the exitpolicy summary line. */
- if (rs_out.has_exitsummary) {
+ if (rs_out.has_exitsummary && flavor == FLAV_NS) {
char buf[MAX_POLICY_LINE_LEN+1];
int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary);
if (r<0) {
@@ -2090,7 +2118,8 @@ dirvote_compute_consensus(void)
consensus_body = networkstatus_compute_consensus(
votes, n_voters,
my_cert->identity_key,
- get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign);
+ get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign,
+ FLAV_NS);
}
if (!consensus_body) {
log_warn(LD_DIR, "Couldn't generate a consensus at all!");
@@ -2389,14 +2418,21 @@ dirvote_get_vote(const char *fp, int flags)
return NULL;
}
-int
-dirvote_create_microdescriptor(char *out, size_t outlen,
- const routerinfo_t *ri)
+/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>.
+ *
+ * XXX Right now, there is only one way to generate microdescriptors from
+ * router descriptors. This may change in future consensus methods. If so,
+ * we'll need an internal way to remember which method we used, and ask for a
+ * particular method.
+ **/
+microdesc_t *
+dirvote_create_microdescriptor(const routerinfo_t *ri)
{
+ microdesc_t *result = NULL;
char *key = NULL, *summary = NULL, *family = NULL;
+ char buf[1024];
size_t keylen;
- int result = -1;
- char *start = out, *end = out+outlen;
+ char *out = buf, *end = buf+sizeof(buf);
if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0)
goto done;
@@ -2417,7 +2453,19 @@ dirvote_create_microdescriptor(char *out, size_t outlen,
goto done;
out += strlen(out);
}
- result = out - start;
+ *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */
+
+ {
+ smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1);
+ if (smartlist_len(lst) != 1) {
+ log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse.");
+ SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md));
+ smartlist_free(lst);
+ goto done;
+ }
+ result = smartlist_get(lst, 0);
+ smartlist_free(lst);
+ }
done:
tor_free(key);
@@ -2426,28 +2474,23 @@ dirvote_create_microdescriptor(char *out, size_t outlen,
return result;
}
-/** Lowest consensus method that generates microdescriptors */
-#define MIN_CM_MICRODESC 7
/** Cached space-separated string to hold */
static char *microdesc_consensus_methods = NULL;
+/** DOCDOC */
int
-dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
- const char *microdesc,
- size_t microdescriptor_len)
+dirvote_format_microdesc_vote_line(char *out, size_t out_len,
+ const microdesc_t *md)
{
- char d[DIGEST256_LEN];
char d64[BASE64_DIGEST256_LEN];
if (!microdesc_consensus_methods) {
microdesc_consensus_methods =
- make_consensus_method_list(MIN_CM_MICRODESC,
+ make_consensus_method_list(MIN_METHOD_FOR_MICRODESC,
MAX_SUPPORTED_CONSENSUS_METHOD,
",");
tor_assert(microdesc_consensus_methods);
}
- if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0)
- return -1;
- if (digest256_to_base64(d64, d)<0)
+ if (digest256_to_base64(d64, md->digest)<0)
return -1;
if (tor_snprintf(out, out_len, "m %s sha256=%s\n",
@@ -2457,3 +2500,44 @@ dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
return strlen(out);
}
+/** DOCDOC */
+int
+vote_routerstatus_find_microdesc_hash(char *digest256_out,
+ const vote_routerstatus_t *vrs,
+ int method)
+{
+ /* XXXX only returns the sha256 method. */
+ const vote_microdesc_hash_t *h;
+ char mstr[64];
+ size_t mlen;
+
+ tor_snprintf(mstr, sizeof(mstr), "%d", method);
+ mlen = strlen(mstr);
+
+ for (h = vrs->microdesc; h; h = h->next) {
+ const char *cp = h->microdesc_hash_line;
+ size_t num_len;
+ /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in
+ * the first part. */
+ while (1) {
+ num_len = strspn(cp, "1234567890");
+ if (num_len == mlen && !memcmp(mstr, cp, mlen)) {
+ /* This is the line. */
+ char buf[BASE64_DIGEST256_LEN+1];
+ /* XXXX ignores extraneous stuff if the digest is too long. This
+ * seems harmless enough, right? */
+ cp = strstr(cp, " sha256=");
+ if (!cp)
+ return -1;
+ cp += strlen(" sha256=");
+ strlcpy(buf, cp, sizeof(buf));
+ return digest256_from_base64(digest256_out, buf);
+ }
+ if (num_len == 0 || cp[num_len] != ',')
+ break;
+ cp += num_len + 1;
+ }
+ }
+ return -1;
+}
+
diff --git a/src/or/microdesc.c b/src/or/microdesc.c
new file mode 100644
index 0000000..0128fbb
--- /dev/null
+++ b/src/or/microdesc.c
@@ -0,0 +1,267 @@
+/* Copyright (c) 2009, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+/** DOCDOC everything here. */
+
+#define MICRODESC_IN_CONSENSUS 1
+#define MICRODESC_IN_VOTE 2
+
+struct microdesc_cache_t {
+ HT_HEAD(microdesc_map, microdesc_t) map;
+
+ char *cache_fname;
+ char *journal_fname;
+ tor_mmap_t *cache_content;
+ size_t journal_len;
+};
+
+static INLINE unsigned int
+_microdesc_hash(microdesc_t *md)
+{
+ unsigned *d = (unsigned*)md->digest;
+#if SIZEOF_INT == 4
+ return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7];
+#else
+ return d[0] ^ d[1] ^ d[2] ^ d[3];
+#endif
+}
+
+static INLINE int
+_microdesc_eq(microdesc_t *a, microdesc_t *b)
+{
+ return !memcmp(a->digest, b->digest, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(microdesc_map, microdesc_t, node,
+ _microdesc_hash, _microdesc_eq);
+HT_GENERATE(microdesc_map, microdesc_t, node,
+ _microdesc_hash, _microdesc_eq, 0.6,
+ _tor_malloc, _tor_realloc, _tor_free);
+
+static int
+dump_microdescriptor(FILE *f, microdesc_t *md)
+{
+ /* XXXX drops unkown annotations. */
+ if (md->last_listed) {
+ char buf[ISO_TIME_LEN+1];
+ format_iso_time(buf, md->last_listed);
+ fprintf(f, "@last-listed %s\n", buf);
+ }
+
+ md->off = (off_t) ftell(f);
+ fwrite(md->body, 1, md->bodylen, f);
+ return 0;
+}
+
+static microdesc_cache_t *the_microdesc_cache = NULL;
+
+microdesc_cache_t *
+get_microdesc_cache(void)
+{
+ if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
+ microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t));
+ HT_INIT(microdesc_map, &cache->map);
+ cache->cache_fname = get_datadir_fname("cached-microdescs");
+ cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+ microdesc_cache_reload(cache);
+ the_microdesc_cache = cache;
+ }
+ return the_microdesc_cache;
+}
+
+/* There are three sources of microdescriptors:
+ 1) Generated us while acting as a directory authority.
+ 2) Loaded from the cache on disk.
+ 3) Downloaded.
+*/
+
+/* Returns list of added microdesc_t. */
+smartlist_t *
+microdescs_add_to_cache(microdesc_cache_t *cache,
+ const char *s, const char *eos, saved_location_t where,
+ int no_save)
+{
+ /*XXXX need an argument that sets last_listed as appropriate. */
+
+ smartlist_t *descriptors, *added;
+ const int allow_annotations = (where != SAVED_NOWHERE);
+ const int copy_body = (where != SAVED_IN_CACHE);
+
+ descriptors = microdescs_parse_from_string(s, eos,
+ allow_annotations,
+ copy_body);
+
+ added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
+ smartlist_free(descriptors);
+ return added;
+}
+
+/* Returns list of added microdesc_t. Frees any not added. */
+smartlist_t *
+microdescs_add_list_to_cache(microdesc_cache_t *cache,
+ smartlist_t *descriptors, saved_location_t where,
+ int no_save)
+{
+ smartlist_t *added;
+ open_file_t *open_file = NULL;
+ FILE *f = NULL;
+ // int n_added = 0;
+
+ if (where == SAVED_NOWHERE && !no_save) {
+ f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND,
+ 0600, &open_file);
+ if (!f)
+ log_warn(LD_DIR, "Couldn't append to journal in %s",
+ cache->journal_fname);
+ }
+
+ added = smartlist_create();
+ SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+ microdesc_t *md2;
+ md2 = HT_FIND(microdesc_map, &cache->map, md);
+ if (md2) {
+ /* We already had this one. */
+ if (md2->last_listed < md->last_listed)
+ md2->last_listed = md->last_listed;
+ microdesc_free(md);
+ continue;
+ }
+
+ /* Okay, it's a new one. */
+ if (f) {
+ dump_microdescriptor(f, md);
+ md->saved_location = SAVED_IN_JOURNAL;
+ } else {
+ md->saved_location = where;
+ }
+
+ md->no_save = no_save;
+
+ HT_INSERT(microdesc_map, &cache->map, md);
+ smartlist_add(added, md);
+ } SMARTLIST_FOREACH_END(md);
+
+ finish_writing_to_file(open_file); /*XXX Check me.*/
+ return added;
+}
+
+void
+microdesc_cache_clear(microdesc_cache_t *cache)
+{
+ microdesc_t **entry, **next;
+ for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
+ next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
+ microdesc_free(*entry);
+ }
+ if (cache->cache_content) {
+ tor_munmap_file(cache->cache_content);
+ cache->cache_content = NULL;
+ }
+}
+
+int
+microdesc_cache_reload(microdesc_cache_t *cache)
+{
+ struct stat st;
+ char *journal_content;
+ smartlist_t *added;
+ tor_mmap_t *mm;
+ int total = 0;
+
+ microdesc_cache_clear(cache);
+
+ mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
+ if (mm) {
+ added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
+ SAVED_IN_CACHE, 0);
+ total += smartlist_len(added);
+ smartlist_free(added);
+ }
+
+ journal_content = read_file_to_str(cache->journal_fname,
+ RFTS_IGNORE_MISSING, &st);
+ if (journal_content) {
+ added = microdescs_add_to_cache(cache, journal_content,
+ journal_content+st.st_size,
+ SAVED_IN_JOURNAL, 0);
+ total += smartlist_len(added);
+ smartlist_free(added);
+ tor_free(journal_content);
+ }
+ log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
+ total);
+ return 0;
+}
+
+int
+microdesc_cache_rebuild(microdesc_cache_t *cache)
+{
+ open_file_t *open_file;
+ FILE *f;
+ microdesc_t **mdp;
+ smartlist_t *wrote;
+
+ f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE,
+ 0600, &open_file);
+ if (!f)
+ return -1;
+
+ wrote = smartlist_create();
+
+ HT_FOREACH(mdp, microdesc_map, &cache->map) {
+ microdesc_t *md = *mdp;
+ if (md->no_save)
+ continue;
+
+ dump_microdescriptor(f, md);
+ if (md->saved_location != SAVED_IN_CACHE) {
+ tor_free(md->body);
+ md->saved_location = SAVED_IN_CACHE;
+ }
+
+ smartlist_add(wrote, md);
+ }
+
+ finish_writing_to_file(open_file); /*XXX Check me.*/
+
+ if (cache->cache_content)
+ tor_munmap_file(cache->cache_content);
+ cache->cache_content = tor_mmap_file(cache->cache_fname);
+ if (!cache->cache_content && smartlist_len(wrote)) {
+ log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+ cache->cache_fname);
+ return -1;
+ }
+ SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
+ if (md->no_save)
+ continue;
+ tor_assert(md->saved_location == SAVED_IN_CACHE);
+ md->body = (char*)cache->cache_content->data + md->off;
+ tor_assert(!memcmp(md->body, "onion-key", 9));
+ } SMARTLIST_FOREACH_END(wrote);
+
+ smartlist_free(wrote);
+
+ return 0;
+}
+
+void
+microdesc_free(microdesc_t *md)
+{
+ /* Must be removed from hash table! */
+ if (md->onion_pkey)
+ crypto_free_pk_env(md->onion_pkey);
+ if (md->body && md->saved_location != SAVED_IN_CACHE)
+ tor_free(md->body);
+
+ if (md->family) {
+ SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
+ smartlist_free(md->family);
+ }
+ tor_free(md->exitsummary);
+
+ tor_free(md);
+}
+
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index 5d1f8b2..752cb42 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -242,8 +242,14 @@ router_reload_consensus_networkstatus(void)
static void
vote_routerstatus_free(vote_routerstatus_t *rs)
{
+ vote_microdesc_hash_t *h, *next;
tor_free(rs->version);
tor_free(rs->status.exitsummary);
+ for (h = rs->microdesc; h; h = next) {
+ tor_free(h->microdesc_hash_line);
+ next = h->next;
+ tor_free(h);
+ }
tor_free(rs);
}
diff --git a/src/or/or.h b/src/or/or.h
index 2c79542..6a7aec6 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -89,6 +89,7 @@
#include "torgzip.h"
#include "address.h"
#include "compat_libevent.h"
+#include "ht.h"
/* These signals are defined to help control_signal_act work.
*/
@@ -1558,12 +1559,29 @@ typedef struct routerstatus_t {
} routerstatus_t;
/**DOCDOC*/
-typedef struct microdescriptor_t {
+typedef struct microdesc_t {
+ HT_ENTRY(microdesc_t) node;
+
+ /* Cache information */
+
+ time_t last_listed;
+ saved_location_t saved_location : 3;
+ unsigned int no_save : 1;
+ off_t off;
+
+ /* The string containing the microdesc. */
+
+ char *body;
+ size_t bodylen;
+ char digest[DIGEST256_LEN];
+
+ /* Fields in the microdescriptor. */
+
crypto_pk_env_t *onion_pkey;
smartlist_t *family;
char *exitsummary; /**< exit policy summary -
- * XXX weasel: this probably should not stay a string. */
-} microdescriptor_t;
+ * XXX this probably should not stay a string. */
+} microdesc_t;
/** How many times will we try to download a router's descriptor before giving
* up? */
@@ -3748,7 +3766,8 @@ int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src);
size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
int compressed);
typedef enum {
- NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT
+ NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT,
+ NS_V3_CONSENSUS_MICRODESC
} routerstatus_format_type_t;
int routerstatus_format_entry(char *buf, size_t buf_len,
routerstatus_t *rs, const char *platform,
@@ -3784,13 +3803,20 @@ int dirserv_read_measured_bandwidths(const char *from_file,
void dirvote_free_all(void);
+/** DOCDOC */
+typedef enum {
+ FLAV_NS,
+ FLAV_MICRODESC,
+} consensus_flavor_t;
+
/* vote manipulation */
char *networkstatus_compute_consensus(smartlist_t *votes,
int total_authorities,
crypto_pk_env_t *identity_key,
crypto_pk_env_t *signing_key,
const char *legacy_identity_key_digest,
- crypto_pk_env_t *legacy_signing_key);
+ crypto_pk_env_t *legacy_signing_key,
+ consensus_flavor_t flavor);
int networkstatus_add_detached_signatures(networkstatus_t *target,
ns_detached_signatures_t *sigs,
const char **msg_out);
@@ -3837,11 +3863,12 @@ networkstatus_t *
dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
authority_cert_t *cert);
-int dirvote_create_microdescriptor(char *out,
- size_t outlen, const routerinfo_t *ri);
-int dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
- const char *microdesc,
- size_t microdescriptor_len);
+microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri);
+int dirvote_format_microdesc_vote_line(char *out, size_t out_len,
+ const microdesc_t *md);
+int vote_routerstatus_find_microdesc_hash(char *digest256_out,
+ const vote_routerstatus_t *vrs,
+ int method);
#ifdef DIRVOTE_PRIVATE
char *format_networkstatus_vote(crypto_pk_env_t *private_key,
@@ -4051,6 +4078,25 @@ void do_hash_password(void);
int tor_init(int argc, char **argv);
#endif
+/********************************* microdesc.c *************************/
+
+typedef struct microdesc_cache_t microdesc_cache_t;
+
+microdesc_cache_t *get_microdesc_cache(void);
+
+smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
+ const char *s, const char *eos, saved_location_t where,
+ int no_save);
+smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
+ smartlist_t *descriptors, saved_location_t where,
+ int no_save);
+
+int microdesc_cache_rebuild(microdesc_cache_t *cache);
+int microdesc_cache_reload(microdesc_cache_t *cache);
+void microdesc_cache_clear(microdesc_cache_t *cache);
+
+void microdesc_free(microdesc_t *md);
+
/********************************* networkstatus.c *********************/
/** How old do we allow a v2 network-status to get before removing it
@@ -4927,7 +4973,8 @@ int router_get_router_hash(const char *s, char *digest);
int router_get_dir_hash(const char *s, char *digest);
int router_get_runningrouters_hash(const char *s, char *digest);
int router_get_networkstatus_v2_hash(const char *s, char *digest);
-int router_get_networkstatus_v3_hash(const char *s, char *digest);
+int router_get_networkstatus_v3_hash(const char *s, char *digest,
+ digest_algorithm_t algorithm);
int router_get_extrainfo_hash(const char *s, char *digest);
int router_append_dirobj_signature(char *buf, size_t buf_len,
const char *digest,
@@ -4971,6 +5018,10 @@ networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
ns_detached_signatures_t *networkstatus_parse_detached_signatures(
const char *s, const char *eos);
+smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
+ int allow_annotations,
+ int copy_body);
+
authority_cert_t *authority_cert_parse_from_string(const char *s,
const char **end_of_string);
int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index dd0d32e..277c7c6 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -111,6 +111,7 @@ typedef enum {
K_LEGACY_DIR_KEY,
A_PURPOSE,
+ A_LAST_LISTED,
_A_UNKNOWN,
R_RENDEZVOUS_SERVICE_DESCRIPTOR,
@@ -496,6 +497,14 @@ static token_rule_t networkstatus_detached_signature_token_table[] = {
END_OF_TABLE
};
+static token_rule_t microdesc_token_table[] = {
+ T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024),
+ T01("family", K_FAMILY, ARGS, NO_OBJ ),
+ T01("p", K_P, CONCAT_ARGS, NO_OBJ ),
+ A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ),
+ END_OF_TABLE
+};
+
#undef T
/* static function prototypes */
@@ -505,7 +514,8 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
static int router_get_hash_impl(const char *s, char *digest,
const char *start_str, const char *end_str,
- char end_char);
+ char end_char,
+ digest_algorithm_t alg);
static void token_free(directory_token_t *tok);
static smartlist_t *find_all_exitpolicy(smartlist_t *s);
static directory_token_t *_find_by_keyword(smartlist_t *s,
@@ -586,7 +596,8 @@ int
router_get_dir_hash(const char *s, char *digest)
{
return router_get_hash_impl(s,digest,
- "signed-directory","\ndirectory-signature",'\n');
+ "signed-directory","\ndirectory-signature",'\n',
+ DIGEST_SHA1);
}
/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
@@ -596,7 +607,8 @@ int
router_get_router_hash(const char *s, char *digest)
{
return router_get_hash_impl(s,digest,
- "router ","\nrouter-signature", '\n');
+ "router ","\nrouter-signature", '\n',
+ DIGEST_SHA1);
}
/** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers
@@ -606,7 +618,8 @@ int
router_get_runningrouters_hash(const char *s, char *digest)
{
return router_get_hash_impl(s,digest,
- "network-status","\ndirectory-signature", '\n');
+ "network-status","\ndirectory-signature", '\n',
+ DIGEST_SHA1);
}
/** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status
@@ -616,17 +629,19 @@ router_get_networkstatus_v2_hash(const char *s, char *digest)
{
return router_get_hash_impl(s,digest,
"network-status-version","\ndirectory-signature",
- '\n');
+ '\n',
+ DIGEST_SHA1);
}
/** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status
* string in <b>s</b>. Return 0 on success, -1 on failure. */
int
-router_get_networkstatus_v3_hash(const char *s, char *digest)
+router_get_networkstatus_v3_hash(const char *s, char *digest,
+ digest_algorithm_t alg)
{
return router_get_hash_impl(s,digest,
"network-status-version","\ndirectory-signature",
- ' ');
+ ' ', alg);
}
/** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo
@@ -634,7 +649,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest)
int
router_get_extrainfo_hash(const char *s, char *digest)
{
- return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n');
+ return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n',
+ DIGEST_SHA1);
}
/** Helper: used to generate signatures for routers, directories and
@@ -643,6 +659,8 @@ router_get_extrainfo_hash(const char *s, char *digest)
* surround it with -----BEGIN/END----- pairs, and write it to the
* <b>buf_len</b>-byte buffer at <b>buf</b>. Return 0 on success, -1 on
* failure.
+ *
+ * DOCDOC alg
*/
int
router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
@@ -1691,7 +1709,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
goto err;
}
if (router_get_hash_impl(s, digest, "dir-key-certificate-version",
- "\ndir-key-certification", '\n') < 0)
+ "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
goto err;
tok = smartlist_get(tokens, 0);
if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
@@ -2298,7 +2316,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
if (eos_out)
*eos_out = NULL;
- if (router_get_networkstatus_v3_hash(s, ns_digest)) {
+ if (router_get_networkstatus_v3_hash(s, ns_digest, DIGEST_SHA1)) {
log_warn(LD_DIR, "Unable to compute digest of network-status");
goto err;
}
@@ -3410,7 +3428,7 @@ find_all_exitpolicy(smartlist_t *s)
return out;
}
-/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first
+/** Compute the digest of the substring of <b>s</b> taken from the first
* occurrence of <b>start_str</b> through the first instance of c after the
* first subsequent occurrence of <b>end_str</b>; store the 20-byte result in
* <b>digest</b>; return 0 on success.
@@ -3420,7 +3438,8 @@ find_all_exitpolicy(smartlist_t *s)
static int
router_get_hash_impl(const char *s, char *digest,
const char *start_str,
- const char *end_str, char end_c)
+ const char *end_str, char end_c,
+ digest_algorithm_t alg)
{
char *start, *end;
start = strstr(s, start_str);
@@ -3446,14 +3465,169 @@ router_get_hash_impl(const char *s, char *digest,
}
++end;
- if (crypto_digest(digest, start, end-start)) {
- log_warn(LD_BUG,"couldn't compute digest");
- return -1;
+ if (alg == DIGEST_SHA1) {
+ if (crypto_digest(digest, start, end-start)) {
+ log_warn(LD_BUG,"couldn't compute digest");
+ return -1;
+ }
+ } else {
+ if (crypto_digest256(digest, start, end-start, alg)) {
+ log_warn(LD_BUG,"couldn't compute digest");
+ return -1;
+ }
}
return 0;
}
+/** DOCDOC Assuming that s starts with a microdesc, return the start of the
+ * *NEXT* one. */
+static const char *
+find_start_of_next_microdesc(const char *s, const char *eos)
+{
+ int started_with_annotations;
+ s = eat_whitespace_eos(s, eos);
+ if (!s)
+ return NULL;
+
+#define CHECK_LENGTH() STMT_BEGIN \
+ if (s+32 > eos) \
+ return NULL; \
+ STMT_END
+
+#define NEXT_LINE() STMT_BEGIN \
+ s = memchr(s, '\n', eos-s); \
+ if (!s || s+1 >= eos) \
+ return NULL; \
+ s++; \
+ STMT_END
+
+ CHECK_LENGTH();
+
+ started_with_annotations = (*s == '@');
+
+ if (started_with_annotations) {
+ /* Start by advancing to the first non-annotation line. */
+ while (*s == '@')
+ NEXT_LINE();
+ }
+ CHECK_LENGTH();
+
+ /* Now we should be pointed at an onion-key line. If we are, then skip
+ * it. */
+ if (!strcmpstart(s, "onion-key"))
+ NEXT_LINE();
+
+ /* Okay, now we're pointed at the first line of the microdescriptor which is
+ not an annotation or onion-key. The next line that _is_ an annotation or
+ onion-key is the start of the next microdescriptor. */
+ while (s+32 < eos) {
+ if (*s == '@' || !strcmpstart(s, "onion-key"))
+ return s;
+ NEXT_LINE();
+ }
+ return NULL;
+
+#undef CHECK_LENGTH
+#undef NEXT_LINE
+}
+
+/**DOCDOC*/
+smartlist_t *
+microdescs_parse_from_string(const char *s, const char *eos,
+ int allow_annotations, int copy_body)
+{
+ smartlist_t *tokens;
+ smartlist_t *result;
+ microdesc_t *md = NULL;
+ memarea_t *area;
+ const char *start = s;
+ const char *start_of_next_microdesc;
+ int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+
+ directory_token_t *tok;
+
+ if (!eos)
+ eos = s + strlen(s);
+
+ s = eat_whitespace_eos(s, eos);
+ area = memarea_new();
+ result = smartlist_create();
+ tokens = smartlist_create();
+
+ while (s < eos) {
+ start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
+ if (!start_of_next_microdesc)
+ start_of_next_microdesc = eos;
+
+ if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+ microdesc_token_table, flags)) {
+ log_warn(LD_DIR, "Unparseable microdescriptor");
+ goto next;
+ }
+
+ md = tor_malloc_zero(sizeof(microdesc_t));
+ {
+ const char *cp = tor_memstr(s, start_of_next_microdesc-s,
+ "onion-key");
+ tor_assert(cp);
+
+ md->bodylen = start_of_next_microdesc - cp;
+ if (copy_body)
+ md->body = tor_strndup(cp, md->bodylen);
+ else
+ md->body = (char*)cp;
+ md->off = cp - start;
+ }
+
+ if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+ if (parse_iso_time(tok->args[0], &md->last_listed)) {
+ log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+ goto next;
+ }
+ }
+
+ tok = find_by_keyword(tokens, K_ONION_KEY);
+ md->onion_pkey = tok->key;
+ tok->key = NULL;
+
+ if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+ int i;
+ md->family = smartlist_create();
+ for (i=0;i<tok->n_args;++i) {
+ if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+ log_warn(LD_DIR, "Illegal nickname %s in family line",
+ escaped(tok->args[i]));
+ goto next;
+ }
+ smartlist_add(md->family, tor_strdup(tok->args[i]));
+ }
+ }
+
+ if ((tok = find_opt_by_keyword(tokens, K_P))) {
+ md->exitsummary = tor_strdup(tok->args[0]);
+ }
+
+ crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+
+ smartlist_add(result, md);
+
+ md = NULL;
+ next:
+ if (md)
+ microdesc_free(md);
+
+ memarea_clear(area);
+ smartlist_clear(tokens);
+ s = start_of_next_microdesc;
+ }
+
+ memarea_drop_all(area);
+ smartlist_free(tokens);
+
+ return result;
+}
+
/** Parse the Tor version of the platform string <b>platform</b>,
* and compare it to the version in <b>cutoff</b>. Return 1 if
* the router is at least as new as the cutoff, else return 0.
@@ -3712,7 +3886,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
/* Compute descriptor hash for later validation. */
if (router_get_hash_impl(desc, desc_hash,
"rendezvous-service-descriptor ",
- "\nsignature", '\n') < 0) {
+ "\nsignature", '\n', DIGEST_SHA1) < 0) {
log_warn(LD_REND, "Couldn't compute descriptor hash.");
goto err;
}
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index a10493e..8e566e2 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -858,7 +858,8 @@ test_dir_v3_networkstatus(void)
cert3->identity_key,
sign_skey_3,
"AAAAAAAAAAAAAAAAAAAA",
- sign_skey_leg1);
+ sign_skey_leg1,
+ FLAV_NS);
test_assert(consensus_text);
con = networkstatus_parse_vote_from_string(consensus_text, NULL,
NS_TYPE_CONSENSUS);
@@ -966,11 +967,13 @@ test_dir_v3_networkstatus(void)
smartlist_shuffle(votes);
consensus_text2 = networkstatus_compute_consensus(votes, 3,
cert2->identity_key,
- sign_skey_2, NULL,NULL);
+ sign_skey_2, NULL,NULL,
+ FLAV_NS);
smartlist_shuffle(votes);
consensus_text3 = networkstatus_compute_consensus(votes, 3,
cert1->identity_key,
- sign_skey_1, NULL,NULL);
+ sign_skey_1, NULL,NULL,
+ FLAV_NS);
test_assert(consensus_text2);
test_assert(consensus_text3);
con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,
--
1.5.6.5
More information about the tor-commits
mailing list