[tor-commits] [tor/master] Split directory.c code into several modules

commit 194acfb51d9b26ae63f9d48d3442c4ff64cbc8cc Author: Nick Mathewson nickm@torproject.org Date: Fri Sep 21 10:26:57 2018 -0400

Split directory.c code into several modules

Parts of this C file naturally belong in dircache, dirclient, and dircommon: so, move them there. --- changes/ticket26744 | 3 + src/core/include.am | 10 +- src/core/mainloop/connection.c | 2 +- src/core/mainloop/mainloop.c | 2 +- src/core/or/circuitbuild.c | 2 +- src/core/or/circuitlist.c | 2 +- src/core/or/circuituse.c | 2 +- src/core/or/connection_edge.c | 2 +- src/core/or/relay.c | 2 +- src/feature/client/bridges.c | 4 +- src/feature/client/entrynodes.c | 2 +- src/feature/control/control.c | 4 +- src/feature/dirauth/dirvote.c | 3 +- src/feature/dirauth/process_descs.c | 3 +- src/feature/dircache/dircache.c | 1739 ++++++++++++ src/feature/dircache/dircache.h | 43 + src/feature/dircache/directory.h | 347 --- src/feature/dircache/dirserv.c | 2 +- .../directory.c => dirclient/dirclient.c} | 2961 +------------------- src/feature/dirclient/dirclient.h | 172 ++ src/feature/dirclient/dlstatus.c | 422 +++ src/feature/dirclient/dlstatus.h | 58 + src/feature/dircommon/directory.c | 651 +++++ src/feature/dircommon/directory.h | 129 + src/feature/hs/hs_client.c | 3 +- src/feature/hs/hs_service.c | 3 +- src/feature/nodelist/authcert.c | 4 +- src/feature/nodelist/dirlist.c | 2 +- src/feature/nodelist/microdesc.c | 3 +- src/feature/nodelist/networkstatus.c | 4 +- src/feature/nodelist/node_select.c | 3 +- src/feature/nodelist/routerlist.c | 4 +- src/feature/relay/router.c | 3 +- src/feature/rend/rendclient.c | 3 +- src/feature/rend/rendservice.c | 3 +- src/test/fuzz/fuzz_http.c | 5 +- src/test/test_connection.c | 2 +- src/test/test_dir.c | 8 +- src/test/test_dir_handle_get.c | 4 +- src/test/test_entrynodes.c | 5 +- src/test/test_hs_cache.c | 6 +- src/test/test_hs_client.c | 2 +- src/test/test_hs_common.c | 2 +- src/test/test_oos.c | 2 +- src/test/test_routerlist.c | 5 +- 45 files changed, 3390 insertions(+), 3253 deletions(-)

diff --git a/changes/ticket26744 b/changes/ticket26744 new file mode 100644 index 000000000..10ff51f06 --- /dev/null +++ b/changes/ticket26744 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Split directory.c into separate pieces for client, server, and + common functionality. Closes ticket 26744. diff --git a/src/core/include.am b/src/core/include.am index d042f697b..61056d7fd 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -62,9 +62,12 @@ LIBTOR_APP_A_SOURCES = \ src/feature/dirauth/keypin.c \ src/feature/dircache/conscache.c \ src/feature/dircache/consdiffmgr.c \ - src/feature/dircache/directory.c \ + src/feature/dircache/dircache.c \ src/feature/dircache/dirserv.c \ + src/feature/dirclient/dirclient.c \ + src/feature/dirclient/dlstatus.c \ src/feature/dircommon/consdiff.c \ + src/feature/dircommon/directory.c \ src/feature/dircommon/fp_pair.c \ src/feature/dircommon/voting_schedule.c \ src/feature/hibernate/hibernate.c \ @@ -257,12 +260,15 @@ noinst_HEADERS += \ src/feature/dircache/cached_dir_st.h \ src/feature/dircache/conscache.h \ src/feature/dircache/consdiffmgr.h \ - src/feature/dircache/directory.h \ + src/feature/dircache/dircache.h \ src/feature/dircache/dirserv.h \ src/feature/dirclient/dir_server_st.h \ + src/feature/dirclient/dirclient.h \ + src/feature/dirclient/dlstatus.h \ src/feature/dirclient/download_status_st.h \ src/feature/dircommon/consdiff.h \ src/feature/dircommon/dir_connection_st.h \ + src/feature/dircommon/directory.h \ src/feature/dircommon/fp_pair.h \ src/feature/dircommon/vote_timing_st.h \ src/feature/dircommon/voting_schedule.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index f85a2b61d..4231bec01 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -77,7 +77,7 @@ #include "core/or/connection_or.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/dircache/dirserv.h" #include "feature/relay/dns.h" #include "feature/client/dnsserv.h" diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 4ab00f92d..1072a23b7 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -68,7 +68,7 @@ #include "feature/control/control.h" #include "core/mainloop/cpuworker.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/reachability.h" #include "feature/relay/dns.h" diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 74f60e6c4..2f404d179 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -44,7 +44,7 @@ #include "core/or/connection_or.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/client/entrynodes.h" #include "core/crypto/hs_ntor.h" #include "core/mainloop/mainloop.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index f231beb61..c07c8e97a 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -69,7 +69,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/crypt_ops/crypto_dh.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/client/entrynodes.h" #include "core/mainloop/mainloop.h" #include "feature/hs/hs_circuit.h" diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 0f2b1ede3..7b1f31559 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -40,7 +40,7 @@ #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "feature/control/control.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/client/entrynodes.h" #include "feature/hs/hs_common.h" #include "feature/hs/hs_client.h" diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index d0fad6700..daf978b99 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -73,7 +73,7 @@ #include "lib/crypt_ops/crypto_util.h" #include "feature/relay/dns.h" #include "feature/client/dnsserv.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/dircache/dirserv.h" #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_common.h" diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 4638d9f21..47be0f403 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -63,7 +63,7 @@ #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/relay/dns.h" #include "feature/stats/geoip.h" #include "feature/hs/hs_cache.h" diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 5ab0f6f4f..e8afb5a92 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -18,7 +18,9 @@ #include "core/or/circuitbuild.h" #include "app/config/config.h" #include "core/mainloop/connection.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" #include "feature/client/entrynodes.h" #include "feature/nodelist/nodelist.h" #include "core/or/policies.h" diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 94bb011fb..521bf33b5 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -125,7 +125,7 @@ #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/client/entrynodes.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/microdesc.h" diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b5a0dfeaf..f307101ed 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -56,7 +56,9 @@ #include "feature/control/fmt_serverstatus.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dircache/dirserv.h" #include "feature/client/dnsserv.h" #include "feature/client/entrynodes.h" diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 5ca9ed4a4..a1fe0d423 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -9,7 +9,8 @@ #include "feature/dirauth/dircollate.h" #include "feature/dirauth/recommend_pkg.h" #include "feature/dirauth/voteflags.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/dirauth/bwauth.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/guardfraction.h" diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index 2c2fefca7..1cf31f85c 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -19,7 +19,8 @@ #include "core/or/policies.h" #include "feature/dirauth/keypin.h" #include "feature/dirauth/reachability.h" -#include "feature/dircache/directory.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/dircommon/directory.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c new file mode 100644 index 000000000..ef4fe391b --- /dev/null +++ b/src/feature/dircache/dircache.c @@ -0,0 +1,1739 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define DIRCACHE_PRIVATE + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/relay.h" +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/mode.h" +#include "feature/dirauth/process_descs.h" +#include "feature/dircache/conscache.h" +#include "feature/dircache/consdiffmgr.h" +#include "feature/dircache/dircache.h" +#include "feature/dircache/dirserv.h" +#include "feature/dircommon/directory.h" +#include "feature/dircommon/fp_pair.h" +#include "feature/hs/hs_cache.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/routerlist.h" +#include "feature/rend/rendcache.h" +#include "feature/stats/geoip.h" +#include "feature/stats/rephist.h" +#include "lib/compress/compress.h" + +#include "feature/dircache/cached_dir_st.h" +#include "feature/dircommon/dir_connection_st.h" +#include "feature/nodelist/authority_cert_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/routerinfo_st.h" + +/** Maximum size, in bytes, for any directory object that we're accepting + * as an upload. */ +#define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */ + +/** HTTP cache control: how long do we tell proxies they can cache each + * kind of document we serve? */ +#define FULL_DIR_CACHE_LIFETIME (60*60) +#define RUNNINGROUTERS_CACHE_LIFETIME (20*60) +#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60) +#define NETWORKSTATUS_CACHE_LIFETIME (5*60) +#define ROUTERDESC_CACHE_LIFETIME (30*60) +#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) +#define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) + +/** Parse an HTTP request string <b>headers</b> of the form + * \verbatim + * "%s [http[s]://]%s HTTP/1..." + * \endverbatim + * If it's well-formed, strdup the second %s into *<b>url</b>, and + * nul-terminate it. If the url doesn't start with "/tor/", rewrite it + * so it does. Return 0. + * Otherwise, return -1. + */ +STATIC int +parse_http_url(const char *headers, char **url) +{ + char *command = NULL; + if (parse_http_command(headers, &command, url) < 0) { + return -1; + } + if (strcmpstart(*url, "/tor/")) { + char *new_url = NULL; + tor_asprintf(&new_url, "/tor%s%s", + *url[0] == '/' ? "" : "/", + *url); + tor_free(*url); + *url = new_url; + } + tor_free(command); + return 0; +} + +/** Create an http response for the client <b>conn</b> out of + * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>. + */ +static void +write_short_http_response(dir_connection_t *conn, int status, + const char *reason_phrase) +{ + char *buf = NULL; + char *datestring = NULL; + + IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */ + reason_phrase = "unspecified"; + } + + if (server_mode(get_options())) { + /* include the Date: header, but only if we're a relay or bridge */ + char datebuf[RFC1123_TIME_LEN+1]; + format_rfc1123_time(datebuf, time(NULL)); + tor_asprintf(&datestring, "Date: %s\r\n", datebuf); + } + + tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n", + status, reason_phrase, datestring?datestring:""); + + log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); + connection_buf_add(buf, strlen(buf), TO_CONN(conn)); + + tor_free(datestring); + tor_free(buf); +} + +/** Write the header for an HTTP/1.0 response onto <b>conn</b>->outbuf, + * with <b>type</b> as the Content-Type. + * + * If <b>length</b> is nonnegative, it is the Content-Length. + * If <b>encoding</b> is provided, it is the Content-Encoding. + * If <b>cache_lifetime</b> is greater than 0, the content may be cached for + * up to cache_lifetime seconds. Otherwise, the content may not be cached. */ +static void +write_http_response_header_impl(dir_connection_t *conn, ssize_t length, + const char *type, const char *encoding, + const char *extra_headers, + long cache_lifetime) +{ + char date[RFC1123_TIME_LEN+1]; + time_t now = time(NULL); + buf_t *buf = buf_new_with_capacity(1024); + + tor_assert(conn); + + format_rfc1123_time(date, now); + + buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date); + if (type) { + buf_add_printf(buf, "Content-Type: %s\r\n", type); + } + if (!is_local_addr(&conn->base_.addr)) { + /* Don't report the source address for a nearby/private connection. + * Otherwise we tend to mis-report in cases where incoming ports are + * being forwarded to a Tor server running behind the firewall. */ + buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address); + } + if (encoding) { + buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding); + } + if (length >= 0) { + buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length); + } + if (cache_lifetime > 0) { + char expbuf[RFC1123_TIME_LEN+1]; + format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime)); + /* We could say 'Cache-control: max-age=%d' here if we start doing + * http/1.1 */ + buf_add_printf(buf, "Expires: %s\r\n", expbuf); + } else if (cache_lifetime == 0) { + /* We could say 'Cache-control: no-cache' here if we start doing + * http/1.1 */ + buf_add_string(buf, "Pragma: no-cache\r\n"); + } + if (extra_headers) { + buf_add_string(buf, extra_headers); + } + buf_add_string(buf, "\r\n"); + + connection_buf_add_buf(TO_CONN(conn), buf); + buf_free(buf); +} + +/** As write_http_response_header_impl, but sets encoding and content-typed + * based on whether the response will be <b>compressed</b> or not. */ +static void +write_http_response_headers(dir_connection_t *conn, ssize_t length, + compress_method_t method, + const char *extra_headers, long cache_lifetime) +{ + const char *methodname = compression_method_get_name(method); + const char *doctype; + if (method == NO_METHOD) + doctype = "text/plain"; + else + doctype = "application/octet-stream"; + write_http_response_header_impl(conn, length, + doctype, + methodname, + extra_headers, + cache_lifetime); +} + +/** As write_http_response_headers, but assumes extra_headers is NULL */ +static void +write_http_response_header(dir_connection_t *conn, ssize_t length, + compress_method_t method, + long cache_lifetime) +{ + write_http_response_headers(conn, length, method, NULL, cache_lifetime); +} + +/** Array of compression methods to use (if supported) for serving + * precompressed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_precompressed[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Array of compression methods to use (if supported) for serving + * streamed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_streaming_compression[] = { + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + +/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>, + * and convert them to a bitfield where compression method x is supported if + * and only if 1 &lt;&lt; x is set in the bitfield. */ +STATIC unsigned +parse_accept_encoding_header(const char *h) +{ + unsigned result = (1u << NO_METHOD); + smartlist_t *methods = smartlist_new(); + smartlist_split_string(methods, h, ",", + SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + SMARTLIST_FOREACH_BEGIN(methods, const char *, m) { + compress_method_t method = compression_method_get_by_name(m); + if (method != UNKNOWN_METHOD) { + tor_assert(((unsigned)method) < 8*sizeof(unsigned)); + result |= (1u << method); + } + } SMARTLIST_FOREACH_END(m); + SMARTLIST_FOREACH_BEGIN(methods, char *, m) { + tor_free(m); + } SMARTLIST_FOREACH_END(m); + smartlist_free(methods); + return result; +} + +/** Decide whether a client would accept the consensus we have. + * + * Clients can say they only want a consensus if it's signed by more + * than half the authorities in a list. They pass this list in + * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>". + * + * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring + * of the full authority identity digest. (Only strings of even length, + * i.e. encodings of full bytes, are handled correctly. In the case + * of an odd number of hex digits the last one is silently ignored.) + * + * Returns 1 if more than half of the requested authorities signed the + * consensus, 0 otherwise. + */ +static int +client_likes_consensus(const struct consensus_cache_entry_t *ent, + const char *want_url) +{ + smartlist_t *voters = smartlist_new(); + int need_at_least; + int have = 0; + + if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) { + smartlist_free(voters); + return 1; // We don't know the voters; assume the client won't mind. */ + } + + smartlist_t *want_authorities = smartlist_new(); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); + need_at_least = smartlist_len(want_authorities)/2+1; + + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) { + + SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) { + if (!strcasecmpstart(digest, want_digest)) { + have++; + break; + }; + } SMARTLIST_FOREACH_END(digest); + + /* early exit, if we already have enough */ + if (have >= need_at_least) + break; + } SMARTLIST_FOREACH_END(want_digest); + + SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); + smartlist_free(want_authorities); + SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp)); + smartlist_free(voters); + return (have >= need_at_least); +} + +/** Return the compression level we should use for sending a compressed + * response of size <b>n_bytes</b>. */ +STATIC compression_level_t +choose_compression_level(ssize_t n_bytes) +{ + if (! have_been_under_memory_pressure()) { + return HIGH_COMPRESSION; /* we have plenty of RAM. */ + } else if (n_bytes < 0) { + return HIGH_COMPRESSION; /* unknown; might be big. */ + } else if (n_bytes < 1024) { + return LOW_COMPRESSION; + } else if (n_bytes < 2048) { + return MEDIUM_COMPRESSION; + } else { + return HIGH_COMPRESSION; + } +} + +/** Information passed to handle a GET request. */ +typedef struct get_handler_args_t { + /** Bitmask of compression methods that the client said (or implied) it + * supported. */ + unsigned compression_supported; + /** If nonzero, the time included an if-modified-since header with this + * value. */ + time_t if_modified_since; + /** String containing the requested URL or resource. */ + const char *url; + /** String containing the HTTP headers */ + const char *headers; +} get_handler_args_t; + +/** Entry for handling an HTTP GET request. + * + * This entry matches a request if "string" is equal to the requested + * resource, or if "is_prefix" is true and "string" is a prefix of the + * requested resource. + * + * The 'handler' function is called to handle the request. It receives + * an arguments structure, and must return 0 on success or -1 if we should + * close the connection. + **/ +typedef struct url_table_ent_s { + const char *string; + int is_prefix; + int (*handler)(dir_connection_t *conn, const get_handler_args_t *args); +} url_table_ent_t; + +static int handle_get_frontpage(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_status_vote(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_microdesc(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_descriptor(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_keys(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_hs_descriptor_v2(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_robots(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_networkstatus_bridges(dir_connection_t *conn, + const get_handler_args_t *args); + +/** Table for handling GET requests. */ +static const url_table_ent_t url_table[] = { + { "/tor/", 0, handle_get_frontpage }, + { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, + { "/tor/status-vote/current/", 1, handle_get_status_vote }, + { "/tor/status-vote/next/", 1, handle_get_status_vote }, + { "/tor/micro/d/", 1, handle_get_microdesc }, + { "/tor/server/", 1, handle_get_descriptor }, + { "/tor/extra/", 1, handle_get_descriptor }, + { "/tor/keys/", 1, handle_get_keys }, + { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 }, + { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 }, + { "/tor/robots.txt", 0, handle_get_robots }, + { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges }, + { NULL, 0, NULL }, +}; + +/** Helper function: called when a dirserver gets a complete HTTP GET + * request. Look for a request for a directory or for a rendezvous + * service descriptor. On finding one, write a response into + * conn->outbuf. If the request is unrecognized, send a 404. + * Return 0 if we handled this successfully, or -1 if we need to close + * the connection. */ +MOCK_IMPL(STATIC int, +directory_handle_command_get,(dir_connection_t *conn, const char *headers, + const char *req_body, size_t req_body_len)) +{ + char *url, *url_mem, *header; + time_t if_modified_since = 0; + int zlib_compressed_in_url; + unsigned compression_methods_supported; + + /* We ignore the body of a GET request. */ + (void)req_body; + (void)req_body_len; + + log_debug(LD_DIRSERV,"Received GET command."); + + conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; + + if (parse_http_url(headers, &url) < 0) { + write_short_http_response(conn, 400, "Bad request"); + return 0; + } + if ((header = http_get_header(headers, "If-Modified-Since: "))) { + struct tm tm; + if (parse_http_time(header, &tm) == 0) { + if (tor_timegm(&tm, &if_modified_since)<0) { + if_modified_since = 0; + } else { + log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header)); + } + } + /* The correct behavior on a malformed If-Modified-Since header is to + * act as if no If-Modified-Since header had been given. */ + tor_free(header); + } + log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); + + url_mem = url; + { + size_t url_len = strlen(url); + + zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z"); + if (zlib_compressed_in_url) { + url[url_len-2] = '\0'; + } + } + + if ((header = http_get_header(headers, "Accept-Encoding: "))) { + compression_methods_supported = parse_accept_encoding_header(header); + tor_free(header); + } else { + compression_methods_supported = (1u << NO_METHOD); + } + if (zlib_compressed_in_url) { + compression_methods_supported |= (1u << ZLIB_METHOD); + } + + /* Remove all methods that we don't both support. */ + compression_methods_supported &= tor_compress_get_supported_method_bitmask(); + + get_handler_args_t args; + args.url = url; + args.headers = headers; + args.if_modified_since = if_modified_since; + args.compression_supported = compression_methods_supported; + + int i, result = -1; + for (i = 0; url_table[i].string; ++i) { + int match; + if (url_table[i].is_prefix) { + match = !strcmpstart(url, url_table[i].string); + } else { + match = !strcmp(url, url_table[i].string); + } + if (match) { + result = url_table[i].handler(conn, &args); + goto done; + } + } + + /* we didn't recognize the url */ + write_short_http_response(conn, 404, "Not found"); + result = 0; + + done: + tor_free(url_mem); + return result; +} + +/** Helper function for GET / or GET /tor/ + */ +static int +handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) +{ + (void) args; /* unused */ + const char *frontpage = get_dirportfrontpage(); + + if (frontpage) { + size_t dlen; + dlen = strlen(frontpage); + /* Let's return a disclaimer page (users shouldn't use V1 anymore, + and caches don't fetch '/', so this is safe). */ + + /* [We don't check for write_bucket_low here, since we want to serve + * this page no matter what.] */ + write_http_response_header_impl(conn, dlen, "text/html", "identity", + NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); + connection_buf_add(frontpage, dlen, TO_CONN(conn)); + } else { + write_short_http_response(conn, 404, "Not found"); + } + return 0; +} + +/** Warn that the cached consensus <b>consensus</b> of type + * <b>flavor</b> is too old and will not be served to clients. Rate-limit the + * warning to avoid logging an entry on every request. + */ +static void +warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now) +{ +#define TOO_OLD_WARNING_INTERVAL (60*60) + static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); + char timestamp[ISO_TIME_LEN+1]; + time_t valid_until; + char *dupes; + + if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) + return; + + if ((dupes = rate_limit_log(&warned, now))) { + format_local_iso_time(timestamp, valid_until); + log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " + "serve it to clients. It was valid until %s local time and we " + "continued to serve it for up to 24 hours after it expired.%s", + flavor ? flavor : "", flavor ? " " : "", timestamp, dupes); + tor_free(dupes); + } +} + +/** + * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into + * <b>digest</b>. Return 0 on success. On failure, report that the hash came + * from <b>location</b>, report that we are taking <b>action</b> with it, and + * return -1. + */ +static int +parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location, + const char *action) +{ + if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) == + DIGEST256_LEN) { + return 0; + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "%s contained bogus digest %s; %s.", + location, escaped(hex), action); + return -1; + } +} + +/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>, + * set <b>digest_out<b> to a new smartlist containing every 256-bit + * hex-encoded digest listed in that header and return 0. Otherwise return + * -1. */ +static int +parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) +{ + char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER); + if (hdr == NULL) { + return -1; + } + smartlist_t *hex_digests = smartlist_new(); + *digests_out = smartlist_new(); + smartlist_split_string(hex_digests, hdr, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { + uint8_t digest[DIGEST256_LEN]; + if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header", + "ignoring")) { + smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); + } + } SMARTLIST_FOREACH_END(hex); + SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); + smartlist_free(hex_digests); + tor_free(hdr); + return 0; +} + +/** Fallback compression method. The fallback compression method is used in + * case a client requests a non-compressed document. We only store compressed + * documents, so we use this compression method to fetch the document and let + * the spooling system do the streaming decompression. + */ +#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD + +/** + * Try to find the best consensus diff possible in order to serve a client + * request for a diff from one of the consensuses in <b>digests</b> to the + * current consensus of flavor <b>flav</b>. The client supports the + * compression methods listed in the <b>compression_methods</b> bitfield: + * place the method chosen (if any) into <b>compression_used_out</b>. + */ +static struct consensus_cache_entry_t * +find_best_diff(const smartlist_t *digests, int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (0 == (compression_methods & (1u<<method))) + continue; // client doesn't like this one, or we don't have it. + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, + diff_from, DIGEST256_LEN, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; + } + } + } SMARTLIST_FOREACH_END(diff_from); + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { + if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from, + DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + } SMARTLIST_FOREACH_END(diff_from); + + return NULL; +} + +/** Lookup the cached consensus document by the flavor found in <b>flav</b>. + * The preferred set of compression methods should be listed in the + * <b>compression_methods</b> bitfield. The compression method chosen (if any) + * is stored in <b>compression_used_out</b>. */ +static struct consensus_cache_entry_t * +find_best_consensus(int flav, + unsigned compression_methods, + compress_method_t *compression_used_out) +{ + struct consensus_cache_entry_t *result = NULL; + unsigned u; + + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + + if (0 == (compression_methods & (1u<<method))) + continue; + + if (consdiffmgr_find_consensus(&result, flav, + method) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = method; + return result; + } + } + + if (consdiffmgr_find_consensus(&result, flav, + FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { + tor_assert_nonfatal(result); + *compression_used_out = FALLBACK_COMPRESS_METHOD; + return result; + } + + return NULL; +} + +/** Try to find the best supported compression method possible from a given + * <b>compression_methods</b>. Return NO_METHOD if no mutually supported + * compression method could be found. */ +static compress_method_t +find_best_compression_method(unsigned compression_methods, int stream) +{ + unsigned u; + compress_method_t *methods; + size_t length; + + if (stream) { + methods = srv_meth_pref_streaming_compression; + length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); + } else { + methods = srv_meth_pref_precompressed; + length = ARRAY_LENGTH(srv_meth_pref_precompressed); + } + + for (u = 0; u < length; ++u) { + compress_method_t method = methods[u]; + if (compression_methods & (1u<<method)) + return method; + } + + return NO_METHOD; +} + +/** Check if any of the digests in <b>digests</b> matches the latest consensus + * flavor (given in <b>flavor</b>) that we have available. */ +static int +digest_list_contains_best_consensus(consensus_flavor_t flavor, + const smartlist_t *digests) +{ + const networkstatus_t *ns = NULL; + + if (digests == NULL) + return 0; + + ns = networkstatus_get_latest_consensus_by_flavor(flavor); + + if (ns == NULL) + return 0; + + SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) { + if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN)) + return 1; + } SMARTLIST_FOREACH_END(digest); + + return 0; +} + +/** Encodes the results of parsing a consensus request to figure out what + * consensus, and possibly what diffs, the user asked for. */ +typedef struct { + /** name of the flavor to retrieve. */ + char *flavor; + /** flavor to retrive, as enum. */ + consensus_flavor_t flav; + /** plus-separated list of authority fingerprints; see + * client_likes_consensus(). Aliases the URL in the request passed to + * parse_consensus_request(). */ + const char *want_fps; + /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses + * to return a diff from. */ + smartlist_t *diff_from_digests; + /** If true, never send a full consensus. If there is no diff, send + * a 404 instead. */ + int diff_only; +} parsed_consensus_request_t; + +/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since + * it is stack-allocated. */ +static void +parsed_consensus_request_clear(parsed_consensus_request_t *req) +{ + if (!req) + return; + tor_free(req->flavor); + if (req->diff_from_digests) { + SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d)); + smartlist_free(req->diff_from_digests); + } + memset(req, 0, sizeof(parsed_consensus_request_t)); +} + +/** + * Parse the URL and relevant headers of <b>args</b> for a current-consensus + * request to learn what flavor of consensus we want, what keys it must be + * signed with, and what diffs we would accept (or demand) instead. Return 0 + * on success and -1 on failure. + */ +static int +parse_consensus_request(parsed_consensus_request_t *out, + const get_handler_args_t *args) +{ + const char *url = args->url; + memset(out, 0, sizeof(parsed_consensus_request_t)); + out->flav = FLAV_NS; + + const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/"; + const char CONSENSUS_FLAVORED_PREFIX[] = + "/tor/status-vote/current/consensus-"; + + /* figure out the flavor if any, and who we wanted to sign the thing */ + const char *after_flavor = NULL; + + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + after_flavor = cp+1; + out->flavor = tor_strndup(f, cp-f); + } else { + out->flavor = tor_strdup(f); + } + int flav = networkstatus_parse_flavor_name(out->flavor); + if (flav < 0) + flav = FLAV_NS; + out->flav = flav; + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + after_flavor = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* see whether we've been asked explicitly for a diff from an older + * consensus. (The user might also have said that a diff would be okay, + * via X-Or-Diff-From-Consensus */ + const char DIFF_COMPONENT[] = "diff/"; + char *diff_hash_in_url = NULL; + if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) { + after_flavor += strlen(DIFF_COMPONENT); + const char *cp = strchr(after_flavor, '/'); + if (cp) { + diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor); + out->want_fps = cp+1; + } else { + diff_hash_in_url = tor_strdup(after_flavor); + out->want_fps = NULL; + } + } else { + out->want_fps = after_flavor; + } + + if (diff_hash_in_url) { + uint8_t diff_from[DIGEST256_LEN]; + out->diff_from_digests = smartlist_new(); + out->diff_only = 1; + int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL", + "rejecting"); + tor_free(diff_hash_in_url); + if (ok) { + smartlist_add(out->diff_from_digests, + tor_memdup(diff_from, DIGEST256_LEN)); + } else { + return -1; + } + } else { + parse_or_diff_from_header(&out->diff_from_digests, args->headers); + } + + return 0; +} + +/** Helper function for GET /tor/status-vote/current/consensus + */ +static int +handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 0); + const time_t if_modified_since = args->if_modified_since; + int clear_spool = 0; + + /* v3 network status fetch. */ + long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + + time_t now = time(NULL); + parsed_consensus_request_t req; + + if (parse_consensus_request(&req, args) < 0) { + write_short_http_response(conn, 404, "Couldn't parse request"); + goto done; + } + + if (digest_list_contains_best_consensus(req.flav, + req.diff_from_digests)) { + write_short_http_response(conn, 304, "Not modified"); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + goto done; + } + + struct consensus_cache_entry_t *cached_consensus = NULL; + + compress_method_t compression_used = NO_METHOD; + if (req.diff_from_digests) { + cached_consensus = find_best_diff(req.diff_from_digests, req.flav, + args->compression_supported, + &compression_used); + } + + if (req.diff_only && !cached_consensus) { + write_short_http_response(conn, 404, "No such diff available"); + // XXXX warn_consensus_is_too_old(v, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (! cached_consensus) { + cached_consensus = find_best_consensus(req.flav, + args->compression_supported, + &compression_used); + } + + time_t fresh_until, valid_until; + int have_fresh_until = 0, have_valid_until = 0; + if (cached_consensus) { + have_fresh_until = + !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); + have_valid_until = + !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); + } + + if (cached_consensus && have_valid_until && + !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { + write_short_http_response(conn, 404, "Consensus is too old"); + warn_consensus_is_too_old(cached_consensus, req.flavor, now); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } + + if (cached_consensus && req.want_fps && + !client_likes_consensus(cached_consensus, req.want_fps)) { + write_short_http_response(conn, 404, "Consensus not signed by sufficient " + "number of requested authorities"); + geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); + goto done; + } + + conn->spool = smartlist_new(); + clear_spool = 1; + { + spooled_resource_t *spooled; + if (cached_consensus) { + spooled = spooled_resource_new_from_cache_entry(cached_consensus); + smartlist_add(conn->spool, spooled); + } + } + + lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; + + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, + compress_method != NO_METHOD, + &size_guess, + &n_expired); + + if (!smartlist_len(conn->spool) && !n_expired) { + write_short_http_response(conn, 404, "Not found"); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if (!smartlist_len(conn->spool)) { + write_short_http_response(conn, 304, "Not modified"); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + goto done; + } + + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { + log_debug(LD_DIRSERV, + "Client asked for network status lists, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_short_http_response(conn, 503, "Directory busy, try again later"); + geoip_note_ns_response(GEOIP_REJECT_BUSY); + goto done; + } + + tor_addr_t addr; + if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); + geoip_note_ns_response(GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (conn->dirreq_id) + geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess, + DIRREQ_DIRECT); + } + + /* Use this header to tell caches that the response depends on the + * X-Or-Diff-From-Consensus header (or lack thereof). */ + const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n"; + + clear_spool = 0; + + // The compress_method might have been NO_METHOD, but we store the data + // compressed. Decompress them using `compression_used`. See fallback code in + // find_best_consensus() and find_best_diff(). + write_http_response_headers(conn, -1, + compress_method == NO_METHOD ? + NO_METHOD : compression_used, + vary_header, + smartlist_len(conn->spool) == 1 ? lifetime : 0); + + if (compress_method == NO_METHOD && smartlist_len(conn->spool)) + conn->compress_state = tor_compress_new(0, compression_used, + HIGH_COMPRESSION); + + /* Prime the connection with some data. */ + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); + goto done; + + done: + parsed_consensus_request_clear(&req); + if (clear_spool) { + dir_conn_clear_spool(conn); + } + return 0; +} + +/** Helper function for GET /tor/status-vote/{current,next}/... + */ +static int +handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + { + ssize_t body_len = 0; + ssize_t estimated_len = 0; + int lifetime = 60; /* XXXX?? should actually use vote intervals. */ + /* This smartlist holds strings that we can compress on the fly. */ + smartlist_t *items = smartlist_new(); + /* This smartlist holds cached_dir_t objects that have a precompressed + * deflated version. */ + smartlist_t *dir_items = smartlist_new(); + dirvote_dirreq_get_status_vote(url, items, dir_items); + if (!smartlist_len(dir_items) && !smartlist_len(items)) { + write_short_http_response(conn, 404, "Not found"); + goto vote_done; + } + + /* We're sending items from at most one kind of source */ + tor_assert_nonfatal(smartlist_len(items) == 0 || + smartlist_len(dir_items) == 0); + + int streaming; + unsigned mask; + if (smartlist_len(items)) { + /* We're taking strings and compressing them on the fly. */ + streaming = 1; + mask = ~0u; + } else { + /* We're taking cached_dir_t objects. We only have them uncompressed + * or deflated. */ + streaming = 0; + mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD); + } + const compress_method_t compress_method = find_best_compression_method( + args->compression_supported&mask, streaming); + + SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, + body_len += compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len); + estimated_len += body_len; + SMARTLIST_FOREACH(items, const char *, item, { + size_t ln = strlen(item); + if (compress_method != NO_METHOD) { + estimated_len += ln/2; + } else { + body_len += ln; estimated_len += ln; + } + }); + + if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) { + write_short_http_response(conn, 503, "Directory busy, try again later"); + goto vote_done; + } + write_http_response_header(conn, body_len ? body_len : -1, + compress_method, + lifetime); + + if (smartlist_len(items)) { + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(estimated_len)); + SMARTLIST_FOREACH(items, const char *, c, + connection_buf_add_compress(c, strlen(c), conn, 0)); + connection_buf_add_compress("", 0, conn, 1); + } else { + SMARTLIST_FOREACH(items, const char *, c, + connection_buf_add(c, strlen(c), TO_CONN(conn))); + } + } else { + SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, + connection_buf_add(compress_method != NO_METHOD ? + d->dir_compressed : d->dir, + compress_method != NO_METHOD ? + d->dir_compressed_len : d->dir_len, + TO_CONN(conn))); + } + vote_done: + smartlist_free(items); + smartlist_free(dir_items); + goto done; + } + done: + return 0; +} + +/** Helper function for GET /tor/micro/d/... + */ +static int +handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + int clear_spool = 1; + { + conn->spool = smartlist_new(); + + dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"), + DIR_SPOOL_MICRODESC, + conn->spool, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + size_t size_guess = 0; + dirserv_spool_remove_missing_and_guess_size(conn, 0, + compress_method != NO_METHOD, + &size_guess, NULL); + if (smartlist_len(conn->spool) == 0) { + write_short_http_response(conn, 404, "Not found"); + goto done; + } + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_short_http_response(conn, 503, "Directory busy, try again later"); + goto done; + } + + clear_spool = 0; + write_http_response_header(conn, -1, + compress_method, + MICRODESC_CACHE_LIFETIME); + + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(size_guess)); + + const int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); + goto done; + } + + done: + if (clear_spool) { + dir_conn_clear_spool(conn); + } + return 0; +} + +/** Helper function for GET /tor/{server,extra}/... + */ +static int +handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + const or_options_t *options = get_options(); + int clear_spool = 1; + if (!strcmpstart(url,"/tor/server/") || + (!options->BridgeAuthoritativeDir && + !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { + int res; + const char *msg = NULL; + int cache_lifetime = 0; + int is_extra = !strcmpstart(url,"/tor/extra/"); + url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/"); + dir_spool_source_t source; + time_t publish_cutoff = 0; + if (!strcmpstart(url, "d/")) { + source = + is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST; + } else { + source = + is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP; + /* We only want to apply a publish cutoff when we're requesting + * resources by fingerprint. */ + publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; + } + + conn->spool = smartlist_new(); + res = dirserv_get_routerdesc_spool(conn->spool, url, + source, + connection_dir_is_encrypted(conn), + &msg); + + if (!strcmpstart(url, "all")) { + cache_lifetime = FULL_DIR_CACHE_LIFETIME; + } else if (smartlist_len(conn->spool) == 1) { + cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; + } + + size_t size_guess = 0; + int n_expired = 0; + dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, + compress_method != NO_METHOD, + &size_guess, &n_expired); + + /* If we are the bridge authority and the descriptor is a bridge + * descriptor, remember that we served this descriptor for desc stats. */ + /* XXXX it's a bit of a kludge to have this here. */ + if (get_options()->BridgeAuthoritativeDir && + source == DIR_SPOOL_SERVER_BY_FP) { + SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) { + const routerinfo_t *router = + router_get_by_id_digest((const char *)spooled->digest); + /* router can be NULL here when the bridge auth is asked for its own + * descriptor. */ + if (router && router->purpose == ROUTER_PURPOSE_BRIDGE) + rep_hist_note_desc_served(router->cache_info.identity_digest); + } SMARTLIST_FOREACH_END(spooled); + } + + if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) { + if (msg == NULL) + msg = "Not found"; + write_short_http_response(conn, 404, msg); + } else { + if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_short_http_response(conn, 503, + "Directory busy, try again later"); + dir_conn_clear_spool(conn); + goto done; + } + write_http_response_header(conn, -1, compress_method, cache_lifetime); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(size_guess)); + clear_spool = 0; + /* Prime the connection with some data. */ + int initial_flush_result = connection_dirserv_flushed_some(conn); + tor_assert_nonfatal(initial_flush_result == 0); + } + goto done; + } + done: + if (clear_spool) + dir_conn_clear_spool(conn); + return 0; +} + +/** Helper function for GET /tor/keys/... + */ +static int +handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + const time_t if_modified_since = args->if_modified_since; + { + smartlist_t *certs = smartlist_new(); + ssize_t len = -1; + if (!strcmp(url, "/tor/keys/all")) { + authority_cert_get_all(certs); + } else if (!strcmp(url, "/tor/keys/authority")) { + authority_cert_t *cert = get_my_v3_authority_cert(); + if (cert) + smartlist_add(certs, cert); + } else if (!strcmpstart(url, "/tor/keys/fp/")) { + smartlist_t *fps = smartlist_new(); + dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); + SMARTLIST_FOREACH(fps, char *, d, { + authority_cert_t *c = authority_cert_get_newest_by_id(d); + if (c) smartlist_add(certs, c); + tor_free(d); + }); + smartlist_free(fps); + } else if (!strcmpstart(url, "/tor/keys/sk/")) { + smartlist_t *fps = smartlist_new(); + dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); + SMARTLIST_FOREACH(fps, char *, d, { + authority_cert_t *c = authority_cert_get_by_sk_digest(d); + if (c) smartlist_add(certs, c); + tor_free(d); + }); + smartlist_free(fps); + } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) { + smartlist_t *fp_sks = smartlist_new(); + dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"), + fp_sks); + SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, { + authority_cert_t *c = authority_cert_get_by_digests(pair->first, + pair->second); + if (c) smartlist_add(certs, c); + tor_free(pair); + }); + smartlist_free(fp_sks); + } else { + write_short_http_response(conn, 400, "Bad request"); + goto keys_done; + } + if (!smartlist_len(certs)) { + write_short_http_response(conn, 404, "Not found"); + goto keys_done; + } + SMARTLIST_FOREACH(certs, authority_cert_t *, c, + if (c->cache_info.published_on < if_modified_since) + SMARTLIST_DEL_CURRENT(certs, c)); + if (!smartlist_len(certs)) { + write_short_http_response(conn, 304, "Not modified"); + goto keys_done; + } + len = 0; + SMARTLIST_FOREACH(certs, authority_cert_t *, c, + len += c->cache_info.signed_descriptor_len); + + if (global_write_bucket_low(TO_CONN(conn), + compress_method != NO_METHOD ? len/2 : len, + 2)) { + write_short_http_response(conn, 503, "Directory busy, try again later"); + goto keys_done; + } + + write_http_response_header(conn, + compress_method != NO_METHOD ? -1 : len, + compress_method, + 60*60); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(len)); + SMARTLIST_FOREACH(certs, authority_cert_t *, c, + connection_buf_add_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_buf_add_compress("", 0, conn, 1); + } else { + SMARTLIST_FOREACH(certs, authority_cert_t *, c, + connection_buf_add(c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + TO_CONN(conn))); + } + keys_done: + smartlist_free(certs); + goto done; + } + done: + return 0; +} + +/** Helper function for GET /tor/rendezvous2/ + */ +static int +handle_get_hs_descriptor_v2(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const char *url = args->url; + if (connection_dir_is_encrypted(conn)) { + /* Handle v2 rendezvous descriptor fetch request. */ + const char *descp; + const char *query = url + strlen("/tor/rendezvous2/"); + if (rend_valid_descriptor_id(query)) { + log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'", + safe_str(escaped(query))); + switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { + case 1: /* valid */ + write_http_response_header(conn, strlen(descp), NO_METHOD, 0); + connection_buf_add(descp, strlen(descp), TO_CONN(conn)); + break; + case 0: /* well-formed but not present */ + write_short_http_response(conn, 404, "Not found"); + break; + case -1: /* not well-formed */ + write_short_http_response(conn, 400, "Bad request"); + break; + } + } else { /* not well-formed */ + write_short_http_response(conn, 400, "Bad request"); + } + goto done; + } else { + /* Not encrypted! */ + write_short_http_response(conn, 404, "Not found"); + } + done: + return 0; +} + +/** Helper function for GET /tor/hs/3/<z>. Only for version 3. + */ +STATIC int +handle_get_hs_descriptor_v3(dir_connection_t *conn, + const get_handler_args_t *args) +{ + int retval; + const char *desc_str = NULL; + const char *pubkey_str = NULL; + const char *url = args->url; + + /* Reject unencrypted dir connections */ + if (!connection_dir_is_encrypted(conn)) { + write_short_http_response(conn, 404, "Not found"); + goto done; + } + + /* After the path prefix follows the base64 encoded blinded pubkey which we + * use to get the descriptor from the cache. Skip the prefix and get the + * pubkey. */ + tor_assert(!strcmpstart(url, "/tor/hs/3/")); + pubkey_str = url + strlen("/tor/hs/3/"); + retval = hs_cache_lookup_as_dir(HS_VERSION_THREE, + pubkey_str, &desc_str); + if (retval <= 0 || desc_str == NULL) { + write_short_http_response(conn, 404, "Not found"); + goto done; + } + + /* Found requested descriptor! Pass it to this nice client. */ + write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0); + connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn)); + + done: + return 0; +} + +/** Helper function for GET /tor/networkstatus-bridges + */ +static int +handle_get_networkstatus_bridges(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const char *headers = args->headers; + + const or_options_t *options = get_options(); + if (options->BridgeAuthoritativeDir && + options->BridgePassword_AuthDigest_ && + connection_dir_is_encrypted(conn)) { + char *status; + char digest[DIGEST256_LEN]; + + char *header = http_get_header(headers, "Authorization: Basic "); + if (header) + crypto_digest256(digest, header, strlen(header), DIGEST_SHA256); + + /* now make sure the password is there and right */ + if (!header || + tor_memneq(digest, + options->BridgePassword_AuthDigest_, DIGEST256_LEN)) { + write_short_http_response(conn, 404, "Not found"); + tor_free(header); + goto done; + } + tor_free(header); + + /* all happy now. send an answer. */ + status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); + size_t dlen = strlen(status); + write_http_response_header(conn, dlen, NO_METHOD, 0); + connection_buf_add(status, dlen, TO_CONN(conn)); + tor_free(status); + goto done; + } + done: + return 0; +} + +/** Helper function for GET robots.txt or /tor/robots.txt */ +static int +handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) +{ + (void)args; + { + const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; + size_t len = strlen(robots); + write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME); + connection_buf_add(robots, len, TO_CONN(conn)); + } + return 0; +} + +/* Given the <b>url</b> from a POST request, try to extract the version number + * using the provided <b>prefix</b>. The version should be after the prefix and + * ending with the separator "/". For instance: + * /tor/hs/3/publish + * + * On success, <b>end_pos</b> points to the position right after the version + * was found. On error, it is set to NULL. + * + * Return version on success else negative value. */ +STATIC int +parse_hs_version_from_post(const char *url, const char *prefix, + const char **end_pos) +{ + int ok; + unsigned long version; + const char *start; + char *end = NULL; + + tor_assert(url); + tor_assert(prefix); + tor_assert(end_pos); + + /* Check if the prefix does start the url. */ + if (strcmpstart(url, prefix)) { + goto err; + } + /* Move pointer to the end of the prefix string. */ + start = url + strlen(prefix); + /* Try this to be the HS version and if we are still at the separator, next + * will be move to the right value. */ + version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end); + if (!ok) { + goto err; + } + + *end_pos = end; + return (int) version; + err: + *end_pos = NULL; + return -1; +} + +/* Handle the POST request for a hidden service descripror. The request is in + * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success + * else return 400 indicating a bad request. */ +STATIC int +handle_post_hs_descriptor(const char *url, const char *body) +{ + int version; + const char *end_pos; + + tor_assert(url); + tor_assert(body); + + version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos); + if (version < 0) { + goto err; + } + + /* We have a valid version number, now make sure it's a publish request. Use + * the end position just after the version and check for the command. */ + if (strcmpstart(end_pos, "/publish")) { + goto err; + } + + switch (version) { + case HS_VERSION_THREE: + if (hs_cache_store_as_dir(body) < 0) { + goto err; + } + log_info(LD_REND, "Publish request for HS descriptor handled " + "successfully."); + break; + default: + /* Unsupported version, return a bad request. */ + goto err; + } + + return 200; + err: + /* Bad request. */ + return 400; +} + +/** Helper function: called when a dirserver gets a complete HTTP POST + * request. Look for an uploaded server descriptor or rendezvous + * service descriptor. On finding one, process it and write a + * response into conn->outbuf. If the request is unrecognized, send a + * 400. Always return 0. */ +MOCK_IMPL(STATIC int, +directory_handle_command_post,(dir_connection_t *conn, const char *headers, + const char *body, size_t body_len)) +{ + char *url = NULL; + const or_options_t *options = get_options(); + + log_debug(LD_DIRSERV,"Received POST command."); + + conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; + + if (!public_server_mode(options)) { + log_info(LD_DIR, "Rejected dir post request from %s " + "since we're not a public relay.", conn->base_.address); + write_short_http_response(conn, 503, "Not acting as a public relay"); + goto done; + } + + if (parse_http_url(headers, &url) < 0) { + write_short_http_response(conn, 400, "Bad request"); + return 0; + } + log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); + + /* Handle v2 rendezvous service publish request. */ + if (connection_dir_is_encrypted(conn) && + !strcmpstart(url,"/tor/rendezvous2/publish")) { + if (rend_cache_store_v2_desc_as_dir(body) < 0) { + log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.", + (int)body_len, conn->base_.address); + write_short_http_response(conn, 400, + "Invalid v2 service descriptor rejected"); + } else { + write_short_http_response(conn, 200, "Service descriptor (v2) stored"); + log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); + } + goto done; + } + + /* Handle HS descriptor publish request. */ + /* XXX: This should be disabled with a consensus param until we want to + * the prop224 be deployed and thus use. */ + if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) { + const char *msg = "HS descriptor stored successfully."; + + /* We most probably have a publish request for an HS descriptor. */ + int code = handle_post_hs_descriptor(url, body); + if (code != 200) { + msg = "Invalid HS descriptor. Rejected."; + } + write_short_http_response(conn, code, msg); + goto done; + } + + if (!authdir_mode(options)) { + /* we just provide cached directories; we don't want to + * receive anything. */ + write_short_http_response(conn, 400, "Nonauthoritative directory does not " + "accept posted server descriptors"); + goto done; + } + + if (authdir_mode(options) && + !strcmp(url,"/tor/")) { /* server descriptor post */ + const char *msg = "[None]"; + uint8_t purpose = authdir_mode_bridge(options) ? + ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; + was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, + conn->base_.address, &msg); + tor_assert(msg); + + if (r == ROUTER_ADDED_SUCCESSFULLY) { + write_short_http_response(conn, 200, msg); + } else if (WRA_WAS_OUTDATED(r)) { + write_http_response_header_impl(conn, -1, NULL, NULL, + "X-Descriptor-Not-New: Yes\r\n", -1); + } else { + log_info(LD_DIRSERV, + "Rejected router descriptor or extra-info from %s " + "("%s").", + conn->base_.address, msg); + write_short_http_response(conn, 400, msg); + } + goto done; + } + + if (authdir_mode_v3(options) && + !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */ + const char *msg = "OK"; + int status; + if (dirvote_add_vote(body, &msg, &status)) { + write_short_http_response(conn, status, "Vote stored"); + } else { + tor_assert(msg); + log_warn(LD_DIRSERV, "Rejected vote from %s ("%s").", + conn->base_.address, msg); + write_short_http_response(conn, status, msg); + } + goto done; + } + + if (authdir_mode_v3(options) && + !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */ + const char *msg = NULL; + if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) { + write_short_http_response(conn, 200, msg?msg:"Signatures stored"); + } else { + log_warn(LD_DIR, "Unable to store signatures posted by %s: %s", + conn->base_.address, msg?msg:"???"); + write_short_http_response(conn, 400, + msg?msg:"Unable to store signatures"); + } + goto done; + } + + /* we didn't recognize the url */ + write_short_http_response(conn, 404, "Not found"); + + done: + tor_free(url); + return 0; +} + +/** If <b>headers</b> indicates that a proxy was involved, then rewrite + * <b>conn</b>->address to describe our best guess of the address that + * originated this HTTP request. */ +static void +http_set_address_origin(const char *headers, connection_t *conn) +{ + char *fwd; + + fwd = http_get_header(headers, "Forwarded-For: "); + if (!fwd) + fwd = http_get_header(headers, "X-Forwarded-For: "); + if (fwd) { + tor_addr_t toraddr; + if (tor_addr_parse(&toraddr,fwd) == -1 || + tor_addr_is_internal(&toraddr,0)) { + log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd)); + tor_free(fwd); + return; + } + + tor_free(conn->address); + conn->address = tor_strdup(fwd); + tor_free(fwd); + } +} + +/** Called when a dirserver receives data on a directory connection; + * looks for an HTTP request. If the request is complete, remove it + * from the inbuf, try to process it; otherwise, leave it on the + * buffer. Return a 0 on success, or -1 on error. + */ +int +directory_handle_command(dir_connection_t *conn) +{ + char *headers=NULL, *body=NULL; + size_t body_len=0; + int r; + + tor_assert(conn); + tor_assert(conn->base_.type == CONN_TYPE_DIR); + + switch (connection_fetch_from_buf_http(TO_CONN(conn), + &headers, MAX_HEADERS_SIZE, + &body, &body_len, MAX_DIR_UL_SIZE, 0)) { + case -1: /* overflow */ + log_warn(LD_DIRSERV, + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->base_.address)); + return -1; + case 0: + log_debug(LD_DIRSERV,"command not all here yet."); + return 0; + /* case 1, fall through */ + } + + http_set_address_origin(headers, TO_CONN(conn)); + // we should escape headers here as well, + // but we can't call escaped() twice, as it uses the same buffer + //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body)); + + if (!strncasecmp(headers,"GET",3)) + r = directory_handle_command_get(conn, headers, body, body_len); + else if (!strncasecmp(headers,"POST",4)) + r = directory_handle_command_post(conn, headers, body, body_len); + else { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Got headers %s with unknown command. Closing.", + escaped(headers)); + r = -1; + } + + tor_free(headers); tor_free(body); + return r; +} diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h new file mode 100644 index 000000000..f05780375 --- /dev/null +++ b/src/feature/dircache/dircache.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dircache.h + * \brief Header file for dircache.c. + **/ + +#ifndef TOR_DIRCACHE_H +#define TOR_DIRCACHE_H + +int directory_handle_command(dir_connection_t *conn); + +#ifdef DIRCACHE_PRIVATE +MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn, + const char *headers, + const char *req_body, + size_t req_body_len)); +MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, + const char *headers, + const char *body, + size_t body_len)); + +STATIC int handle_post_hs_descriptor(const char *url, const char *body); +enum compression_level_t; +STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); + +struct get_handler_args_t; +STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, + const struct get_handler_args_t *args); + +STATIC int parse_http_url(const char *headers, char **url); + +STATIC int parse_hs_version_from_post(const char *url, const char *prefix, + const char **end_pos); + +STATIC unsigned parse_accept_encoding_header(const char *h); +#endif + +#endif /* !defined(TOR_DIRCACHE_H) */ diff --git a/src/feature/dircache/directory.h b/src/feature/dircache/directory.h deleted file mode 100644 index 54ea0c584..000000000 --- a/src/feature/dircache/directory.h +++ /dev/null @@ -1,347 +0,0 @@ -/* Copyright (c) 2001 Matej Pfajfar. - * Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file directory.h - * \brief Header file for directory.c. - **/ - -#ifndef TOR_DIRECTORY_H -#define TOR_DIRECTORY_H - -#include "feature/hs/hs_ident.h" -enum compress_method_t; - -dir_connection_t *TO_DIR_CONN(connection_t *c); - -#define DIR_CONN_STATE_MIN_ 1 -/** State for connection to directory server: waiting for connect(). */ -#define DIR_CONN_STATE_CONNECTING 1 -/** State for connection to directory server: sending HTTP request. */ -#define DIR_CONN_STATE_CLIENT_SENDING 2 -/** State for connection to directory server: reading HTTP response. */ -#define DIR_CONN_STATE_CLIENT_READING 3 -/** State for connection to directory server: happy and finished. */ -#define DIR_CONN_STATE_CLIENT_FINISHED 4 -/** State for connection at directory server: waiting for HTTP request. */ -#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5 -/** State for connection at directory server: sending HTTP response. */ -#define DIR_CONN_STATE_SERVER_WRITING 6 -#define DIR_CONN_STATE_MAX_ 6 - -#define DIR_PURPOSE_MIN_ 4 -/** A connection to a directory server: set after a v2 rendezvous - * descriptor is downloaded. */ -#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4 -/** A connection to a directory server: download one or more server - * descriptors. */ -#define DIR_PURPOSE_FETCH_SERVERDESC 6 -/** A connection to a directory server: download one or more extra-info - * documents. */ -#define DIR_PURPOSE_FETCH_EXTRAINFO 7 -/** A connection to a directory server: upload a server descriptor. */ -#define DIR_PURPOSE_UPLOAD_DIR 8 -/** A connection to a directory server: upload a v3 networkstatus vote. */ -#define DIR_PURPOSE_UPLOAD_VOTE 10 -/** A connection to a directory server: upload a v3 consensus signature */ -#define DIR_PURPOSE_UPLOAD_SIGNATURES 11 -/** A connection to a directory server: download one or more v3 networkstatus - * votes. */ -#define DIR_PURPOSE_FETCH_STATUS_VOTE 12 -/** A connection to a directory server: download a v3 detached signatures - * object for a consensus. */ -#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13 -/** A connection to a directory server: download a v3 networkstatus - * consensus. */ -#define DIR_PURPOSE_FETCH_CONSENSUS 14 -/** A connection to a directory server: download one or more directory - * authority certificates. */ -#define DIR_PURPOSE_FETCH_CERTIFICATE 15 - -/** Purpose for connection at a directory server. */ -#define DIR_PURPOSE_SERVER 16 -/** A connection to a hidden service directory server: upload a v2 rendezvous - * descriptor. */ -#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17 -/** A connection to a hidden service directory server: download a v2 rendezvous - * descriptor. */ -#define DIR_PURPOSE_FETCH_RENDDESC_V2 18 -/** A connection to a directory server: download a microdescriptor. */ -#define DIR_PURPOSE_FETCH_MICRODESC 19 -/** A connection to a hidden service directory: upload a v3 descriptor. */ -#define DIR_PURPOSE_UPLOAD_HSDESC 20 -/** A connection to a hidden service directory: fetch a v3 descriptor. */ -#define DIR_PURPOSE_FETCH_HSDESC 21 -/** A connection to a directory server: set after a hidden service descriptor - * is downloaded. */ -#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22 -#define DIR_PURPOSE_MAX_ 22 - -/** True iff <b>p</b> is a purpose corresponding to uploading - * data to a directory server. */ -#define DIR_PURPOSE_IS_UPLOAD(p) \ - ((p)==DIR_PURPOSE_UPLOAD_DIR || \ - (p)==DIR_PURPOSE_UPLOAD_VOTE || \ - (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \ - (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 || \ - (p)==DIR_PURPOSE_UPLOAD_HSDESC) - -int directories_have_accepted_server_descriptor(void); -void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, - dirinfo_type_t type, const char *payload, - size_t payload_len, size_t extrainfo_len); -MOCK_DECL(void, directory_get_from_dirserver, ( - uint8_t dir_purpose, - uint8_t router_purpose, - const char *resource, - int pds_flags, - download_want_authority_t want_authority)); -void directory_get_from_all_authorities(uint8_t dir_purpose, - uint8_t router_purpose, - const char *resource); - -/** Enumeration of ways to connect to a directory server */ -typedef enum { - /** Default: connect over a one-hop Tor circuit. Relays fall back to direct - * DirPort connections, clients, onion services, and bridges do not */ - DIRIND_ONEHOP=0, - /** Connect over a multi-hop anonymizing Tor circuit */ - DIRIND_ANONYMOUS=1, - /** Connect to the DirPort directly */ - DIRIND_DIRECT_CONN, - /** Connect over a multi-hop anonymizing Tor circuit to our dirport */ - DIRIND_ANON_DIRPORT, -} dir_indirection_t; - -int directory_must_use_begindir(const or_options_t *options); - -/** - * A directory_request_t describes the information about a directory request - * at the client side. It describes what we're going to ask for, which - * directory we're going to ask for it, how we're going to contact that - * directory, and (in some cases) what to do with it when we're done. - */ -typedef struct directory_request_t directory_request_t; -directory_request_t *directory_request_new(uint8_t dir_purpose); -void directory_request_free_(directory_request_t *req); -#define directory_request_free(req) \ - FREE_AND_NULL(directory_request_t, directory_request_free_, (req)) -void directory_request_set_or_addr_port(directory_request_t *req, - const tor_addr_port_t *p); -void directory_request_set_dir_addr_port(directory_request_t *req, - const tor_addr_port_t *p); -void directory_request_set_directory_id_digest(directory_request_t *req, - const char *digest); -struct circuit_guard_state_t; -void directory_request_set_guard_state(directory_request_t *req, - struct circuit_guard_state_t *state); -void directory_request_set_router_purpose(directory_request_t *req, - uint8_t router_purpose); -void directory_request_set_indirection(directory_request_t *req, - dir_indirection_t indirection); -void directory_request_set_resource(directory_request_t *req, - const char *resource); -void directory_request_set_payload(directory_request_t *req, - const char *payload, - size_t payload_len); -void directory_request_set_if_modified_since(directory_request_t *req, - time_t if_modified_since); -void directory_request_set_rend_query(directory_request_t *req, - const rend_data_t *query); -void directory_request_upload_set_hs_ident(directory_request_t *req, - const hs_ident_dir_conn_t *ident); -void directory_request_fetch_set_hs_ident(directory_request_t *req, - const hs_ident_dir_conn_t *ident); - -void directory_request_set_routerstatus(directory_request_t *req, - const routerstatus_t *rs); -void directory_request_add_header(directory_request_t *req, - const char *key, - const char *val); -MOCK_DECL(void, directory_initiate_request, (directory_request_t *request)); - -int parse_http_response(const char *headers, int *code, time_t *date, - enum compress_method_t *compression, char **response); -int parse_http_command(const char *headers, - char **command_out, char **url_out); -char *http_get_header(const char *headers, const char *which); - -int connection_dir_is_encrypted(const dir_connection_t *conn); -int connection_dir_reached_eof(dir_connection_t *conn); -int connection_dir_process_inbuf(dir_connection_t *conn); -int connection_dir_finished_flushing(dir_connection_t *conn); -int connection_dir_finished_connecting(dir_connection_t *conn); -void connection_dir_about_to_close(dir_connection_t *dir_conn); - -#define DSR_HEX (1<<0) -#define DSR_BASE64 (1<<1) -#define DSR_DIGEST256 (1<<2) -#define DSR_SORT_UNIQ (1<<3) -int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compressed_out, - int flags); -enum dir_spool_source_t; -int dir_split_resource_into_spoolable(const char *resource, - enum dir_spool_source_t source, - smartlist_t *spool_out, - int *compressed_out, - int flags); -int dir_split_resource_into_fingerprint_pairs(const char *res, - smartlist_t *pairs_out); -char *directory_dump_request_log(void); -void note_request(const char *key, size_t bytes); -int router_supports_extrainfo(const char *identity_digest, int is_authority); - -time_t download_status_increment_failure(download_status_t *dls, - int status_code, const char *item, - int server, time_t now); -time_t download_status_increment_attempt(download_status_t *dls, - const char *item, time_t now); -/** Increment the failure count of the download_status_t <b>dls</b>, with - * the optional status code <b>sc</b>. */ -#define download_status_failed(dls, sc) \ - download_status_increment_failure((dls), (sc), NULL, \ - dir_server_mode(get_options()), \ - time(NULL)) - -void download_status_reset(download_status_t *dls); -int download_status_is_ready(download_status_t *dls, time_t now); -time_t download_status_get_next_attempt_at(const download_status_t *dls); -void download_status_mark_impossible(download_status_t *dl); - -int download_status_get_n_failures(const download_status_t *dls); -int download_status_get_n_attempts(const download_status_t *dls); - -int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource); - -#ifdef DIRECTORY_PRIVATE - -/** A structure to hold arguments passed into each directory response - * handler */ -typedef struct response_handler_args_t { - int status_code; - const char *reason; - const char *body; - size_t body_len; - const char *headers; -} response_handler_args_t; - -struct directory_request_t { - /** - * These fields specify which directory we're contacting. Routerstatus, - * if present, overrides the other fields. - * - * @{ */ - tor_addr_port_t or_addr_port; - tor_addr_port_t dir_addr_port; - char digest[DIGEST_LEN]; - - const routerstatus_t *routerstatus; - /** @} */ - /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what - * kind of operation we'll be doing (upload/download), and of what kind - * of document. */ - uint8_t dir_purpose; - /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo - * and extrainfo docs. */ - uint8_t router_purpose; - /** Enum: determines whether to anonymize, and whether to use dirport or - * orport. */ - dir_indirection_t indirection; - /** Alias to the variable part of the URL for this request */ - const char *resource; - /** Alias to the payload to upload (if any) */ - const char *payload; - /** Number of bytes to upload from payload</b> */ - size_t payload_len; - /** Value to send in an if-modified-since header, or 0 for none. */ - time_t if_modified_since; - /** Hidden-service-specific information v2. */ - const rend_data_t *rend_query; - /** Extra headers to append to the request */ - struct config_line_t *additional_headers; - /** Hidden-service-specific information for v3+. */ - const hs_ident_dir_conn_t *hs_ident; - /** Used internally to directory.c: gets informed when the attempt to - * connect to the directory succeeds or fails, if that attempt bears on the - * directory's usability as a directory guard. */ - struct circuit_guard_state_t *guard_state; -}; - -struct get_handler_args_t; -STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn, - const struct get_handler_args_t *args); -STATIC int directory_handle_command(dir_connection_t *conn); -STATIC char *accept_encoding_header(void); -STATIC int allowed_anonymous_connection_compression_method( - enum compress_method_t); -STATIC void warn_disallowed_anonymous_compression_method( - enum compress_method_t); - -STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn, - const response_handler_args_t *args); -STATIC int handle_response_fetch_microdesc(dir_connection_t *conn, - const response_handler_args_t *args); - -STATIC int handle_response_fetch_consensus(dir_connection_t *conn, - const response_handler_args_t *args); - -#endif /* defined(DIRECTORY_PRIVATE) */ - -#ifdef TOR_UNIT_TESTS -/* Used only by test_dir.c and test_hs_cache.c */ - -STATIC int parse_http_url(const char *headers, char **url); -STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose, - const char *resource); -MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn, - const char *headers, - const char *req_body, - size_t req_body_len)); -MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn, - const char *headers, - const char *body, - size_t body_len)); -STATIC int download_status_schedule_get_delay(download_status_t *dls, - int min_delay, - time_t now); - -STATIC int handle_post_hs_descriptor(const char *url, const char *body); - -STATIC char* authdir_type_to_string(dirinfo_type_t auth); -STATIC const char * dir_conn_purpose_to_string(int purpose); -STATIC int should_use_directory_guards(const or_options_t *options); -enum compression_level_t; -STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes); -STATIC int find_dl_min_delay(const download_status_t *dls, - const or_options_t *options); - -STATIC int next_random_exponential_delay(int delay, - int base_delay); - -STATIC void next_random_exponential_delay_range(int *low_bound_out, - int *high_bound_out, - int delay, - int base_delay); - -STATIC int parse_hs_version_from_post(const char *url, const char *prefix, - const char **end_pos); - -STATIC unsigned parse_accept_encoding_header(const char *h); -#endif /* defined(TOR_UNIT_TESTS) */ - -#if defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE) -/* Used only by directory.c and test_dir.c */ - -/* no more than quadruple the previous delay (multiplier + 1) */ -#define DIR_DEFAULT_RANDOM_MULTIPLIER (3) -/* no more than triple the previous delay */ -#define DIR_TEST_NET_RANDOM_MULTIPLIER (2) - -#endif /* defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE) */ - -#endif /* !defined(TOR_DIRECTORY_H) */ diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index 2fd2572a2..65c3d6d91 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -10,7 +10,7 @@ #include "core/mainloop/connection.h" #include "feature/dircache/conscache.h" #include "feature/dircache/consdiffmgr.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/dircache/dirserv.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/routerlist.h" diff --git a/src/feature/dircache/directory.c b/src/feature/dirclient/dirclient.c similarity index 54% rename from src/feature/dircache/directory.c rename to src/feature/dirclient/dirclient.c index 7066e92c2..0f5f87031 100644 --- a/src/feature/dircache/directory.c +++ b/src/feature/dirclient/dirclient.c @@ -3,131 +3,68 @@ * Copyright (c) 2007-2018, The Tor Project, Inc. */ /* See LICENSE for licensing information */

-#define DIRECTORY_PRIVATE +#define DIRCLIENT_PRIVATE

#include "core/or/or.h" -#include "lib/err/backtrace.h" -#include "feature/client/bridges.h" -#include "lib/container/buffers.h" -#include "core/or/circuitbuild.h" + #include "app/config/config.h" #include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" #include "core/or/connection_edge.h" -#include "feature/dircache/conscache.h" -#include "feature/dircommon/consdiff.h" -#include "feature/dircache/consdiffmgr.h" +#include "core/or/policies.h" +#include "feature/client/bridges.h" +#include "feature/client/entrynodes.h" #include "feature/control/control.h" -#include "lib/compress/compress.h" -#include "lib/crypt_ops/crypto_rand.h" -#include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/mode.h" +#include "feature/dirauth/shared_random.h" #include "feature/dircache/dirserv.h" -#include "feature/dirauth/process_descs.h" -#include "feature/client/entrynodes.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/dircommon/consdiff.h" +#include "feature/dircommon/directory.h" #include "feature/dircommon/fp_pair.h" -#include "feature/stats/geoip.h" #include "feature/hs/hs_cache.h" -#include "feature/hs/hs_common.h" -#include "feature/hs/hs_control.h" #include "feature/hs/hs_client.h" -#include "core/mainloop/mainloop.h" +#include "feature/hs/hs_control.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/node_select.h" #include "feature/nodelist/nodelist.h" -#include "core/or/policies.h" -#include "core/or/relay.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerset.h" +#include "feature/relay/router.h" +#include "feature/rend/rendcache.h" #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/rend/rendservice.h" #include "feature/stats/rephist.h" -#include "feature/relay/router.h" -#include "feature/nodelist/authcert.h" -#include "feature/nodelist/dirlist.h" -#include "feature/nodelist/node_select.h" -#include "feature/nodelist/routerlist.h"

-#include "feature/nodelist/routerparse.h" -#include "feature/nodelist/routerset.h" -#include "lib/encoding/confline.h" +#include "lib/compress/compress.h" #include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/encoding/confline.h" +#include "lib/err/backtrace.h"

-#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) -#if !defined(OpenBSD) -#include <malloc.h> -#endif -#endif - -#include "feature/dirauth/dirvote.h" -#include "feature/dirauth/mode.h" -#include "feature/dirauth/shared_random.h" - -#include "feature/nodelist/authority_cert_st.h" +#include "core/or/entry_connection_st.h" #include "feature/dircache/cached_dir_st.h" -#include "feature/dircommon/dir_connection_st.h" #include "feature/dirclient/dir_server_st.h" -#include "core/or/entry_connection_st.h" +#include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" -#include "feature/rend/rend_service_descriptor_st.h" #include "feature/nodelist/routerinfo_st.h" +#include "feature/rend/rend_service_descriptor_st.h"

-/** - * \file directory.c - * \brief Code to send and fetch information from directory authorities and - * caches via HTTP. - * - * Directory caches and authorities use dirserv.c to generate the results of a - * query and stream them to the connection; clients use routerparse.c to parse - * them. - * - * Every directory request has a dir_connection_t on the client side and on - * the server side. In most cases, the dir_connection_t object is a linked - * connection, tunneled through an edge_connection_t so that it can be a - * stream on the Tor network. The only non-tunneled connections are those - * that are used to upload material (descriptors and votes) to authorities. - * Among tunneled connections, some use one-hop circuits, and others use - * multi-hop circuits for anonymity. - * - * Directory requests are launched by calling - * directory_initiate_request(). This - * launch the connection, will construct an HTTP request with - * directory_send_command(), send the and wait for a response. The client - * later handles the response with connection_dir_client_reached_eof(), - * which passes the information received to another part of Tor. - * - * On the server side, requests are read in directory_handle_command(), - * which dispatches first on the request type (GET or POST), and then on - * the URL requested. GET requests are processed with a table-based - * dispatcher in url_table[]. The process of handling larger GET requests - * is complicated because we need to avoid allocating a copy of all the - * data to be sent to the client in one huge buffer. Instead, we spool the - * data into the buffer using logic in connection_dirserv_flushed_some() in - * dirserv.c. (TODO: If we extended buf.c to have a zero-copy - * reference-based buffer type, we could remove most of that code, at the - * cost of a bit more reference counting.) - **/ +/** Maximum size, in bytes, for any directory object that we've downloaded. */ +#define MAX_DIR_DL_SIZE ((1<<24)-1) /* 16 MB - 1 */ + +/** How far in the future do we allow a directory server to tell us it is + * before deciding that one of us has the wrong time? */ +#define ALLOW_DIRECTORY_TIME_SKEW (30*60)

-/* In-points to directory.c: - * - * - directory_post_to_dirservers(), called from - * router_upload_dir_desc_to_dirservers() in router.c - * upload_service_descriptor() in rendservice.c - * - directory_get_from_dirserver(), called from - * rend_client_refetch_renddesc() in rendclient.c - * run_scheduled_events() in main.c - * do_hup() in main.c - * - connection_dir_process_inbuf(), called from - * connection_process_inbuf() in connection.c - * - connection_dir_finished_flushing(), called from - * connection_finished_flushing() in connection.c - * - connection_dir_finished_connecting(), called from - * connection_finished_connecting() in connection.c - */ -static void directory_send_command(dir_connection_t *conn, - int direct, - const directory_request_t *request); static int body_is_plausible(const char *body, size_t body_len, int purpose); -static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( @@ -141,126 +78,12 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, static void dir_microdesc_download_failed(smartlist_t *failed, int status_code, const char *dir_id); -static int client_likes_consensus(const struct consensus_cache_entry_t *ent, - const char *want_url); - +static void directory_send_command(dir_connection_t *conn, + const int direct, + const directory_request_t *req); static void connection_dir_close_consensus_fetches( dir_connection_t *except_this_one, const char *resource);

-/********* START VARIABLES **********/ - -/** Maximum size, in bytes, for resized buffers. */ -#define MAX_BUF_SIZE ((1<<24)-1) /* 16MB-1 */ -/** Maximum size, in bytes, for any directory object that we've downloaded. */ -#define MAX_DIR_DL_SIZE MAX_BUF_SIZE - -/** Maximum size, in bytes, for any directory object that we're accepting - * as an upload. */ -#define MAX_DIR_UL_SIZE MAX_BUF_SIZE - -/** How far in the future do we allow a directory server to tell us it is - * before deciding that one of us has the wrong time? */ -#define ALLOW_DIRECTORY_TIME_SKEW (30*60) - -#define X_ADDRESS_HEADER "X-Your-Address-Is: " -#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: " - -/** HTTP cache control: how long do we tell proxies they can cache each - * kind of document we serve? */ -#define FULL_DIR_CACHE_LIFETIME (60*60) -#define RUNNINGROUTERS_CACHE_LIFETIME (20*60) -#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60) -#define NETWORKSTATUS_CACHE_LIFETIME (5*60) -#define ROUTERDESC_CACHE_LIFETIME (30*60) -#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) -#define ROBOTS_CACHE_LIFETIME (24*60*60) -#define MICRODESC_CACHE_LIFETIME (48*60*60) - -/********* END VARIABLES ************/ - -/** Convert a connection_t* to a dir_connection_t*; assert if the cast is - * invalid. */ -dir_connection_t * -TO_DIR_CONN(connection_t *c) -{ - tor_assert(c->magic == DIR_CONNECTION_MAGIC); - return DOWNCAST(dir_connection_t, c); -} - -/** Return false if the directory purpose <b>dir_purpose</b> - * does not require an anonymous (three-hop) connection. - * - * Return true 1) by default, 2) if all directory actions have - * specifically been configured to be over an anonymous connection, - * or 3) if the router is a bridge */ -int -purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource) -{ - if (get_options()->AllDirActionsPrivate) - return 1; - - if (router_purpose == ROUTER_PURPOSE_BRIDGE) { - if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC - && resource && !strcmp(resource, "authority.z")) { - /* We are asking a bridge for its own descriptor. That doesn't need - anonymity. */ - return 0; - } - /* Assume all other bridge stuff needs anonymity. */ - return 1; /* if no circuits yet, this might break bootstrapping, but it's - * needed to be safe. */ - } - - switch (dir_purpose) - { - case DIR_PURPOSE_UPLOAD_DIR: - case DIR_PURPOSE_UPLOAD_VOTE: - case DIR_PURPOSE_UPLOAD_SIGNATURES: - case DIR_PURPOSE_FETCH_STATUS_VOTE: - case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: - case DIR_PURPOSE_FETCH_CONSENSUS: - case DIR_PURPOSE_FETCH_CERTIFICATE: - case DIR_PURPOSE_FETCH_SERVERDESC: - case DIR_PURPOSE_FETCH_EXTRAINFO: - case DIR_PURPOSE_FETCH_MICRODESC: - return 0; - case DIR_PURPOSE_HAS_FETCHED_HSDESC: - case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2: - case DIR_PURPOSE_UPLOAD_RENDDESC_V2: - case DIR_PURPOSE_FETCH_RENDDESC_V2: - case DIR_PURPOSE_FETCH_HSDESC: - case DIR_PURPOSE_UPLOAD_HSDESC: - return 1; - case DIR_PURPOSE_SERVER: - default: - log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d", - dir_purpose, router_purpose); - tor_assert_nonfatal_unreached(); - return 1; /* Assume it needs anonymity; better safe than sorry. */ - } -} - -/** Return a newly allocated string describing <b>auth</b>. Only describes - * authority features. */ -STATIC char * -authdir_type_to_string(dirinfo_type_t auth) -{ - char *result; - smartlist_t *lst = smartlist_new(); - if (auth & V3_DIRINFO) - smartlist_add(lst, (void*)"V3"); - if (auth & BRIDGE_DIRINFO) - smartlist_add(lst, (void*)"Bridge"); - if (smartlist_len(lst)) { - result = smartlist_join_strings(lst, ", ", 0, NULL); - } else { - result = tor_strdup("[Not an authority]"); - } - smartlist_free(lst); - return result; -} - /** Return a string describing a given directory connection purpose. */ STATIC const char * dir_conn_purpose_to_string(int purpose) @@ -606,7 +429,6 @@ dir_consensus_request_set_additional_headers(directory_request_t *req, directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex); } } - /** Start a connection to a random running directory server, using * connection purpose <b>dir_purpose</b>, intending to fetch descriptors * of purpose <b>router_purpose</b>, and requesting <b>resource</b>. @@ -890,8 +712,8 @@ directory_conn_is_self_reachability_test(dir_connection_t *conn) * server due to a network error: Mark the router as down and try again if * possible. */ -static void -connection_dir_request_failed(dir_connection_t *conn) +void +connection_dir_client_request_failed(dir_connection_t *conn) { if (conn->guard_state) { /* We haven't seen a success on this guard state, so consider it to have @@ -1608,22 +1430,11 @@ directory_initiate_request,(directory_request_t *request)) } }

-/** Return true iff anything we say on <b>conn</b> is being encrypted before - * we send it to the client/server. */ -int -connection_dir_is_encrypted(const dir_connection_t *conn) -{ - /* Right now it's sufficient to see if conn is or has been linked, since - * the only thing it could be linked to is an edge connection on a - * circuit, and the only way it could have been unlinked is at the edge - * connection getting closed. - */ - return TO_CONN(conn)->linked; -} - /** Helper for sorting * * sort strings alphabetically + * + * XXXX we have a smartlist_sort_strings() function, right? */ static int compare_strs_(const void **a, const void **b) @@ -1952,226 +1763,6 @@ directory_send_command(dir_connection_t *conn, (payload ? payload_len : 0)); }

-/** Parse an HTTP request string <b>headers</b> of the form - * \verbatim - * "%s [http[s]://]%s HTTP/1..." - * \endverbatim - * If it's well-formed, strdup the second %s into *<b>url</b>, and - * nul-terminate it. If the url doesn't start with "/tor/", rewrite it - * so it does. Return 0. - * Otherwise, return -1. - */ -STATIC int -parse_http_url(const char *headers, char **url) -{ - char *command = NULL; - if (parse_http_command(headers, &command, url) < 0) { - return -1; - } - if (strcmpstart(*url, "/tor/")) { - char *new_url = NULL; - tor_asprintf(&new_url, "/tor%s%s", - *url[0] == '/' ? "" : "/", - *url); - tor_free(*url); - *url = new_url; - } - tor_free(command); - return 0; -} - -/** Parse an HTTP request line at the start of a headers string. On failure, - * return -1. On success, set *<b>command_out</b> to a copy of the HTTP - * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and - * return 0. */ -int -parse_http_command(const char *headers, char **command_out, char **url_out) -{ - const char *command, *end_of_command; - char *s, *start, *tmp; - - s = (char *)eat_whitespace_no_nl(headers); - if (!*s) return -1; - command = s; - s = (char *)find_whitespace(s); /* get past GET/POST */ - if (!*s) return -1; - end_of_command = s; - s = (char *)eat_whitespace_no_nl(s); - if (!*s) return -1; - start = s; /* this is the URL, assuming it's valid */ - s = (char *)find_whitespace(start); - if (!*s) return -1; - - /* tolerate the http[s] proxy style of putting the hostname in the url */ - if (s-start >= 4 && !strcmpstart(start,"http")) { - tmp = start + 4; - if (*tmp == 's') - tmp++; - if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { - tmp = strchr(tmp+3, '/'); - if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); - start = tmp; - } - } - } - - /* Check if the header is well formed (next sequence - * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */ - { - unsigned minor_ver; - char ch; - char *e = (char *)eat_whitespace_no_nl(s); - if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) { - return -1; - } - if (ch != '\r') - return -1; - } - - *url_out = tor_memdup_nulterm(start, s-start); - *command_out = tor_memdup_nulterm(command, end_of_command - command); - return 0; -} - -/** Return a copy of the first HTTP header in <b>headers</b> whose key is - * <b>which</b>. The key should be given with a terminating colon and space; - * this function copies everything after, up to but not including the - * following \r\n. */ -char * -http_get_header(const char *headers, const char *which) -{ - const char *cp = headers; - while (cp) { - if (!strcasecmpstart(cp, which)) { - char *eos; - cp += strlen(which); - if ((eos = strchr(cp,'\r'))) - return tor_strndup(cp, eos-cp); - else - return tor_strdup(cp); - } - cp = strchr(cp, '\n'); - if (cp) - ++cp; - } - return NULL; -} - -/** If <b>headers</b> indicates that a proxy was involved, then rewrite - * <b>conn</b>->address to describe our best guess of the address that - * originated this HTTP request. */ -static void -http_set_address_origin(const char *headers, connection_t *conn) -{ - char *fwd; - - fwd = http_get_header(headers, "Forwarded-For: "); - if (!fwd) - fwd = http_get_header(headers, "X-Forwarded-For: "); - if (fwd) { - tor_addr_t toraddr; - if (tor_addr_parse(&toraddr,fwd) == -1 || - tor_addr_is_internal(&toraddr,0)) { - log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd)); - tor_free(fwd); - return; - } - - tor_free(conn->address); - conn->address = tor_strdup(fwd); - tor_free(fwd); - } -} - -/** Parse an HTTP response string <b>headers</b> of the form - * \verbatim - * "HTTP/1.%d %d%s\r\n...". - * \endverbatim - * - * If it's well-formed, assign the status code to *<b>code</b> and - * return 0. Otherwise, return -1. - * - * On success: If <b>date</b> is provided, set *date to the Date - * header in the http headers, or 0 if no such header is found. If - * <b>compression</b> is provided, set *<b>compression</b> to the - * compression method given in the Content-Encoding header, or 0 if no - * such header is found, or -1 if the value of the header is not - * recognized. If <b>reason</b> is provided, strdup the reason string - * into it. - */ -int -parse_http_response(const char *headers, int *code, time_t *date, - compress_method_t *compression, char **reason) -{ - unsigned n1, n2; - char datestr[RFC1123_TIME_LEN+1]; - smartlist_t *parsed_headers; - tor_assert(headers); - tor_assert(code); - - while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */ - - if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 || - (n1 != 0 && n1 != 1) || - (n2 < 100 || n2 >= 600)) { - log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers)); - return -1; - } - *code = n2; - - parsed_headers = smartlist_new(); - smartlist_split_string(parsed_headers, headers, "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - if (reason) { - smartlist_t *status_line_elements = smartlist_new(); - tor_assert(smartlist_len(parsed_headers)); - smartlist_split_string(status_line_elements, - smartlist_get(parsed_headers, 0), - " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); - tor_assert(smartlist_len(status_line_elements) <= 3); - if (smartlist_len(status_line_elements) == 3) { - *reason = smartlist_get(status_line_elements, 2); - smartlist_set(status_line_elements, 2, NULL); /* Prevent free */ - } - SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp)); - smartlist_free(status_line_elements); - } - if (date) { - *date = 0; - SMARTLIST_FOREACH(parsed_headers, const char *, s, - if (!strcmpstart(s, "Date: ")) { - strlcpy(datestr, s+6, sizeof(datestr)); - /* This will do nothing on failure, so we don't need to check - the result. We shouldn't warn, since there are many other valid - date formats besides the one we use. */ - parse_rfc1123_time(datestr, date); - break; - }); - } - if (compression) { - const char *enc = NULL; - SMARTLIST_FOREACH(parsed_headers, const char *, s, - if (!strcmpstart(s, "Content-Encoding: ")) { - enc = s+18; break; - }); - - if (enc == NULL) - *compression = NO_METHOD; - else { - *compression = compression_method_get_by_name(enc); - - if (*compression == UNKNOWN_METHOD) - log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", - escaped(enc)); - } - } - SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s)); - smartlist_free(parsed_headers); - - return 0; -} - /** Return true iff <b>body</b> doesn't start with a plausible router or * network-status or microdescriptor opening. This is a sign of possible * compression. */ @@ -3399,63 +2990,11 @@ connection_dir_reached_eof(dir_connection_t *conn) connection_mark_for_close(TO_CONN(conn)); return retval; } - -/** If any directory object is arriving, and it's over 10MB large, we're - * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never - * ask for more than 96 router descriptors at a time.) - */ -#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20)) - -#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5) - -/** Read handler for directory connections. (That's connections <em>to</em> - * directory servers and connections <em>at</em> directory servers.) - */ -int -connection_dir_process_inbuf(dir_connection_t *conn) -{ - size_t max_size; - tor_assert(conn); - tor_assert(conn->base_.type == CONN_TYPE_DIR); - - /* Directory clients write, then read data until they receive EOF; - * directory servers read data until they get an HTTP command, then - * write their response (when it's finished flushing, they mark for - * close). - */ - - /* If we're on the dirserver side, look for a command. */ - if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) { - if (directory_handle_command(conn) < 0) { - connection_mark_for_close(TO_CONN(conn)); - return -1; - } - return 0; - } - - max_size = - (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ? - MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE; - - if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) { - log_warn(LD_HTTP, - "Too much data received from directory connection (%s): " - "denial of service attempt, or you need to upgrade?", - conn->base_.address); - connection_mark_for_close(TO_CONN(conn)); - return -1; - } - - if (!conn->base_.inbuf_reached_eof) - log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf."); - return 0; -} - /** We are closing a dir connection: If <b>dir_conn</b> is a dir connection * that tried to fetch an HS descriptor, check if it successfully fetched it, * or if we need to try again. */ -static void -refetch_hsdesc_if_needed(dir_connection_t *dir_conn) +void +connection_dir_client_refetch_hsdesc_if_needed(dir_connection_t *dir_conn) { connection_t *conn = TO_CONN(dir_conn);

@@ -3478,1776 +3017,78 @@ refetch_hsdesc_if_needed(dir_connection_t *dir_conn) } }

-/** Called when we're about to finally unlink and free a directory connection: - * perform necessary accounting and cleanup */ -void -connection_dir_about_to_close(dir_connection_t *dir_conn) -{ - connection_t *conn = TO_CONN(dir_conn); - - if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) { - /* It's a directory connection and connecting or fetching - * failed: forget about this router, and maybe try again. */ - connection_dir_request_failed(dir_conn); - } +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +};

- refetch_hsdesc_if_needed(dir_conn); -} +/** Array of allowed compression methods to use (if supported) when receiving a + * response from a request that was required to be anonymous. */ +static compress_method_t client_meth_allowed_anonymous_compression[] = { + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +};

-/** Create an http response for the client <b>conn</b> out of - * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>. - */ -static void -write_short_http_response(dir_connection_t *conn, int status, - const char *reason_phrase) +/** Return a newly allocated string containing a comma separated list of + * supported encodings. */ +STATIC char * +accept_encoding_header(void) { - char *buf = NULL; - char *datestring = NULL; - - IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */ - reason_phrase = "unspecified"; - } + smartlist_t *methods = smartlist_new(); + char *header = NULL; + compress_method_t method; + unsigned i;

- if (server_mode(get_options())) { - /* include the Date: header, but only if we're a relay or bridge */ - char datebuf[RFC1123_TIME_LEN+1]; - format_rfc1123_time(datebuf, time(NULL)); - tor_asprintf(&datestring, "Date: %s\r\n", datebuf); + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; + if (tor_compress_supports_method(method)) + smartlist_add(methods, (char *)compression_method_get_name(method)); }

- tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n", - status, reason_phrase, datestring?datestring:""); - - log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); - connection_buf_add(buf, strlen(buf), TO_CONN(conn)); + header = smartlist_join_strings(methods, ", ", 0, NULL); + smartlist_free(methods);

- tor_free(datestring); - tor_free(buf); + return header; }

-/** Write the header for an HTTP/1.0 response onto <b>conn</b>->outbuf, - * with <b>type</b> as the Content-Type. - * - * If <b>length</b> is nonnegative, it is the Content-Length. - * If <b>encoding</b> is provided, it is the Content-Encoding. - * If <b>cache_lifetime</b> is greater than 0, the content may be cached for - * up to cache_lifetime seconds. Otherwise, the content may not be cached. */ -static void -write_http_response_header_impl(dir_connection_t *conn, ssize_t length, - const char *type, const char *encoding, - const char *extra_headers, - long cache_lifetime) +/** Check if the given compression method is allowed for a connection that is + * supposed to be anonymous. Returns 1 if the compression method is allowed, + * otherwise 0. */ +STATIC int +allowed_anonymous_connection_compression_method(compress_method_t method) { - char date[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); - buf_t *buf = buf_new_with_capacity(1024); - - tor_assert(conn); - - format_rfc1123_time(date, now); - - buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date); - if (type) { - buf_add_printf(buf, "Content-Type: %s\r\n", type); - } - if (!is_local_addr(&conn->base_.addr)) { - /* Don't report the source address for a nearby/private connection. - * Otherwise we tend to mis-report in cases where incoming ports are - * being forwarded to a Tor server running behind the firewall. */ - buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address); - } - if (encoding) { - buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding); - } - if (length >= 0) { - buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length); - } - if (cache_lifetime > 0) { - char expbuf[RFC1123_TIME_LEN+1]; - format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime)); - /* We could say 'Cache-control: max-age=%d' here if we start doing - * http/1.1 */ - buf_add_printf(buf, "Expires: %s\r\n", expbuf); - } else if (cache_lifetime == 0) { - /* We could say 'Cache-control: no-cache' here if we start doing - * http/1.1 */ - buf_add_string(buf, "Pragma: no-cache\r\n"); - } - if (extra_headers) { - buf_add_string(buf, extra_headers); - } - buf_add_string(buf, "\r\n"); - - connection_buf_add_buf(TO_CONN(conn), buf); - buf_free(buf); -} - -/** As write_http_response_header_impl, but sets encoding and content-typed - * based on whether the response will be <b>compressed</b> or not. */ -static void -write_http_response_headers(dir_connection_t *conn, ssize_t length, - compress_method_t method, - const char *extra_headers, long cache_lifetime) -{ - const char *methodname = compression_method_get_name(method); - const char *doctype; - if (method == NO_METHOD) - doctype = "text/plain"; - else - doctype = "application/octet-stream"; - write_http_response_header_impl(conn, length, - doctype, - methodname, - extra_headers, - cache_lifetime); -} - -/** As write_http_response_headers, but assumes extra_headers is NULL */ -static void -write_http_response_header(dir_connection_t *conn, ssize_t length, - compress_method_t method, - long cache_lifetime) -{ - write_http_response_headers(conn, length, method, NULL, cache_lifetime); -} - -/** Array of compression methods to use (if supported) for serving - * precompressed data, ordered from best to worst. */ -static compress_method_t srv_meth_pref_precompressed[] = { - LZMA_METHOD, - ZSTD_METHOD, - ZLIB_METHOD, - GZIP_METHOD, - NO_METHOD -}; - -/** Array of compression methods to use (if supported) for serving - * streamed data, ordered from best to worst. */ -static compress_method_t srv_meth_pref_streaming_compression[] = { - ZSTD_METHOD, - ZLIB_METHOD, - GZIP_METHOD, - NO_METHOD -}; - -/** Array of allowed compression methods to use (if supported) when receiving a - * response from a request that was required to be anonymous. */ -static compress_method_t client_meth_allowed_anonymous_compression[] = { - ZLIB_METHOD, - GZIP_METHOD, - NO_METHOD -}; - -/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>, - * and convert them to a bitfield where compression method x is supported if - * and only if 1 &lt;&lt; x is set in the bitfield. */ -STATIC unsigned -parse_accept_encoding_header(const char *h) -{ - unsigned result = (1u << NO_METHOD); - smartlist_t *methods = smartlist_new(); - smartlist_split_string(methods, h, ",", - SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0); - - SMARTLIST_FOREACH_BEGIN(methods, const char *, m) { - compress_method_t method = compression_method_get_by_name(m); - if (method != UNKNOWN_METHOD) { - tor_assert(((unsigned)method) < 8*sizeof(unsigned)); - result |= (1u << method); - } - } SMARTLIST_FOREACH_END(m); - SMARTLIST_FOREACH_BEGIN(methods, char *, m) { - tor_free(m); - } SMARTLIST_FOREACH_END(m); - smartlist_free(methods); - return result; -} - -/** Array of compression methods to use (if supported) for requesting - * compressed data, ordered from best to worst. */ -static compress_method_t client_meth_pref[] = { - LZMA_METHOD, - ZSTD_METHOD, - ZLIB_METHOD, - GZIP_METHOD, - NO_METHOD -}; - -/** Return a newly allocated string containing a comma separated list of - * supported encodings. */ -STATIC char * -accept_encoding_header(void) -{ - smartlist_t *methods = smartlist_new(); - char *header = NULL; - compress_method_t method; - unsigned i; - - for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { - method = client_meth_pref[i]; - if (tor_compress_supports_method(method)) - smartlist_add(methods, (char *)compression_method_get_name(method)); - } - - header = smartlist_join_strings(methods, ", ", 0, NULL); - smartlist_free(methods); - - return header; -} - -/** Decide whether a client would accept the consensus we have. - * - * Clients can say they only want a consensus if it's signed by more - * than half the authorities in a list. They pass this list in - * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>". - * - * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring - * of the full authority identity digest. (Only strings of even length, - * i.e. encodings of full bytes, are handled correctly. In the case - * of an odd number of hex digits the last one is silently ignored.) - * - * Returns 1 if more than half of the requested authorities signed the - * consensus, 0 otherwise. - */ -int -client_likes_consensus(const struct consensus_cache_entry_t *ent, - const char *want_url) -{ - smartlist_t *voters = smartlist_new(); - int need_at_least; - int have = 0; - - if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) { - smartlist_free(voters); - return 1; // We don't know the voters; assume the client won't mind. */ - } - - smartlist_t *want_authorities = smartlist_new(); - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); - need_at_least = smartlist_len(want_authorities)/2+1; - - SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) { - - SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) { - if (!strcasecmpstart(digest, want_digest)) { - have++; - break; - }; - } SMARTLIST_FOREACH_END(digest); - - /* early exit, if we already have enough */ - if (have >= need_at_least) - break; - } SMARTLIST_FOREACH_END(want_digest); - - SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); - smartlist_free(want_authorities); - SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp)); - smartlist_free(voters); - return (have >= need_at_least); -} - -/** Return the compression level we should use for sending a compressed - * response of size <b>n_bytes</b>. */ -STATIC compression_level_t -choose_compression_level(ssize_t n_bytes) -{ - if (! have_been_under_memory_pressure()) { - return HIGH_COMPRESSION; /* we have plenty of RAM. */ - } else if (n_bytes < 0) { - return HIGH_COMPRESSION; /* unknown; might be big. */ - } else if (n_bytes < 1024) { - return LOW_COMPRESSION; - } else if (n_bytes < 2048) { - return MEDIUM_COMPRESSION; - } else { - return HIGH_COMPRESSION; - } -} - -/** Information passed to handle a GET request. */ -typedef struct get_handler_args_t { - /** Bitmask of compression methods that the client said (or implied) it - * supported. */ - unsigned compression_supported; - /** If nonzero, the time included an if-modified-since header with this - * value. */ - time_t if_modified_since; - /** String containing the requested URL or resource. */ - const char *url; - /** String containing the HTTP headers */ - const char *headers; -} get_handler_args_t; - -/** Entry for handling an HTTP GET request. - * - * This entry matches a request if "string" is equal to the requested - * resource, or if "is_prefix" is true and "string" is a prefix of the - * requested resource. - * - * The 'handler' function is called to handle the request. It receives - * an arguments structure, and must return 0 on success or -1 if we should - * close the connection. - **/ -typedef struct url_table_ent_s { - const char *string; - int is_prefix; - int (*handler)(dir_connection_t *conn, const get_handler_args_t *args); -} url_table_ent_t; - -static int handle_get_frontpage(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_current_consensus(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_status_vote(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_microdesc(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_descriptor(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_keys(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_hs_descriptor_v2(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_robots(dir_connection_t *conn, - const get_handler_args_t *args); -static int handle_get_networkstatus_bridges(dir_connection_t *conn, - const get_handler_args_t *args); - -/** Table for handling GET requests. */ -static const url_table_ent_t url_table[] = { - { "/tor/", 0, handle_get_frontpage }, - { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, - { "/tor/status-vote/current/", 1, handle_get_status_vote }, - { "/tor/status-vote/next/", 1, handle_get_status_vote }, - { "/tor/micro/d/", 1, handle_get_microdesc }, - { "/tor/server/", 1, handle_get_descriptor }, - { "/tor/extra/", 1, handle_get_descriptor }, - { "/tor/keys/", 1, handle_get_keys }, - { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 }, - { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 }, - { "/tor/robots.txt", 0, handle_get_robots }, - { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges }, - { NULL, 0, NULL }, -}; - -/** Helper function: called when a dirserver gets a complete HTTP GET - * request. Look for a request for a directory or for a rendezvous - * service descriptor. On finding one, write a response into - * conn->outbuf. If the request is unrecognized, send a 404. - * Return 0 if we handled this successfully, or -1 if we need to close - * the connection. */ -MOCK_IMPL(STATIC int, -directory_handle_command_get,(dir_connection_t *conn, const char *headers, - const char *req_body, size_t req_body_len)) -{ - char *url, *url_mem, *header; - time_t if_modified_since = 0; - int zlib_compressed_in_url; - unsigned compression_methods_supported; - - /* We ignore the body of a GET request. */ - (void)req_body; - (void)req_body_len; - - log_debug(LD_DIRSERV,"Received GET command."); - - conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; - - if (parse_http_url(headers, &url) < 0) { - write_short_http_response(conn, 400, "Bad request"); - return 0; - } - if ((header = http_get_header(headers, "If-Modified-Since: "))) { - struct tm tm; - if (parse_http_time(header, &tm) == 0) { - if (tor_timegm(&tm, &if_modified_since)<0) { - if_modified_since = 0; - } else { - log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header)); - } - } - /* The correct behavior on a malformed If-Modified-Since header is to - * act as if no If-Modified-Since header had been given. */ - tor_free(header); - } - log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); - - url_mem = url; - { - size_t url_len = strlen(url); - - zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z"); - if (zlib_compressed_in_url) { - url[url_len-2] = '\0'; - } - } - - if ((header = http_get_header(headers, "Accept-Encoding: "))) { - compression_methods_supported = parse_accept_encoding_header(header); - tor_free(header); - } else { - compression_methods_supported = (1u << NO_METHOD); - } - if (zlib_compressed_in_url) { - compression_methods_supported |= (1u << ZLIB_METHOD); - } - - /* Remove all methods that we don't both support. */ - compression_methods_supported &= tor_compress_get_supported_method_bitmask(); - - get_handler_args_t args; - args.url = url; - args.headers = headers; - args.if_modified_since = if_modified_since; - args.compression_supported = compression_methods_supported; - - int i, result = -1; - for (i = 0; url_table[i].string; ++i) { - int match; - if (url_table[i].is_prefix) { - match = !strcmpstart(url, url_table[i].string); - } else { - match = !strcmp(url, url_table[i].string); - } - if (match) { - result = url_table[i].handler(conn, &args); - goto done; - } - } - - /* we didn't recognize the url */ - write_short_http_response(conn, 404, "Not found"); - result = 0; - - done: - tor_free(url_mem); - return result; -} - -/** Helper function for GET / or GET /tor/ - */ -static int -handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) -{ - (void) args; /* unused */ - const char *frontpage = get_dirportfrontpage(); - - if (frontpage) { - size_t dlen; - dlen = strlen(frontpage); - /* Let's return a disclaimer page (users shouldn't use V1 anymore, - and caches don't fetch '/', so this is safe). */ - - /* [We don't check for write_bucket_low here, since we want to serve - * this page no matter what.] */ - write_http_response_header_impl(conn, dlen, "text/html", "identity", - NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); - connection_buf_add(frontpage, dlen, TO_CONN(conn)); - } else { - write_short_http_response(conn, 404, "Not found"); - } - return 0; -} - -/** Warn that the cached consensus <b>consensus</b> of type - * <b>flavor</b> is too old and will not be served to clients. Rate-limit the - * warning to avoid logging an entry on every request. - */ -static void -warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, - const char *flavor, time_t now) -{ -#define TOO_OLD_WARNING_INTERVAL (60*60) - static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); - char timestamp[ISO_TIME_LEN+1]; - time_t valid_until; - char *dupes; - - if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) - return; - - if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, valid_until); - log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " - "serve it to clients. It was valid until %s local time and we " - "continued to serve it for up to 24 hours after it expired.%s", - flavor ? flavor : "", flavor ? " " : "", timestamp, dupes); - tor_free(dupes); - } -} - -/** - * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into - * <b>digest</b>. Return 0 on success. On failure, report that the hash came - * from <b>location</b>, report that we are taking <b>action</b> with it, and - * return -1. - */ -static int -parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location, - const char *action) -{ - if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) == - DIGEST256_LEN) { - return 0; - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "%s contained bogus digest %s; %s.", - location, escaped(hex), action); - return -1; - } -} - -/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>, - * set <b>digest_out<b> to a new smartlist containing every 256-bit - * hex-encoded digest listed in that header and return 0. Otherwise return - * -1. */ -static int -parse_or_diff_from_header(smartlist_t **digests_out, const char *headers) -{ - char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER); - if (hdr == NULL) { - return -1; - } - smartlist_t *hex_digests = smartlist_new(); - *digests_out = smartlist_new(); - smartlist_split_string(hex_digests, hdr, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) { - uint8_t digest[DIGEST256_LEN]; - if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header", - "ignoring")) { - smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest))); - } - } SMARTLIST_FOREACH_END(hex); - SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp)); - smartlist_free(hex_digests); - tor_free(hdr); - return 0; -} - -/** Fallback compression method. The fallback compression method is used in - * case a client requests a non-compressed document. We only store compressed - * documents, so we use this compression method to fetch the document and let - * the spooling system do the streaming decompression. - */ -#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD - -/** - * Try to find the best consensus diff possible in order to serve a client - * request for a diff from one of the consensuses in <b>digests</b> to the - * current consensus of flavor <b>flav</b>. The client supports the - * compression methods listed in the <b>compression_methods</b> bitfield: - * place the method chosen (if any) into <b>compression_used_out</b>. - */ -static struct consensus_cache_entry_t * -find_best_diff(const smartlist_t *digests, int flav, - unsigned compression_methods, - compress_method_t *compression_used_out) -{ - struct consensus_cache_entry_t *result = NULL; - - SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { - unsigned u; - for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { - compress_method_t method = srv_meth_pref_precompressed[u]; - if (0 == (compression_methods & (1u<<method))) - continue; // client doesn't like this one, or we don't have it. - if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, - diff_from, DIGEST256_LEN, - method) == CONSDIFF_AVAILABLE) { - tor_assert_nonfatal(result); - *compression_used_out = method; - return result; - } - } - } SMARTLIST_FOREACH_END(diff_from); - - SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) { - if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from, - DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { - tor_assert_nonfatal(result); - *compression_used_out = FALLBACK_COMPRESS_METHOD; - return result; - } - } SMARTLIST_FOREACH_END(diff_from); - - return NULL; -} - -/** Lookup the cached consensus document by the flavor found in <b>flav</b>. - * The preferred set of compression methods should be listed in the - * <b>compression_methods</b> bitfield. The compression method chosen (if any) - * is stored in <b>compression_used_out</b>. */ -static struct consensus_cache_entry_t * -find_best_consensus(int flav, - unsigned compression_methods, - compress_method_t *compression_used_out) -{ - struct consensus_cache_entry_t *result = NULL; - unsigned u; - - for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { - compress_method_t method = srv_meth_pref_precompressed[u]; - - if (0 == (compression_methods & (1u<<method))) - continue; - - if (consdiffmgr_find_consensus(&result, flav, - method) == CONSDIFF_AVAILABLE) { - tor_assert_nonfatal(result); - *compression_used_out = method; - return result; - } - } - - if (consdiffmgr_find_consensus(&result, flav, - FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) { - tor_assert_nonfatal(result); - *compression_used_out = FALLBACK_COMPRESS_METHOD; - return result; - } - - return NULL; -} - -/** Try to find the best supported compression method possible from a given - * <b>compression_methods</b>. Return NO_METHOD if no mutually supported - * compression method could be found. */ -static compress_method_t -find_best_compression_method(unsigned compression_methods, int stream) -{ - unsigned u; - compress_method_t *methods; - size_t length; - - if (stream) { - methods = srv_meth_pref_streaming_compression; - length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); - } else { - methods = srv_meth_pref_precompressed; - length = ARRAY_LENGTH(srv_meth_pref_precompressed); - } - - for (u = 0; u < length; ++u) { - compress_method_t method = methods[u]; - if (compression_methods & (1u<<method)) - return method; - } - - return NO_METHOD; -} - -/** Check if any of the digests in <b>digests</b> matches the latest consensus - * flavor (given in <b>flavor</b>) that we have available. */ -static int -digest_list_contains_best_consensus(consensus_flavor_t flavor, - const smartlist_t *digests) -{ - const networkstatus_t *ns = NULL; - - if (digests == NULL) - return 0; - - ns = networkstatus_get_latest_consensus_by_flavor(flavor); - - if (ns == NULL) - return 0; - - SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) { - if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN)) - return 1; - } SMARTLIST_FOREACH_END(digest); - - return 0; -} - -/** Check if the given compression method is allowed for a connection that is - * supposed to be anonymous. Returns 1 if the compression method is allowed, - * otherwise 0. */ -STATIC int -allowed_anonymous_connection_compression_method(compress_method_t method) -{ - unsigned u; - - for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression); - ++u) { - compress_method_t allowed_method = - client_meth_allowed_anonymous_compression[u]; - - if (! tor_compress_supports_method(allowed_method)) - continue; - - if (method == allowed_method) - return 1; - } - - return 0; -} - -/** Log a warning when a remote server has sent us a document using a - * compression method that is not allowed for anonymous directory requests. */ -STATIC void -warn_disallowed_anonymous_compression_method(compress_method_t method) -{ - log_fn(LOG_PROTOCOL_WARN, LD_HTTP, - "Received a %s HTTP response, which is not " - "allowed for anonymous directory requests.", - compression_method_get_human_name(method)); -} - -/** Encodes the results of parsing a consensus request to figure out what - * consensus, and possibly what diffs, the user asked for. */ -typedef struct { - /** name of the flavor to retrieve. */ - char *flavor; - /** flavor to retrive, as enum. */ - consensus_flavor_t flav; - /** plus-separated list of authority fingerprints; see - * client_likes_consensus(). Aliases the URL in the request passed to - * parse_consensus_request(). */ - const char *want_fps; - /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses - * to return a diff from. */ - smartlist_t *diff_from_digests; - /** If true, never send a full consensus. If there is no diff, send - * a 404 instead. */ - int diff_only; -} parsed_consensus_request_t; - -/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since - * it is stack-allocated. */ -static void -parsed_consensus_request_clear(parsed_consensus_request_t *req) -{ - if (!req) - return; - tor_free(req->flavor); - if (req->diff_from_digests) { - SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d)); - smartlist_free(req->diff_from_digests); - } - memset(req, 0, sizeof(parsed_consensus_request_t)); -} - -/** - * Parse the URL and relevant headers of <b>args</b> for a current-consensus - * request to learn what flavor of consensus we want, what keys it must be - * signed with, and what diffs we would accept (or demand) instead. Return 0 - * on success and -1 on failure. - */ -static int -parse_consensus_request(parsed_consensus_request_t *out, - const get_handler_args_t *args) -{ - const char *url = args->url; - memset(out, 0, sizeof(parsed_consensus_request_t)); - out->flav = FLAV_NS; - - const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/"; - const char CONSENSUS_FLAVORED_PREFIX[] = - "/tor/status-vote/current/consensus-"; - - /* figure out the flavor if any, and who we wanted to sign the thing */ - const char *after_flavor = NULL; - - if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { - const char *f, *cp; - f = url + strlen(CONSENSUS_FLAVORED_PREFIX); - cp = strchr(f, '/'); - if (cp) { - after_flavor = cp+1; - out->flavor = tor_strndup(f, cp-f); - } else { - out->flavor = tor_strdup(f); - } - int flav = networkstatus_parse_flavor_name(out->flavor); - if (flav < 0) - flav = FLAV_NS; - out->flav = flav; - } else { - if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) - after_flavor = url+strlen(CONSENSUS_URL_PREFIX); - } - - /* see whether we've been asked explicitly for a diff from an older - * consensus. (The user might also have said that a diff would be okay, - * via X-Or-Diff-From-Consensus */ - const char DIFF_COMPONENT[] = "diff/"; - char *diff_hash_in_url = NULL; - if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) { - after_flavor += strlen(DIFF_COMPONENT); - const char *cp = strchr(after_flavor, '/'); - if (cp) { - diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor); - out->want_fps = cp+1; - } else { - diff_hash_in_url = tor_strdup(after_flavor); - out->want_fps = NULL; - } - } else { - out->want_fps = after_flavor; - } - - if (diff_hash_in_url) { - uint8_t diff_from[DIGEST256_LEN]; - out->diff_from_digests = smartlist_new(); - out->diff_only = 1; - int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL", - "rejecting"); - tor_free(diff_hash_in_url); - if (ok) { - smartlist_add(out->diff_from_digests, - tor_memdup(diff_from, DIGEST256_LEN)); - } else { - return -1; - } - } else { - parse_or_diff_from_header(&out->diff_from_digests, args->headers); - } - - return 0; -} - -/** Helper function for GET /tor/status-vote/current/consensus - */ -static int -handle_get_current_consensus(dir_connection_t *conn, - const get_handler_args_t *args) -{ - const compress_method_t compress_method = - find_best_compression_method(args->compression_supported, 0); - const time_t if_modified_since = args->if_modified_since; - int clear_spool = 0; - - /* v3 network status fetch. */ - long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - - time_t now = time(NULL); - parsed_consensus_request_t req; - - if (parse_consensus_request(&req, args) < 0) { - write_short_http_response(conn, 404, "Couldn't parse request"); - goto done; - } - - if (digest_list_contains_best_consensus(req.flav, - req.diff_from_digests)) { - write_short_http_response(conn, 304, "Not modified"); - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); - goto done; - } - - struct consensus_cache_entry_t *cached_consensus = NULL; - - compress_method_t compression_used = NO_METHOD; - if (req.diff_from_digests) { - cached_consensus = find_best_diff(req.diff_from_digests, req.flav, - args->compression_supported, - &compression_used); - } - - if (req.diff_only && !cached_consensus) { - write_short_http_response(conn, 404, "No such diff available"); - // XXXX warn_consensus_is_too_old(v, req.flavor, now); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - goto done; - } - - if (! cached_consensus) { - cached_consensus = find_best_consensus(req.flav, - args->compression_supported, - &compression_used); - } - - time_t fresh_until, valid_until; - int have_fresh_until = 0, have_valid_until = 0; - if (cached_consensus) { - have_fresh_until = - !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); - have_valid_until = - !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); - } - - if (cached_consensus && have_valid_until && - !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { - write_short_http_response(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(cached_consensus, req.flavor, now); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - goto done; - } - - if (cached_consensus && req.want_fps && - !client_likes_consensus(cached_consensus, req.want_fps)) { - write_short_http_response(conn, 404, "Consensus not signed by sufficient " - "number of requested authorities"); - geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS); - goto done; - } - - conn->spool = smartlist_new(); - clear_spool = 1; - { - spooled_resource_t *spooled; - if (cached_consensus) { - spooled = spooled_resource_new_from_cache_entry(cached_consensus); - smartlist_add(conn->spool, spooled); - } - } - - lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0; - - size_t size_guess = 0; - int n_expired = 0; - dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, - compress_method != NO_METHOD, - &size_guess, - &n_expired); - - if (!smartlist_len(conn->spool) && !n_expired) { - write_short_http_response(conn, 404, "Not found"); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - goto done; - } else if (!smartlist_len(conn->spool)) { - write_short_http_response(conn, 304, "Not modified"); - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); - goto done; - } - - if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { - log_debug(LD_DIRSERV, - "Client asked for network status lists, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_short_http_response(conn, 503, "Directory busy, try again later"); - geoip_note_ns_response(GEOIP_REJECT_BUSY); - goto done; - } - - tor_addr_t addr; - if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, - &addr, NULL, - time(NULL)); - geoip_note_ns_response(GEOIP_SUCCESS); - /* Note that a request for a network status has started, so that we - * can measure the download time later on. */ - if (conn->dirreq_id) - geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED); - else - geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess, - DIRREQ_DIRECT); - } - - /* Use this header to tell caches that the response depends on the - * X-Or-Diff-From-Consensus header (or lack thereof). */ - const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n"; - - clear_spool = 0; - - // The compress_method might have been NO_METHOD, but we store the data - // compressed. Decompress them using `compression_used`. See fallback code in - // find_best_consensus() and find_best_diff(). - write_http_response_headers(conn, -1, - compress_method == NO_METHOD ? - NO_METHOD : compression_used, - vary_header, - smartlist_len(conn->spool) == 1 ? lifetime : 0); - - if (compress_method == NO_METHOD && smartlist_len(conn->spool)) - conn->compress_state = tor_compress_new(0, compression_used, - HIGH_COMPRESSION); - - /* Prime the connection with some data. */ - const int initial_flush_result = connection_dirserv_flushed_some(conn); - tor_assert_nonfatal(initial_flush_result == 0); - goto done; - - done: - parsed_consensus_request_clear(&req); - if (clear_spool) { - dir_conn_clear_spool(conn); - } - return 0; -} - -/** Helper function for GET /tor/status-vote/{current,next}/... - */ -static int -handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) -{ - const char *url = args->url; - { - ssize_t body_len = 0; - ssize_t estimated_len = 0; - int lifetime = 60; /* XXXX?? should actually use vote intervals. */ - /* This smartlist holds strings that we can compress on the fly. */ - smartlist_t *items = smartlist_new(); - /* This smartlist holds cached_dir_t objects that have a precompressed - * deflated version. */ - smartlist_t *dir_items = smartlist_new(); - dirvote_dirreq_get_status_vote(url, items, dir_items); - if (!smartlist_len(dir_items) && !smartlist_len(items)) { - write_short_http_response(conn, 404, "Not found"); - goto vote_done; - } - - /* We're sending items from at most one kind of source */ - tor_assert_nonfatal(smartlist_len(items) == 0 || - smartlist_len(dir_items) == 0); - - int streaming; - unsigned mask; - if (smartlist_len(items)) { - /* We're taking strings and compressing them on the fly. */ - streaming = 1; - mask = ~0u; - } else { - /* We're taking cached_dir_t objects. We only have them uncompressed - * or deflated. */ - streaming = 0; - mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD); - } - const compress_method_t compress_method = find_best_compression_method( - args->compression_supported&mask, streaming); - - SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compress_method != NO_METHOD ? - d->dir_compressed_len : d->dir_len); - estimated_len += body_len; - SMARTLIST_FOREACH(items, const char *, item, { - size_t ln = strlen(item); - if (compress_method != NO_METHOD) { - estimated_len += ln/2; - } else { - body_len += ln; estimated_len += ln; - } - }); - - if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) { - write_short_http_response(conn, 503, "Directory busy, try again later"); - goto vote_done; - } - write_http_response_header(conn, body_len ? body_len : -1, - compress_method, - lifetime); - - if (smartlist_len(items)) { - if (compress_method != NO_METHOD) { - conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(estimated_len)); - SMARTLIST_FOREACH(items, const char *, c, - connection_buf_add_compress(c, strlen(c), conn, 0)); - connection_buf_add_compress("", 0, conn, 1); - } else { - SMARTLIST_FOREACH(items, const char *, c, - connection_buf_add(c, strlen(c), TO_CONN(conn))); - } - } else { - SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_buf_add(compress_method != NO_METHOD ? - d->dir_compressed : d->dir, - compress_method != NO_METHOD ? - d->dir_compressed_len : d->dir_len, - TO_CONN(conn))); - } - vote_done: - smartlist_free(items); - smartlist_free(dir_items); - goto done; - } - done: - return 0; -} - -/** Helper function for GET /tor/micro/d/... - */ -static int -handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) -{ - const char *url = args->url; - const compress_method_t compress_method = - find_best_compression_method(args->compression_supported, 1); - int clear_spool = 1; - { - conn->spool = smartlist_new(); - - dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"), - DIR_SPOOL_MICRODESC, - conn->spool, NULL, - DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); - - size_t size_guess = 0; - dirserv_spool_remove_missing_and_guess_size(conn, 0, - compress_method != NO_METHOD, - &size_guess, NULL); - if (smartlist_len(conn->spool) == 0) { - write_short_http_response(conn, 404, "Not found"); - goto done; - } - if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { - log_info(LD_DIRSERV, - "Client asked for server descriptors, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_short_http_response(conn, 503, "Directory busy, try again later"); - goto done; - } - - clear_spool = 0; - write_http_response_header(conn, -1, - compress_method, - MICRODESC_CACHE_LIFETIME); - - if (compress_method != NO_METHOD) - conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); - - const int initial_flush_result = connection_dirserv_flushed_some(conn); - tor_assert_nonfatal(initial_flush_result == 0); - goto done; - } - - done: - if (clear_spool) { - dir_conn_clear_spool(conn); - } - return 0; -} - -/** Helper function for GET /tor/{server,extra}/... - */ -static int -handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) -{ - const char *url = args->url; - const compress_method_t compress_method = - find_best_compression_method(args->compression_supported, 1); - const or_options_t *options = get_options(); - int clear_spool = 1; - if (!strcmpstart(url,"/tor/server/") || - (!options->BridgeAuthoritativeDir && - !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { - int res; - const char *msg = NULL; - int cache_lifetime = 0; - int is_extra = !strcmpstart(url,"/tor/extra/"); - url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/"); - dir_spool_source_t source; - time_t publish_cutoff = 0; - if (!strcmpstart(url, "d/")) { - source = - is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST; - } else { - source = - is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP; - /* We only want to apply a publish cutoff when we're requesting - * resources by fingerprint. */ - publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; - } - - conn->spool = smartlist_new(); - res = dirserv_get_routerdesc_spool(conn->spool, url, - source, - connection_dir_is_encrypted(conn), - &msg); - - if (!strcmpstart(url, "all")) { - cache_lifetime = FULL_DIR_CACHE_LIFETIME; - } else if (smartlist_len(conn->spool) == 1) { - cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; - } - - size_t size_guess = 0; - int n_expired = 0; - dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, - compress_method != NO_METHOD, - &size_guess, &n_expired); - - /* If we are the bridge authority and the descriptor is a bridge - * descriptor, remember that we served this descriptor for desc stats. */ - /* XXXX it's a bit of a kludge to have this here. */ - if (get_options()->BridgeAuthoritativeDir && - source == DIR_SPOOL_SERVER_BY_FP) { - SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) { - const routerinfo_t *router = - router_get_by_id_digest((const char *)spooled->digest); - /* router can be NULL here when the bridge auth is asked for its own - * descriptor. */ - if (router && router->purpose == ROUTER_PURPOSE_BRIDGE) - rep_hist_note_desc_served(router->cache_info.identity_digest); - } SMARTLIST_FOREACH_END(spooled); - } - - if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) { - if (msg == NULL) - msg = "Not found"; - write_short_http_response(conn, 404, msg); - } else { - if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) { - log_info(LD_DIRSERV, - "Client asked for server descriptors, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_short_http_response(conn, 503, - "Directory busy, try again later"); - dir_conn_clear_spool(conn); - goto done; - } - write_http_response_header(conn, -1, compress_method, cache_lifetime); - if (compress_method != NO_METHOD) - conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(size_guess)); - clear_spool = 0; - /* Prime the connection with some data. */ - int initial_flush_result = connection_dirserv_flushed_some(conn); - tor_assert_nonfatal(initial_flush_result == 0); - } - goto done; - } - done: - if (clear_spool) - dir_conn_clear_spool(conn); - return 0; -} - -/** Helper function for GET /tor/keys/... - */ -static int -handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) -{ - const char *url = args->url; - const compress_method_t compress_method = - find_best_compression_method(args->compression_supported, 1); - const time_t if_modified_since = args->if_modified_since; - { - smartlist_t *certs = smartlist_new(); - ssize_t len = -1; - if (!strcmp(url, "/tor/keys/all")) { - authority_cert_get_all(certs); - } else if (!strcmp(url, "/tor/keys/authority")) { - authority_cert_t *cert = get_my_v3_authority_cert(); - if (cert) - smartlist_add(certs, cert); - } else if (!strcmpstart(url, "/tor/keys/fp/")) { - smartlist_t *fps = smartlist_new(); - dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, - DSR_HEX|DSR_SORT_UNIQ); - SMARTLIST_FOREACH(fps, char *, d, { - authority_cert_t *c = authority_cert_get_newest_by_id(d); - if (c) smartlist_add(certs, c); - tor_free(d); - }); - smartlist_free(fps); - } else if (!strcmpstart(url, "/tor/keys/sk/")) { - smartlist_t *fps = smartlist_new(); - dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, - DSR_HEX|DSR_SORT_UNIQ); - SMARTLIST_FOREACH(fps, char *, d, { - authority_cert_t *c = authority_cert_get_by_sk_digest(d); - if (c) smartlist_add(certs, c); - tor_free(d); - }); - smartlist_free(fps); - } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) { - smartlist_t *fp_sks = smartlist_new(); - dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"), - fp_sks); - SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, { - authority_cert_t *c = authority_cert_get_by_digests(pair->first, - pair->second); - if (c) smartlist_add(certs, c); - tor_free(pair); - }); - smartlist_free(fp_sks); - } else { - write_short_http_response(conn, 400, "Bad request"); - goto keys_done; - } - if (!smartlist_len(certs)) { - write_short_http_response(conn, 404, "Not found"); - goto keys_done; - } - SMARTLIST_FOREACH(certs, authority_cert_t *, c, - if (c->cache_info.published_on < if_modified_since) - SMARTLIST_DEL_CURRENT(certs, c)); - if (!smartlist_len(certs)) { - write_short_http_response(conn, 304, "Not modified"); - goto keys_done; - } - len = 0; - SMARTLIST_FOREACH(certs, authority_cert_t *, c, - len += c->cache_info.signed_descriptor_len); - - if (global_write_bucket_low(TO_CONN(conn), - compress_method != NO_METHOD ? len/2 : len, - 2)) { - write_short_http_response(conn, 503, "Directory busy, try again later"); - goto keys_done; - } - - write_http_response_header(conn, - compress_method != NO_METHOD ? -1 : len, - compress_method, - 60*60); - if (compress_method != NO_METHOD) { - conn->compress_state = tor_compress_new(1, compress_method, - choose_compression_level(len)); - SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_buf_add_compress( - c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_buf_add_compress("", 0, conn, 1); - } else { - SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_buf_add(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - TO_CONN(conn))); - } - keys_done: - smartlist_free(certs); - goto done; - } - done: - return 0; -} - -/** Helper function for GET /tor/rendezvous2/ - */ -static int -handle_get_hs_descriptor_v2(dir_connection_t *conn, - const get_handler_args_t *args) -{ - const char *url = args->url; - if (connection_dir_is_encrypted(conn)) { - /* Handle v2 rendezvous descriptor fetch request. */ - const char *descp; - const char *query = url + strlen("/tor/rendezvous2/"); - if (rend_valid_descriptor_id(query)) { - log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'", - safe_str(escaped(query))); - switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { - case 1: /* valid */ - write_http_response_header(conn, strlen(descp), NO_METHOD, 0); - connection_buf_add(descp, strlen(descp), TO_CONN(conn)); - break; - case 0: /* well-formed but not present */ - write_short_http_response(conn, 404, "Not found"); - break; - case -1: /* not well-formed */ - write_short_http_response(conn, 400, "Bad request"); - break; - } - } else { /* not well-formed */ - write_short_http_response(conn, 400, "Bad request"); - } - goto done; - } else { - /* Not encrypted! */ - write_short_http_response(conn, 404, "Not found"); - } - done: - return 0; -} - -/** Helper function for GET /tor/hs/3/<z>. Only for version 3. - */ -STATIC int -handle_get_hs_descriptor_v3(dir_connection_t *conn, - const get_handler_args_t *args) -{ - int retval; - const char *desc_str = NULL; - const char *pubkey_str = NULL; - const char *url = args->url; - - /* Reject unencrypted dir connections */ - if (!connection_dir_is_encrypted(conn)) { - write_short_http_response(conn, 404, "Not found"); - goto done; - } - - /* After the path prefix follows the base64 encoded blinded pubkey which we - * use to get the descriptor from the cache. Skip the prefix and get the - * pubkey. */ - tor_assert(!strcmpstart(url, "/tor/hs/3/")); - pubkey_str = url + strlen("/tor/hs/3/"); - retval = hs_cache_lookup_as_dir(HS_VERSION_THREE, - pubkey_str, &desc_str); - if (retval <= 0 || desc_str == NULL) { - write_short_http_response(conn, 404, "Not found"); - goto done; - } - - /* Found requested descriptor! Pass it to this nice client. */ - write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0); - connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn)); - - done: - return 0; -} - -/** Helper function for GET /tor/networkstatus-bridges - */ -static int -handle_get_networkstatus_bridges(dir_connection_t *conn, - const get_handler_args_t *args) -{ - const char *headers = args->headers; - - const or_options_t *options = get_options(); - if (options->BridgeAuthoritativeDir && - options->BridgePassword_AuthDigest_ && - connection_dir_is_encrypted(conn)) { - char *status; - char digest[DIGEST256_LEN]; - - char *header = http_get_header(headers, "Authorization: Basic "); - if (header) - crypto_digest256(digest, header, strlen(header), DIGEST_SHA256); - - /* now make sure the password is there and right */ - if (!header || - tor_memneq(digest, - options->BridgePassword_AuthDigest_, DIGEST256_LEN)) { - write_short_http_response(conn, 404, "Not found"); - tor_free(header); - goto done; - } - tor_free(header); - - /* all happy now. send an answer. */ - status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); - size_t dlen = strlen(status); - write_http_response_header(conn, dlen, NO_METHOD, 0); - connection_buf_add(status, dlen, TO_CONN(conn)); - tor_free(status); - goto done; - } - done: - return 0; -} - -/** Helper function for GET robots.txt or /tor/robots.txt */ -static int -handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) -{ - (void)args; - { - const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; - size_t len = strlen(robots); - write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME); - connection_buf_add(robots, len, TO_CONN(conn)); - } - return 0; -} - -/* Given the <b>url</b> from a POST request, try to extract the version number - * using the provided <b>prefix</b>. The version should be after the prefix and - * ending with the separator "/". For instance: - * /tor/hs/3/publish - * - * On success, <b>end_pos</b> points to the position right after the version - * was found. On error, it is set to NULL. - * - * Return version on success else negative value. */ -STATIC int -parse_hs_version_from_post(const char *url, const char *prefix, - const char **end_pos) -{ - int ok; - unsigned long version; - const char *start; - char *end = NULL; - - tor_assert(url); - tor_assert(prefix); - tor_assert(end_pos); - - /* Check if the prefix does start the url. */ - if (strcmpstart(url, prefix)) { - goto err; - } - /* Move pointer to the end of the prefix string. */ - start = url + strlen(prefix); - /* Try this to be the HS version and if we are still at the separator, next - * will be move to the right value. */ - version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end); - if (!ok) { - goto err; - } - - *end_pos = end; - return (int) version; - err: - *end_pos = NULL; - return -1; -} - -/* Handle the POST request for a hidden service descripror. The request is in - * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success - * else return 400 indicating a bad request. */ -STATIC int -handle_post_hs_descriptor(const char *url, const char *body) -{ - int version; - const char *end_pos; - - tor_assert(url); - tor_assert(body); - - version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos); - if (version < 0) { - goto err; - } - - /* We have a valid version number, now make sure it's a publish request. Use - * the end position just after the version and check for the command. */ - if (strcmpstart(end_pos, "/publish")) { - goto err; - } - - switch (version) { - case HS_VERSION_THREE: - if (hs_cache_store_as_dir(body) < 0) { - goto err; - } - log_info(LD_REND, "Publish request for HS descriptor handled " - "successfully."); - break; - default: - /* Unsupported version, return a bad request. */ - goto err; - } - - return 200; - err: - /* Bad request. */ - return 400; -} - -/** Helper function: called when a dirserver gets a complete HTTP POST - * request. Look for an uploaded server descriptor or rendezvous - * service descriptor. On finding one, process it and write a - * response into conn->outbuf. If the request is unrecognized, send a - * 400. Always return 0. */ -MOCK_IMPL(STATIC int, -directory_handle_command_post,(dir_connection_t *conn, const char *headers, - const char *body, size_t body_len)) -{ - char *url = NULL; - const or_options_t *options = get_options(); - - log_debug(LD_DIRSERV,"Received POST command."); - - conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; - - if (!public_server_mode(options)) { - log_info(LD_DIR, "Rejected dir post request from %s " - "since we're not a public relay.", conn->base_.address); - write_short_http_response(conn, 503, "Not acting as a public relay"); - goto done; - } - - if (parse_http_url(headers, &url) < 0) { - write_short_http_response(conn, 400, "Bad request"); - return 0; - } - log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); - - /* Handle v2 rendezvous service publish request. */ - if (connection_dir_is_encrypted(conn) && - !strcmpstart(url,"/tor/rendezvous2/publish")) { - if (rend_cache_store_v2_desc_as_dir(body) < 0) { - log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.", - (int)body_len, conn->base_.address); - write_short_http_response(conn, 400, - "Invalid v2 service descriptor rejected"); - } else { - write_short_http_response(conn, 200, "Service descriptor (v2) stored"); - log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); - } - goto done; - } - - /* Handle HS descriptor publish request. */ - /* XXX: This should be disabled with a consensus param until we want to - * the prop224 be deployed and thus use. */ - if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) { - const char *msg = "HS descriptor stored successfully."; - - /* We most probably have a publish request for an HS descriptor. */ - int code = handle_post_hs_descriptor(url, body); - if (code != 200) { - msg = "Invalid HS descriptor. Rejected."; - } - write_short_http_response(conn, code, msg); - goto done; - } - - if (!authdir_mode(options)) { - /* we just provide cached directories; we don't want to - * receive anything. */ - write_short_http_response(conn, 400, "Nonauthoritative directory does not " - "accept posted server descriptors"); - goto done; - } + unsigned u;

- if (authdir_mode(options) && - !strcmp(url,"/tor/")) { /* server descriptor post */ - const char *msg = "[None]"; - uint8_t purpose = authdir_mode_bridge(options) ? - ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; - was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, - conn->base_.address, &msg); - tor_assert(msg); - - if (r == ROUTER_ADDED_SUCCESSFULLY) { - write_short_http_response(conn, 200, msg); - } else if (WRA_WAS_OUTDATED(r)) { - write_http_response_header_impl(conn, -1, NULL, NULL, - "X-Descriptor-Not-New: Yes\r\n", -1); - } else { - log_info(LD_DIRSERV, - "Rejected router descriptor or extra-info from %s " - "("%s").", - conn->base_.address, msg); - write_short_http_response(conn, 400, msg); - } - goto done; - } + for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression); + ++u) { + compress_method_t allowed_method = + client_meth_allowed_anonymous_compression[u];

- if (authdir_mode_v3(options) && - !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */ - const char *msg = "OK"; - int status; - if (dirvote_add_vote(body, &msg, &status)) { - write_short_http_response(conn, status, "Vote stored"); - } else { - tor_assert(msg); - log_warn(LD_DIRSERV, "Rejected vote from %s ("%s").", - conn->base_.address, msg); - write_short_http_response(conn, status, msg); - } - goto done; - } + if (! tor_compress_supports_method(allowed_method)) + continue;

- if (authdir_mode_v3(options) && - !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */ - const char *msg = NULL; - if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) { - write_short_http_response(conn, 200, msg?msg:"Signatures stored"); - } else { - log_warn(LD_DIR, "Unable to store signatures posted by %s: %s", - conn->base_.address, msg?msg:"???"); - write_short_http_response(conn, 400, - msg?msg:"Unable to store signatures"); - } - goto done; + if (method == allowed_method) + return 1; }

- /* we didn't recognize the url */ - write_short_http_response(conn, 404, "Not found"); - - done: - tor_free(url); return 0; }

-/** Called when a dirserver receives data on a directory connection; - * looks for an HTTP request. If the request is complete, remove it - * from the inbuf, try to process it; otherwise, leave it on the - * buffer. Return a 0 on success, or -1 on error. - */ -STATIC int -directory_handle_command(dir_connection_t *conn) -{ - char *headers=NULL, *body=NULL; - size_t body_len=0; - int r; - - tor_assert(conn); - tor_assert(conn->base_.type == CONN_TYPE_DIR); - - switch (connection_fetch_from_buf_http(TO_CONN(conn), - &headers, MAX_HEADERS_SIZE, - &body, &body_len, MAX_DIR_UL_SIZE, 0)) { - case -1: /* overflow */ - log_warn(LD_DIRSERV, - "Request too large from address '%s' to DirPort. Closing.", - safe_str(conn->base_.address)); - return -1; - case 0: - log_debug(LD_DIRSERV,"command not all here yet."); - return 0; - /* case 1, fall through */ - } - - http_set_address_origin(headers, TO_CONN(conn)); - // we should escape headers here as well, - // but we can't call escaped() twice, as it uses the same buffer - //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body)); - - if (!strncasecmp(headers,"GET",3)) - r = directory_handle_command_get(conn, headers, body, body_len); - else if (!strncasecmp(headers,"POST",4)) - r = directory_handle_command_post(conn, headers, body, body_len); - else { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Got headers %s with unknown command. Closing.", - escaped(headers)); - r = -1; - } - - tor_free(headers); tor_free(body); - return r; -} - -/** Write handler for directory connections; called when all data has - * been flushed. Close the connection or wait for a response as - * appropriate. - */ -int -connection_dir_finished_flushing(dir_connection_t *conn) +/** Log a warning when a remote server has sent us a document using a + * compression method that is not allowed for anonymous directory requests. */ +STATIC void +warn_disallowed_anonymous_compression_method(compress_method_t method) { - tor_assert(conn); - tor_assert(conn->base_.type == CONN_TYPE_DIR); - - if (conn->base_.marked_for_close) - return 0; - - /* Note that we have finished writing the directory response. For direct - * connections this means we're done; for tunneled connections it's only - * an intermediate step. */ - if (conn->dirreq_id) - geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, - DIRREQ_FLUSHING_DIR_CONN_FINISHED); - else - geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, - DIRREQ_DIRECT, - DIRREQ_FLUSHING_DIR_CONN_FINISHED); - switch (conn->base_.state) { - case DIR_CONN_STATE_CONNECTING: - case DIR_CONN_STATE_CLIENT_SENDING: - log_debug(LD_DIR,"client finished sending command."); - conn->base_.state = DIR_CONN_STATE_CLIENT_READING; - return 0; - case DIR_CONN_STATE_SERVER_WRITING: - if (conn->spool) { - log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!"); - connection_mark_for_close(TO_CONN(conn)); - } else { - log_debug(LD_DIRSERV, "Finished writing server response. Closing."); - connection_mark_for_close(TO_CONN(conn)); - } - return 0; - default: - log_warn(LD_BUG,"called in unexpected state %d.", - conn->base_.state); - tor_fragile_assert(); - return -1; - } - return 0; + log_fn(LOG_PROTOCOL_WARN, LD_HTTP, + "Received a %s HTTP response, which is not " + "allowed for anonymous directory requests.", + compression_method_get_human_name(method)); }

/* We just got a new consensus! If there are other in-progress requests @@ -5278,430 +3119,6 @@ connection_dir_close_consensus_fetches(dir_connection_t *except_this_one, } SMARTLIST_FOREACH_END(d); smartlist_free(conns_to_close); } - -/** Connected handler for directory connections: begin sending data to the - * server, and return 0. - * Only used when connections don't immediately connect. */ -int -connection_dir_finished_connecting(dir_connection_t *conn) -{ - tor_assert(conn); - tor_assert(conn->base_.type == CONN_TYPE_DIR); - tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING); - - log_debug(LD_HTTP,"Dir connection to router %s:%u established.", - conn->base_.address,conn->base_.port); - - /* start flushing conn */ - conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; - return 0; -} - -/** Decide which download schedule we want to use based on descriptor type - * in <b>dls</b> and <b>options</b>. - * - * Then, return the initial delay for that download schedule, in seconds. - * - * Helper function for download_status_increment_failure(), - * download_status_reset(), and download_status_increment_attempt(). */ -STATIC int -find_dl_min_delay(const download_status_t *dls, const or_options_t *options) -{ - tor_assert(dls); - tor_assert(options); - - switch (dls->schedule) { - case DL_SCHED_GENERIC: - /* Any other directory document */ - if (dir_server_mode(options)) { - /* A directory authority or directory mirror */ - return options->TestingServerDownloadInitialDelay; - } else { - return options->TestingClientDownloadInitialDelay; - } - case DL_SCHED_CONSENSUS: - if (!networkstatus_consensus_can_use_multiple_directories(options)) { - /* A public relay */ - return options->TestingServerConsensusDownloadInitialDelay; - } else { - /* A client or bridge */ - if (networkstatus_consensus_is_bootstrapping(time(NULL))) { - /* During bootstrapping */ - if (!networkstatus_consensus_can_use_extra_fallbacks(options)) { - /* A bootstrapping client without extra fallback directories */ - return options-> - ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay; - } else if (dls->want_authority) { - /* A bootstrapping client with extra fallback directories, but - * connecting to an authority */ - return - options->ClientBootstrapConsensusAuthorityDownloadInitialDelay; - } else { - /* A bootstrapping client connecting to extra fallback directories - */ - return - options->ClientBootstrapConsensusFallbackDownloadInitialDelay; - } - } else { - /* A client with a reasonably live consensus, with or without - * certificates */ - return options->TestingClientConsensusDownloadInitialDelay; - } - } - case DL_SCHED_BRIDGE: - if (options->UseBridges && num_bridges_usable(0) > 0) { - /* A bridge client that is sure that one or more of its bridges are - * running can afford to wait longer to update bridge descriptors. */ - return options->TestingBridgeDownloadInitialDelay; - } else { - /* A bridge client which might have no running bridges, must try to - * get bridge descriptors straight away. */ - return options->TestingBridgeBootstrapDownloadInitialDelay; - } - default: - tor_assert(0); - } - - /* Impossible, but gcc will fail with -Werror without a `return`. */ - return 0; -} - -/** As next_random_exponential_delay() below, but does not compute a random - * value. Instead, compute the range of values that - * next_random_exponential_delay() should use when computing its random value. - * Store the low bound into *<b>low_bound_out</b>, and the high bound into - * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less - * than the high bound. */ -STATIC void -next_random_exponential_delay_range(int *low_bound_out, - int *high_bound_out, - int delay, - int base_delay) -{ - // This is the "decorrelated jitter" approach, from - // https://www.awsarchitectureblog.com/2015/03/backoff.html - // The formula is - // sleep = min(cap, random_between(base, sleep * 3)) - - const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX; - *low_bound_out = base_delay; - if (delay_times_3 > base_delay) { - *high_bound_out = delay_times_3; - } else { - *high_bound_out = base_delay+1; - } -} - -/** Advance one delay step. The algorithm will generate a random delay, - * such that each failure is possibly (random) longer than the ones before. - * - * We then clamp that value to be no larger than max_delay, and return it. - * - * The <b>base_delay</b> parameter is lowest possible delay time (can't be - * zero); the <b>backoff_position</b> parameter is the number of times we've - * generated a delay; and the <b>delay</b> argument is the most recently used - * delay. - */ -STATIC int -next_random_exponential_delay(int delay, - int base_delay) -{ - /* Check preconditions */ - if (BUG(delay < 0)) - delay = 0; - - if (base_delay < 1) - base_delay = 1; - - int low_bound=0, high_bound=INT_MAX; - - next_random_exponential_delay_range(&low_bound, &high_bound, - delay, base_delay); - - return crypto_rand_int_range(low_bound, high_bound); -} - -/** Find the current delay for dls based on min_delay. - * - * This function sets dls->next_attempt_at based on now, and returns the delay. - * Helper for download_status_increment_failure and - * download_status_increment_attempt. */ -STATIC int -download_status_schedule_get_delay(download_status_t *dls, - int min_delay, - time_t now) -{ - tor_assert(dls); - /* If we're using random exponential backoff, we do need min/max delay */ - tor_assert(min_delay >= 0); - - int delay = INT_MAX; - uint8_t dls_schedule_position = (dls->increment_on - == DL_SCHED_INCREMENT_ATTEMPT - ? dls->n_download_attempts - : dls->n_download_failures); - - /* Check if we missed a reset somehow */ - IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) { - dls->last_backoff_position = 0; - dls->last_delay_used = 0; - } - - if (dls_schedule_position > 0) { - delay = dls->last_delay_used; - - while (dls->last_backoff_position < dls_schedule_position) { - /* Do one increment step */ - delay = next_random_exponential_delay(delay, min_delay); - /* Update our position */ - ++(dls->last_backoff_position); - } - } else { - /* If we're just starting out, use the minimum delay */ - delay = min_delay; - } - - /* Clamp it within min/max if we have them */ - if (min_delay >= 0 && delay < min_delay) delay = min_delay; - - /* Store it for next time */ - dls->last_backoff_position = dls_schedule_position; - dls->last_delay_used = delay; - - /* A negative delay makes no sense. Knowing that delay is - * non-negative allows us to safely do the wrapping check below. */ - tor_assert(delay >= 0); - - /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction - * that won't overflow (since delay is non-negative). */ - if (delay < INT_MAX && now <= TIME_MAX - delay) { - dls->next_attempt_at = now+delay; - } else { - dls->next_attempt_at = TIME_MAX; - } - - return delay; -} - -/* Log a debug message about item, which increments on increment_action, has - * incremented dls_n_download_increments times. The message varies based on - * was_schedule_incremented (if not, not_incremented_response is logged), and - * the values of increment, dls_next_attempt_at, and now. - * Helper for download_status_increment_failure and - * download_status_increment_attempt. */ -static void -download_status_log_helper(const char *item, int was_schedule_incremented, - const char *increment_action, - const char *not_incremented_response, - uint8_t dls_n_download_increments, int increment, - time_t dls_next_attempt_at, time_t now) -{ - if (item) { - if (!was_schedule_incremented) - log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.", - item, increment_action, (int)dls_n_download_increments, - not_incremented_response); - else if (increment == 0) - log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.", - item, increment_action, (int)dls_n_download_increments); - else if (dls_next_attempt_at < TIME_MAX) - log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.", - item, increment_action, (int)dls_n_download_increments, - (int)(dls_next_attempt_at-now)); - else - log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.", - item, increment_action, (int)dls_n_download_increments); - } -} - -/** Determine when a failed download attempt should be retried. - * Called when an attempt to download <b>dls</b> has failed with HTTP status - * <b>status_code</b>. Increment the failure count (if the code indicates a - * real failure, or if we're a server) and set <b>dls</b>->next_attempt_at to - * an appropriate time in the future and return it. - * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the - * failure count, and return a time in the far future for the next attempt (to - * avoid an immediate retry). */ -time_t -download_status_increment_failure(download_status_t *dls, int status_code, - const char *item, int server, time_t now) -{ - (void) status_code; // XXXX no longer used. - (void) server; // XXXX no longer used. - int increment = -1; - int min_delay = 0; - - tor_assert(dls); - - /* dls wasn't reset before it was used */ - if (dls->next_attempt_at == 0) { - download_status_reset(dls); - } - - /* count the failure */ - if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) { - ++dls->n_download_failures; - } - - if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { - /* We don't find out that a failure-based schedule has attempted a - * connection until that connection fails. - * We'll never find out about successful connections, but this doesn't - * matter, because schedules are reset after a successful download. - */ - if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) - ++dls->n_download_attempts; - - /* only return a failure retry time if this schedule increments on failures - */ - min_delay = find_dl_min_delay(dls, get_options()); - increment = download_status_schedule_get_delay(dls, min_delay, now); - } - - download_status_log_helper(item, !dls->increment_on, "failed", - "concurrently", dls->n_download_failures, - increment, - download_status_get_next_attempt_at(dls), - now); - - if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) { - /* stop this schedule retrying on failure, it will launch concurrent - * connections instead */ - return TIME_MAX; - } else { - return download_status_get_next_attempt_at(dls); - } -} - -/** Determine when the next download attempt should be made when using an - * attempt-based (potentially concurrent) download schedule. - * Called when an attempt to download <b>dls</b> is being initiated. - * Increment the attempt count and set <b>dls</b>->next_attempt_at to an - * appropriate time in the future and return it. - * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment - * the attempts, and return a time in the far future (to avoid launching a - * concurrent attempt). */ -time_t -download_status_increment_attempt(download_status_t *dls, const char *item, - time_t now) -{ - int delay = -1; - int min_delay = 0; - - tor_assert(dls); - - /* dls wasn't reset before it was used */ - if (dls->next_attempt_at == 0) { - download_status_reset(dls); - } - - if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { - /* this schedule should retry on failure, and not launch any concurrent - attempts */ - log_warn(LD_BUG, "Tried to launch an attempt-based connection on a " - "failure-based schedule."); - return TIME_MAX; - } - - if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) - ++dls->n_download_attempts; - - min_delay = find_dl_min_delay(dls, get_options()); - delay = download_status_schedule_get_delay(dls, min_delay, now); - - download_status_log_helper(item, dls->increment_on, "attempted", - "on failure", dls->n_download_attempts, - delay, download_status_get_next_attempt_at(dls), - now); - - return download_status_get_next_attempt_at(dls); -} - -static time_t -download_status_get_initial_delay_from_now(const download_status_t *dls) -{ - /* We use constant initial delays, even in exponential backoff - * schedules. */ - return time(NULL) + find_dl_min_delay(dls, get_options()); -} - -/** Reset <b>dls</b> so that it will be considered downloadable - * immediately, and/or to show that we don't need it anymore. - * - * Must be called to initialise a download schedule, otherwise the zeroth item - * in the schedule will never be used. - * - * (We find the zeroth element of the download schedule, and set - * next_attempt_at to be the appropriate offset from 'now'. In most - * cases this means setting it to 'now', so the item will be immediately - * downloadable; when using authorities with fallbacks, there is a few seconds' - * delay.) */ -void -download_status_reset(download_status_t *dls) -{ - if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD - || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD) - return; /* Don't reset this. */ - - dls->n_download_failures = 0; - dls->n_download_attempts = 0; - dls->next_attempt_at = download_status_get_initial_delay_from_now(dls); - dls->last_backoff_position = 0; - dls->last_delay_used = 0; - /* Don't reset dls->want_authority or dls->increment_on */ -} - -/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is - * ready to get its download reattempted. */ -int -download_status_is_ready(download_status_t *dls, time_t now) -{ - /* dls wasn't reset before it was used */ - if (dls->next_attempt_at == 0) { - download_status_reset(dls); - } - - return download_status_get_next_attempt_at(dls) <= now; -} - -/** Mark <b>dl</b> as never downloadable. */ -void -download_status_mark_impossible(download_status_t *dl) -{ - dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; - dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD; -} - -/** Return the number of failures on <b>dls</b> since the last success (if - * any). */ -int -download_status_get_n_failures(const download_status_t *dls) -{ - return dls->n_download_failures; -} - -/** Return the number of attempts to download <b>dls</b> since the last success - * (if any). This can differ from download_status_get_n_failures() due to - * outstanding concurrent attempts. */ -int -download_status_get_n_attempts(const download_status_t *dls) -{ - return dls->n_download_attempts; -} - -/** Return the next time to attempt to download <b>dls</b>. */ -time_t -download_status_get_next_attempt_at(const download_status_t *dls) -{ - /* dls wasn't reset before it was used */ - if (dls->next_attempt_at == 0) { - /* so give the answer we would have given if it had been */ - return download_status_get_initial_delay_from_now(dls); - } - - return dls->next_attempt_at; -} - /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on @@ -5784,189 +3201,3 @@ dir_microdesc_download_failed(smartlist_t *failed, } } SMARTLIST_FOREACH_END(d); } - -/** Helper. Compare two fp_pair_t objects, and return negative, 0, or - * positive as appropriate. */ -static int -compare_pairs_(const void **a, const void **b) -{ - const fp_pair_t *fp1 = *a, *fp2 = *b; - int r; - if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN))) - return r; - else - return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN); -} - -/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each - * FP is a hex-encoded fingerprint, into a sequence of distinct sorted - * fp_pair_t. Skip malformed pairs. On success, return 0 and add those - * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */ -int -dir_split_resource_into_fingerprint_pairs(const char *res, - smartlist_t *pairs_out) -{ - smartlist_t *pairs_tmp = smartlist_new(); - smartlist_t *pairs_result = smartlist_new(); - - smartlist_split_string(pairs_tmp, res, "+", 0, 0); - if (smartlist_len(pairs_tmp)) { - char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1); - size_t last_len = strlen(last); - if (last_len > 2 && !strcmp(last+last_len-2, ".z")) { - last[last_len-2] = '\0'; - } - } - SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) { - if (strlen(cp) != HEX_DIGEST_LEN*2+1) { - log_info(LD_DIR, - "Skipping digest pair %s with non-standard length.", escaped(cp)); - } else if (cp[HEX_DIGEST_LEN] != '-') { - log_info(LD_DIR, - "Skipping digest pair %s with missing dash.", escaped(cp)); - } else { - fp_pair_t pair; - if (base16_decode(pair.first, DIGEST_LEN, - cp, HEX_DIGEST_LEN) != DIGEST_LEN || - base16_decode(pair.second,DIGEST_LEN, - cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) { - log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp)); - } else { - smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair))); - } - } - tor_free(cp); - } SMARTLIST_FOREACH_END(cp); - smartlist_free(pairs_tmp); - - /* Uniq-and-sort */ - smartlist_sort(pairs_result, compare_pairs_); - smartlist_uniq(pairs_result, compare_pairs_, tor_free_); - - smartlist_add_all(pairs_out, pairs_result); - smartlist_free(pairs_result); - return 0; -} - -/** Given a directory <b>resource</b> request, containing zero - * or more strings separated by plus signs, followed optionally by ".z", store - * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. - * - * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and - * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as - * a separator, delete all the elements that aren't base64-encoded digests, - * and decode the rest. If (flags & DSR_DIGEST256), these digests should be - * 256 bits long; else they should be 160. - * - * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. - */ -int -dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compressed_out, - int flags) -{ - const int decode_hex = flags & DSR_HEX; - const int decode_base64 = flags & DSR_BASE64; - const int digests_are_256 = flags & DSR_DIGEST256; - const int sort_uniq = flags & DSR_SORT_UNIQ; - - const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; - const int hex_digest_len = digests_are_256 ? - HEX_DIGEST256_LEN : HEX_DIGEST_LEN; - const int base64_digest_len = digests_are_256 ? - BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; - smartlist_t *fp_tmp = smartlist_new(); - - tor_assert(!(decode_hex && decode_base64)); - tor_assert(fp_out); - - smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); - if (compressed_out) - *compressed_out = 0; - if (smartlist_len(fp_tmp)) { - char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1); - size_t last_len = strlen(last); - if (last_len > 2 && !strcmp(last+last_len-2, ".z")) { - last[last_len-2] = '\0'; - if (compressed_out) - *compressed_out = 1; - } - } - if (decode_hex || decode_base64) { - const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; - int i; - char *cp, *d = NULL; - for (i = 0; i < smartlist_len(fp_tmp); ++i) { - cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != encoded_len) { - log_info(LD_DIR, - "Skipping digest %s with non-standard length.", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; - } - d = tor_malloc_zero(digest_len); - if (decode_hex ? - (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) : - (base64_decode(d, digest_len, cp, base64_digest_len) - != digest_len)) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; - } - smartlist_set(fp_tmp, i, d); - d = NULL; - again: - tor_free(cp); - tor_free(d); - } - } - if (sort_uniq) { - if (decode_hex || decode_base64) { - if (digests_are_256) { - smartlist_sort_digests256(fp_tmp); - smartlist_uniq_digests256(fp_tmp); - } else { - smartlist_sort_digests(fp_tmp); - smartlist_uniq_digests(fp_tmp); - } - } else { - smartlist_sort_strings(fp_tmp); - smartlist_uniq_strings(fp_tmp); - } - } - smartlist_add_all(fp_out, fp_tmp); - smartlist_free(fp_tmp); - return 0; -} - -/** As dir_split_resource_into_fingerprints, but instead fills - * <b>spool_out</b> with a list of spoolable_resource_t for the resource - * identified through <b>source</b>. */ -int -dir_split_resource_into_spoolable(const char *resource, - dir_spool_source_t source, - smartlist_t *spool_out, - int *compressed_out, - int flags) -{ - smartlist_t *fingerprints = smartlist_new(); - - tor_assert(flags & (DSR_HEX|DSR_BASE64)); - const size_t digest_len = - (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN; - - int r = dir_split_resource_into_fingerprints(resource, fingerprints, - compressed_out, flags); - /* This is not a very efficient implementation XXXX */ - SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) { - spooled_resource_t *spooled = - spooled_resource_new(source, digest, digest_len); - if (spooled) - smartlist_add(spool_out, spooled); - tor_free(digest); - } SMARTLIST_FOREACH_END(digest); - - smartlist_free(fingerprints); - return r; -} diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h new file mode 100644 index 000000000..6bbff2f84 --- /dev/null +++ b/src/feature/dirclient/dirclient.h @@ -0,0 +1,172 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirclient.h + * \brief Header file for dirclient.c. + **/ + +#ifndef TOR_DIRCLIENT_H +#define TOR_DIRCLIENT_H + +#include "feature/hs/hs_ident.h" + +int directories_have_accepted_server_descriptor(void); +void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, + dirinfo_type_t type, const char *payload, + size_t payload_len, size_t extrainfo_len); +MOCK_DECL(void, directory_get_from_dirserver, ( + uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags, + download_want_authority_t want_authority)); +void directory_get_from_all_authorities(uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource); + +/** Enumeration of ways to connect to a directory server */ +typedef enum { + /** Default: connect over a one-hop Tor circuit. Relays fall back to direct + * DirPort connections, clients, onion services, and bridges do not */ + DIRIND_ONEHOP=0, + /** Connect over a multi-hop anonymizing Tor circuit */ + DIRIND_ANONYMOUS=1, + /** Connect to the DirPort directly */ + DIRIND_DIRECT_CONN, + /** Connect over a multi-hop anonymizing Tor circuit to our dirport */ + DIRIND_ANON_DIRPORT, +} dir_indirection_t; + +int directory_must_use_begindir(const or_options_t *options); + +/** + * A directory_request_t describes the information about a directory request + * at the client side. It describes what we're going to ask for, which + * directory we're going to ask for it, how we're going to contact that + * directory, and (in some cases) what to do with it when we're done. + */ +typedef struct directory_request_t directory_request_t; +directory_request_t *directory_request_new(uint8_t dir_purpose); +void directory_request_free_(directory_request_t *req); +#define directory_request_free(req) \ + FREE_AND_NULL(directory_request_t, directory_request_free_, (req)) +void directory_request_set_or_addr_port(directory_request_t *req, + const tor_addr_port_t *p); +void directory_request_set_dir_addr_port(directory_request_t *req, + const tor_addr_port_t *p); +void directory_request_set_directory_id_digest(directory_request_t *req, + const char *digest); +struct circuit_guard_state_t; +void directory_request_set_guard_state(directory_request_t *req, + struct circuit_guard_state_t *state); +void directory_request_set_router_purpose(directory_request_t *req, + uint8_t router_purpose); +void directory_request_set_indirection(directory_request_t *req, + dir_indirection_t indirection); +void directory_request_set_resource(directory_request_t *req, + const char *resource); +void directory_request_set_payload(directory_request_t *req, + const char *payload, + size_t payload_len); +void directory_request_set_if_modified_since(directory_request_t *req, + time_t if_modified_since); +void directory_request_set_rend_query(directory_request_t *req, + const rend_data_t *query); +void directory_request_upload_set_hs_ident(directory_request_t *req, + const hs_ident_dir_conn_t *ident); +void directory_request_fetch_set_hs_ident(directory_request_t *req, + const hs_ident_dir_conn_t *ident); + +void directory_request_set_routerstatus(directory_request_t *req, + const routerstatus_t *rs); +void directory_request_add_header(directory_request_t *req, + const char *key, + const char *val); +MOCK_DECL(void, directory_initiate_request, (directory_request_t *request)); + +int router_supports_extrainfo(const char *identity_digest, int is_authority); + +void connection_dir_client_request_failed(dir_connection_t *conn); +void connection_dir_client_refetch_hsdesc_if_needed( + dir_connection_t *dir_conn); + +#ifdef DIRCLIENT_PRIVATE +struct directory_request_t { + /** + * These fields specify which directory we're contacting. Routerstatus, + * if present, overrides the other fields. + * + * @{ */ + tor_addr_port_t or_addr_port; + tor_addr_port_t dir_addr_port; + char digest[DIGEST_LEN]; + + const routerstatus_t *routerstatus; + /** @} */ + /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what + * kind of operation we'll be doing (upload/download), and of what kind + * of document. */ + uint8_t dir_purpose; + /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo + * and extrainfo docs. */ + uint8_t router_purpose; + /** Enum: determines whether to anonymize, and whether to use dirport or + * orport. */ + dir_indirection_t indirection; + /** Alias to the variable part of the URL for this request */ + const char *resource; + /** Alias to the payload to upload (if any) */ + const char *payload; + /** Number of bytes to upload from payload</b> */ + size_t payload_len; + /** Value to send in an if-modified-since header, or 0 for none. */ + time_t if_modified_since; + /** Hidden-service-specific information v2. */ + const rend_data_t *rend_query; + /** Extra headers to append to the request */ + struct config_line_t *additional_headers; + /** Hidden-service-specific information for v3+. */ + const hs_ident_dir_conn_t *hs_ident; + /** Used internally to directory.c: gets informed when the attempt to + * connect to the directory succeeds or fails, if that attempt bears on the + * directory's usability as a directory guard. */ + struct circuit_guard_state_t *guard_state; +}; + +/** A structure to hold arguments passed into each directory response + * handler */ +typedef struct response_handler_args_t { + int status_code; + const char *reason; + const char *body; + size_t body_len; + const char *headers; +} response_handler_args_t; + +enum compress_method_t; +STATIC int allowed_anonymous_connection_compression_method( + enum compress_method_t); +STATIC void warn_disallowed_anonymous_compression_method( + enum compress_method_t); + +STATIC int should_use_directory_guards(const or_options_t *options); +STATIC char *accept_encoding_header(void); +STATIC const char *dir_conn_purpose_to_string(int purpose); + +STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn, + const response_handler_args_t *args); +STATIC int handle_response_fetch_microdesc(dir_connection_t *conn, + const response_handler_args_t *args); + +STATIC int handle_response_fetch_consensus(dir_connection_t *conn, + const response_handler_args_t *args); + +STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose, + const char *resource); +#endif + +#endif /* !defined(TOR_DIRCLIENT_H) */ diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c new file mode 100644 index 000000000..017281e00 --- /dev/null +++ b/src/feature/dirclient/dlstatus.c @@ -0,0 +1,422 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define DLSTATUS_PRIVATE + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "feature/client/entrynodes.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/relay/router.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include "feature/dirclient/download_status_st.h" + +/** Decide which download schedule we want to use based on descriptor type + * in <b>dls</b> and <b>options</b>. + * + * Then, return the initial delay for that download schedule, in seconds. + * + * Helper function for download_status_increment_failure(), + * download_status_reset(), and download_status_increment_attempt(). */ +STATIC int +find_dl_min_delay(const download_status_t *dls, const or_options_t *options) +{ + tor_assert(dls); + tor_assert(options); + + switch (dls->schedule) { + case DL_SCHED_GENERIC: + /* Any other directory document */ + if (dir_server_mode(options)) { + /* A directory authority or directory mirror */ + return options->TestingServerDownloadInitialDelay; + } else { + return options->TestingClientDownloadInitialDelay; + } + case DL_SCHED_CONSENSUS: + if (!networkstatus_consensus_can_use_multiple_directories(options)) { + /* A public relay */ + return options->TestingServerConsensusDownloadInitialDelay; + } else { + /* A client or bridge */ + if (networkstatus_consensus_is_bootstrapping(time(NULL))) { + /* During bootstrapping */ + if (!networkstatus_consensus_can_use_extra_fallbacks(options)) { + /* A bootstrapping client without extra fallback directories */ + return options-> + ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay; + } else if (dls->want_authority) { + /* A bootstrapping client with extra fallback directories, but + * connecting to an authority */ + return + options->ClientBootstrapConsensusAuthorityDownloadInitialDelay; + } else { + /* A bootstrapping client connecting to extra fallback directories + */ + return + options->ClientBootstrapConsensusFallbackDownloadInitialDelay; + } + } else { + /* A client with a reasonably live consensus, with or without + * certificates */ + return options->TestingClientConsensusDownloadInitialDelay; + } + } + case DL_SCHED_BRIDGE: + if (options->UseBridges && num_bridges_usable(0) > 0) { + /* A bridge client that is sure that one or more of its bridges are + * running can afford to wait longer to update bridge descriptors. */ + return options->TestingBridgeDownloadInitialDelay; + } else { + /* A bridge client which might have no running bridges, must try to + * get bridge descriptors straight away. */ + return options->TestingBridgeBootstrapDownloadInitialDelay; + } + default: + tor_assert(0); + } + + /* Impossible, but gcc will fail with -Werror without a `return`. */ + return 0; +} + +/** As next_random_exponential_delay() below, but does not compute a random + * value. Instead, compute the range of values that + * next_random_exponential_delay() should use when computing its random value. + * Store the low bound into *<b>low_bound_out</b>, and the high bound into + * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less + * than the high bound. */ +STATIC void +next_random_exponential_delay_range(int *low_bound_out, + int *high_bound_out, + int delay, + int base_delay) +{ + // This is the "decorrelated jitter" approach, from + // https://www.awsarchitectureblog.com/2015/03/backoff.html + // The formula is + // sleep = min(cap, random_between(base, sleep * 3)) + + const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX; + *low_bound_out = base_delay; + if (delay_times_3 > base_delay) { + *high_bound_out = delay_times_3; + } else { + *high_bound_out = base_delay+1; + } +} + +/** Advance one delay step. The algorithm will generate a random delay, + * such that each failure is possibly (random) longer than the ones before. + * + * We then clamp that value to be no larger than max_delay, and return it. + * + * The <b>base_delay</b> parameter is lowest possible delay time (can't be + * zero); the <b>backoff_position</b> parameter is the number of times we've + * generated a delay; and the <b>delay</b> argument is the most recently used + * delay. + */ +STATIC int +next_random_exponential_delay(int delay, + int base_delay) +{ + /* Check preconditions */ + if (BUG(delay < 0)) + delay = 0; + + if (base_delay < 1) + base_delay = 1; + + int low_bound=0, high_bound=INT_MAX; + + next_random_exponential_delay_range(&low_bound, &high_bound, + delay, base_delay); + + return crypto_rand_int_range(low_bound, high_bound); +} + +/** Find the current delay for dls based on min_delay. + * + * This function sets dls->next_attempt_at based on now, and returns the delay. + * Helper for download_status_increment_failure and + * download_status_increment_attempt. */ +STATIC int +download_status_schedule_get_delay(download_status_t *dls, + int min_delay, + time_t now) +{ + tor_assert(dls); + /* If we're using random exponential backoff, we do need min/max delay */ + tor_assert(min_delay >= 0); + + int delay = INT_MAX; + uint8_t dls_schedule_position = (dls->increment_on + == DL_SCHED_INCREMENT_ATTEMPT + ? dls->n_download_attempts + : dls->n_download_failures); + + /* Check if we missed a reset somehow */ + IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) { + dls->last_backoff_position = 0; + dls->last_delay_used = 0; + } + + if (dls_schedule_position > 0) { + delay = dls->last_delay_used; + + while (dls->last_backoff_position < dls_schedule_position) { + /* Do one increment step */ + delay = next_random_exponential_delay(delay, min_delay); + /* Update our position */ + ++(dls->last_backoff_position); + } + } else { + /* If we're just starting out, use the minimum delay */ + delay = min_delay; + } + + /* Clamp it within min/max if we have them */ + if (min_delay >= 0 && delay < min_delay) delay = min_delay; + + /* Store it for next time */ + dls->last_backoff_position = dls_schedule_position; + dls->last_delay_used = delay; + + /* A negative delay makes no sense. Knowing that delay is + * non-negative allows us to safely do the wrapping check below. */ + tor_assert(delay >= 0); + + /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction + * that won't overflow (since delay is non-negative). */ + if (delay < INT_MAX && now <= TIME_MAX - delay) { + dls->next_attempt_at = now+delay; + } else { + dls->next_attempt_at = TIME_MAX; + } + + return delay; +} + +/* Log a debug message about item, which increments on increment_action, has + * incremented dls_n_download_increments times. The message varies based on + * was_schedule_incremented (if not, not_incremented_response is logged), and + * the values of increment, dls_next_attempt_at, and now. + * Helper for download_status_increment_failure and + * download_status_increment_attempt. */ +static void +download_status_log_helper(const char *item, int was_schedule_incremented, + const char *increment_action, + const char *not_incremented_response, + uint8_t dls_n_download_increments, int increment, + time_t dls_next_attempt_at, time_t now) +{ + if (item) { + if (!was_schedule_incremented) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.", + item, increment_action, (int)dls_n_download_increments, + not_incremented_response); + else if (increment == 0) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.", + item, increment_action, (int)dls_n_download_increments); + else if (dls_next_attempt_at < TIME_MAX) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.", + item, increment_action, (int)dls_n_download_increments, + (int)(dls_next_attempt_at-now)); + else + log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.", + item, increment_action, (int)dls_n_download_increments); + } +} + +/** Determine when a failed download attempt should be retried. + * Called when an attempt to download <b>dls</b> has failed with HTTP status + * <b>status_code</b>. Increment the failure count (if the code indicates a + * real failure, or if we're a server) and set <b>dls</b>->next_attempt_at to + * an appropriate time in the future and return it. + * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the + * failure count, and return a time in the far future for the next attempt (to + * avoid an immediate retry). */ +time_t +download_status_increment_failure(download_status_t *dls, int status_code, + const char *item, int server, time_t now) +{ + (void) status_code; // XXXX no longer used. + (void) server; // XXXX no longer used. + int increment = -1; + int min_delay = 0; + + tor_assert(dls); + + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + download_status_reset(dls); + } + + /* count the failure */ + if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) { + ++dls->n_download_failures; + } + + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { + /* We don't find out that a failure-based schedule has attempted a + * connection until that connection fails. + * We'll never find out about successful connections, but this doesn't + * matter, because schedules are reset after a successful download. + */ + if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) + ++dls->n_download_attempts; + + /* only return a failure retry time if this schedule increments on failures + */ + min_delay = find_dl_min_delay(dls, get_options()); + increment = download_status_schedule_get_delay(dls, min_delay, now); + } + + download_status_log_helper(item, !dls->increment_on, "failed", + "concurrently", dls->n_download_failures, + increment, + download_status_get_next_attempt_at(dls), + now); + + if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) { + /* stop this schedule retrying on failure, it will launch concurrent + * connections instead */ + return TIME_MAX; + } else { + return download_status_get_next_attempt_at(dls); + } +} + +/** Determine when the next download attempt should be made when using an + * attempt-based (potentially concurrent) download schedule. + * Called when an attempt to download <b>dls</b> is being initiated. + * Increment the attempt count and set <b>dls</b>->next_attempt_at to an + * appropriate time in the future and return it. + * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment + * the attempts, and return a time in the far future (to avoid launching a + * concurrent attempt). */ +time_t +download_status_increment_attempt(download_status_t *dls, const char *item, + time_t now) +{ + int delay = -1; + int min_delay = 0; + + tor_assert(dls); + + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + download_status_reset(dls); + } + + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { + /* this schedule should retry on failure, and not launch any concurrent + attempts */ + log_warn(LD_BUG, "Tried to launch an attempt-based connection on a " + "failure-based schedule."); + return TIME_MAX; + } + + if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) + ++dls->n_download_attempts; + + min_delay = find_dl_min_delay(dls, get_options()); + delay = download_status_schedule_get_delay(dls, min_delay, now); + + download_status_log_helper(item, dls->increment_on, "attempted", + "on failure", dls->n_download_attempts, + delay, download_status_get_next_attempt_at(dls), + now); + + return download_status_get_next_attempt_at(dls); +} + +static time_t +download_status_get_initial_delay_from_now(const download_status_t *dls) +{ + /* We use constant initial delays, even in exponential backoff + * schedules. */ + return time(NULL) + find_dl_min_delay(dls, get_options()); +} + +/** Reset <b>dls</b> so that it will be considered downloadable + * immediately, and/or to show that we don't need it anymore. + * + * Must be called to initialise a download schedule, otherwise the zeroth item + * in the schedule will never be used. + * + * (We find the zeroth element of the download schedule, and set + * next_attempt_at to be the appropriate offset from 'now'. In most + * cases this means setting it to 'now', so the item will be immediately + * downloadable; when using authorities with fallbacks, there is a few seconds' + * delay.) */ +void +download_status_reset(download_status_t *dls) +{ + if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD + || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD) + return; /* Don't reset this. */ + + dls->n_download_failures = 0; + dls->n_download_attempts = 0; + dls->next_attempt_at = download_status_get_initial_delay_from_now(dls); + dls->last_backoff_position = 0; + dls->last_delay_used = 0; + /* Don't reset dls->want_authority or dls->increment_on */ +} + +/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is + * ready to get its download reattempted. */ +int +download_status_is_ready(download_status_t *dls, time_t now) +{ + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + download_status_reset(dls); + } + + return download_status_get_next_attempt_at(dls) <= now; +} + +/** Mark <b>dl</b> as never downloadable. */ +void +download_status_mark_impossible(download_status_t *dl) +{ + dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; + dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD; +} + +/** Return the number of failures on <b>dls</b> since the last success (if + * any). */ +int +download_status_get_n_failures(const download_status_t *dls) +{ + return dls->n_download_failures; +} + +/** Return the number of attempts to download <b>dls</b> since the last success + * (if any). This can differ from download_status_get_n_failures() due to + * outstanding concurrent attempts. */ +int +download_status_get_n_attempts(const download_status_t *dls) +{ + return dls->n_download_attempts; +} + +/** Return the next time to attempt to download <b>dls</b>. */ +time_t +download_status_get_next_attempt_at(const download_status_t *dls) +{ + /* dls wasn't reset before it was used */ + if (dls->next_attempt_at == 0) { + /* so give the answer we would have given if it had been */ + return download_status_get_initial_delay_from_now(dls); + } + + return dls->next_attempt_at; +} diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h new file mode 100644 index 000000000..aeceb8cb2 --- /dev/null +++ b/src/feature/dirclient/dlstatus.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dlstatus.h + * \brief Header file for dlstatus.c. + **/ + +#ifndef TOR_DLSTATUS_H +#define TOR_DLSTATUS_H + +time_t download_status_increment_failure(download_status_t *dls, + int status_code, const char *item, + int server, time_t now); +time_t download_status_increment_attempt(download_status_t *dls, + const char *item, time_t now); +/** Increment the failure count of the download_status_t <b>dls</b>, with + * the optional status code <b>sc</b>. */ +#define download_status_failed(dls, sc) \ + download_status_increment_failure((dls), (sc), NULL, \ + dir_server_mode(get_options()), \ + time(NULL)) + +void download_status_reset(download_status_t *dls); +int download_status_is_ready(download_status_t *dls, time_t now); +time_t download_status_get_next_attempt_at(const download_status_t *dls); +void download_status_mark_impossible(download_status_t *dl); + +int download_status_get_n_failures(const download_status_t *dls); +int download_status_get_n_attempts(const download_status_t *dls); + +#ifdef DLSTATUS_PRIVATE +STATIC int download_status_schedule_get_delay(download_status_t *dls, + int min_delay, + time_t now); + +STATIC int find_dl_min_delay(const download_status_t *dls, + const or_options_t *options); + +STATIC int next_random_exponential_delay(int delay, + int base_delay); + +STATIC void next_random_exponential_delay_range(int *low_bound_out, + int *high_bound_out, + int delay, + int base_delay); + +/* no more than quadruple the previous delay (multiplier + 1) */ +#define DIR_DEFAULT_RANDOM_MULTIPLIER (3) +/* no more than triple the previous delay */ +#define DIR_TEST_NET_RANDOM_MULTIPLIER (2) + +#endif + +#endif /* !defined(TOR_DLSTATUS_H) */ diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c new file mode 100644 index 000000000..8116d72c6 --- /dev/null +++ b/src/feature/dircommon/directory.c @@ -0,0 +1,651 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "feature/dircache/dircache.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dircommon/directory.h" +#include "feature/dircommon/fp_pair.h" +#include "feature/stats/geoip.h" +#include "lib/compress/compress.h" + +#include "feature/dircommon/dir_connection_st.h" +#include "feature/nodelist/routerinfo_st.h" + +/** + * \file directory.c + * \brief Code to send and fetch information from directory authorities and + * caches via HTTP. + * + * Directory caches and authorities use dirserv.c to generate the results of a + * query and stream them to the connection; clients use routerparse.c to parse + * them. + * + * Every directory request has a dir_connection_t on the client side and on + * the server side. In most cases, the dir_connection_t object is a linked + * connection, tunneled through an edge_connection_t so that it can be a + * stream on the Tor network. The only non-tunneled connections are those + * that are used to upload material (descriptors and votes) to authorities. + * Among tunneled connections, some use one-hop circuits, and others use + * multi-hop circuits for anonymity. + * + * Directory requests are launched by calling + * directory_initiate_request(). This + * launch the connection, will construct an HTTP request with + * directory_send_command(), send the and wait for a response. The client + * later handles the response with connection_dir_client_reached_eof(), + * which passes the information received to another part of Tor. + * + * On the server side, requests are read in directory_handle_command(), + * which dispatches first on the request type (GET or POST), and then on + * the URL requested. GET requests are processed with a table-based + * dispatcher in url_table[]. The process of handling larger GET requests + * is complicated because we need to avoid allocating a copy of all the + * data to be sent to the client in one huge buffer. Instead, we spool the + * data into the buffer using logic in connection_dirserv_flushed_some() in + * dirserv.c. (TODO: If we extended buf.c to have a zero-copy + * reference-based buffer type, we could remove most of that code, at the + * cost of a bit more reference counting.) + **/ + +/* In-points to directory.c: + * + * - directory_post_to_dirservers(), called from + * router_upload_dir_desc_to_dirservers() in router.c + * upload_service_descriptor() in rendservice.c + * - directory_get_from_dirserver(), called from + * rend_client_refetch_renddesc() in rendclient.c + * run_scheduled_events() in main.c + * do_hup() in main.c + * - connection_dir_process_inbuf(), called from + * connection_process_inbuf() in connection.c + * - connection_dir_finished_flushing(), called from + * connection_finished_flushing() in connection.c + * - connection_dir_finished_connecting(), called from + * connection_finished_connecting() in connection.c + */ + +/** Convert a connection_t* to a dir_connection_t*; assert if the cast is + * invalid. */ +dir_connection_t * +TO_DIR_CONN(connection_t *c) +{ + tor_assert(c->magic == DIR_CONNECTION_MAGIC); + return DOWNCAST(dir_connection_t, c); +} + +/** Return false if the directory purpose <b>dir_purpose</b> + * does not require an anonymous (three-hop) connection. + * + * Return true 1) by default, 2) if all directory actions have + * specifically been configured to be over an anonymous connection, + * or 3) if the router is a bridge */ +int +purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource) +{ + if (get_options()->AllDirActionsPrivate) + return 1; + + if (router_purpose == ROUTER_PURPOSE_BRIDGE) { + if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC + && resource && !strcmp(resource, "authority.z")) { + /* We are asking a bridge for its own descriptor. That doesn't need + anonymity. */ + return 0; + } + /* Assume all other bridge stuff needs anonymity. */ + return 1; /* if no circuits yet, this might break bootstrapping, but it's + * needed to be safe. */ + } + + switch (dir_purpose) + { + case DIR_PURPOSE_UPLOAD_DIR: + case DIR_PURPOSE_UPLOAD_VOTE: + case DIR_PURPOSE_UPLOAD_SIGNATURES: + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CONSENSUS: + case DIR_PURPOSE_FETCH_CERTIFICATE: + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + case DIR_PURPOSE_FETCH_MICRODESC: + return 0; + case DIR_PURPOSE_HAS_FETCHED_HSDESC: + case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2: + case DIR_PURPOSE_UPLOAD_RENDDESC_V2: + case DIR_PURPOSE_FETCH_RENDDESC_V2: + case DIR_PURPOSE_FETCH_HSDESC: + case DIR_PURPOSE_UPLOAD_HSDESC: + return 1; + case DIR_PURPOSE_SERVER: + default: + log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d", + dir_purpose, router_purpose); + tor_assert_nonfatal_unreached(); + return 1; /* Assume it needs anonymity; better safe than sorry. */ + } +} + +/** Return a newly allocated string describing <b>auth</b>. Only describes + * authority features. */ +char * +authdir_type_to_string(dirinfo_type_t auth) +{ + char *result; + smartlist_t *lst = smartlist_new(); + if (auth & V3_DIRINFO) + smartlist_add(lst, (void*)"V3"); + if (auth & BRIDGE_DIRINFO) + smartlist_add(lst, (void*)"Bridge"); + if (smartlist_len(lst)) { + result = smartlist_join_strings(lst, ", ", 0, NULL); + } else { + result = tor_strdup("[Not an authority]"); + } + smartlist_free(lst); + return result; +} + +/** Return true iff anything we say on <b>conn</b> is being encrypted before + * we send it to the client/server. */ +int +connection_dir_is_encrypted(const dir_connection_t *conn) +{ + /* Right now it's sufficient to see if conn is or has been linked, since + * the only thing it could be linked to is an edge connection on a + * circuit, and the only way it could have been unlinked is at the edge + * connection getting closed. + */ + return TO_CONN(conn)->linked; +} + +/** Parse an HTTP request line at the start of a headers string. On failure, + * return -1. On success, set *<b>command_out</b> to a copy of the HTTP + * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and + * return 0. */ +int +parse_http_command(const char *headers, char **command_out, char **url_out) +{ + const char *command, *end_of_command; + char *s, *start, *tmp; + + s = (char *)eat_whitespace_no_nl(headers); + if (!*s) return -1; + command = s; + s = (char *)find_whitespace(s); /* get past GET/POST */ + if (!*s) return -1; + end_of_command = s; + s = (char *)eat_whitespace_no_nl(s); + if (!*s) return -1; + start = s; /* this is the URL, assuming it's valid */ + s = (char *)find_whitespace(start); + if (!*s) return -1; + + /* tolerate the http[s] proxy style of putting the hostname in the url */ + if (s-start >= 4 && !strcmpstart(start,"http")) { + tmp = start + 4; + if (*tmp == 's') + tmp++; + if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { + tmp = strchr(tmp+3, '/'); + if (tmp && tmp < s) { + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); + start = tmp; + } + } + } + + /* Check if the header is well formed (next sequence + * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */ + { + unsigned minor_ver; + char ch; + char *e = (char *)eat_whitespace_no_nl(s); + if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) { + return -1; + } + if (ch != '\r') + return -1; + } + + *url_out = tor_memdup_nulterm(start, s-start); + *command_out = tor_memdup_nulterm(command, end_of_command - command); + return 0; +} + +/** Return a copy of the first HTTP header in <b>headers</b> whose key is + * <b>which</b>. The key should be given with a terminating colon and space; + * this function copies everything after, up to but not including the + * following \r\n. */ +char * +http_get_header(const char *headers, const char *which) +{ + const char *cp = headers; + while (cp) { + if (!strcasecmpstart(cp, which)) { + char *eos; + cp += strlen(which); + if ((eos = strchr(cp,'\r'))) + return tor_strndup(cp, eos-cp); + else + return tor_strdup(cp); + } + cp = strchr(cp, '\n'); + if (cp) + ++cp; + } + return NULL; +} +/** Parse an HTTP response string <b>headers</b> of the form + * \verbatim + * "HTTP/1.%d %d%s\r\n...". + * \endverbatim + * + * If it's well-formed, assign the status code to *<b>code</b> and + * return 0. Otherwise, return -1. + * + * On success: If <b>date</b> is provided, set *date to the Date + * header in the http headers, or 0 if no such header is found. If + * <b>compression</b> is provided, set *<b>compression</b> to the + * compression method given in the Content-Encoding header, or 0 if no + * such header is found, or -1 if the value of the header is not + * recognized. If <b>reason</b> is provided, strdup the reason string + * into it. + */ +int +parse_http_response(const char *headers, int *code, time_t *date, + compress_method_t *compression, char **reason) +{ + unsigned n1, n2; + char datestr[RFC1123_TIME_LEN+1]; + smartlist_t *parsed_headers; + tor_assert(headers); + tor_assert(code); + + while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */ + + if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 || + (n1 != 0 && n1 != 1) || + (n2 < 100 || n2 >= 600)) { + log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers)); + return -1; + } + *code = n2; + + parsed_headers = smartlist_new(); + smartlist_split_string(parsed_headers, headers, "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + if (reason) { + smartlist_t *status_line_elements = smartlist_new(); + tor_assert(smartlist_len(parsed_headers)); + smartlist_split_string(status_line_elements, + smartlist_get(parsed_headers, 0), + " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); + tor_assert(smartlist_len(status_line_elements) <= 3); + if (smartlist_len(status_line_elements) == 3) { + *reason = smartlist_get(status_line_elements, 2); + smartlist_set(status_line_elements, 2, NULL); /* Prevent free */ + } + SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp)); + smartlist_free(status_line_elements); + } + if (date) { + *date = 0; + SMARTLIST_FOREACH(parsed_headers, const char *, s, + if (!strcmpstart(s, "Date: ")) { + strlcpy(datestr, s+6, sizeof(datestr)); + /* This will do nothing on failure, so we don't need to check + the result. We shouldn't warn, since there are many other valid + date formats besides the one we use. */ + parse_rfc1123_time(datestr, date); + break; + }); + } + if (compression) { + const char *enc = NULL; + SMARTLIST_FOREACH(parsed_headers, const char *, s, + if (!strcmpstart(s, "Content-Encoding: ")) { + enc = s+18; break; + }); + + if (enc == NULL) + *compression = NO_METHOD; + else { + *compression = compression_method_get_by_name(enc); + + if (*compression == UNKNOWN_METHOD) + log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", + escaped(enc)); + } + } + SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s)); + smartlist_free(parsed_headers); + + return 0; +} + +/** If any directory object is arriving, and it's over 10MB large, we're + * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never + * ask for more than 96 router descriptors at a time.) + */ +#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20)) + +#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5) + +/** Read handler for directory connections. (That's connections <em>to</em> + * directory servers and connections <em>at</em> directory servers.) + */ +int +connection_dir_process_inbuf(dir_connection_t *conn) +{ + size_t max_size; + tor_assert(conn); + tor_assert(conn->base_.type == CONN_TYPE_DIR); + + /* Directory clients write, then read data until they receive EOF; + * directory servers read data until they get an HTTP command, then + * write their response (when it's finished flushing, they mark for + * close). + */ + + /* If we're on the dirserver side, look for a command. */ + if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) { + if (directory_handle_command(conn) < 0) { + connection_mark_for_close(TO_CONN(conn)); + return -1; + } + return 0; + } + + max_size = + (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ? + MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE; + + if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) { + log_warn(LD_HTTP, + "Too much data received from directory connection (%s): " + "denial of service attempt, or you need to upgrade?", + conn->base_.address); + connection_mark_for_close(TO_CONN(conn)); + return -1; + } + + if (!conn->base_.inbuf_reached_eof) + log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf."); + return 0; +} + +/** Called when we're about to finally unlink and free a directory connection: + * perform necessary accounting and cleanup */ +void +connection_dir_about_to_close(dir_connection_t *dir_conn) +{ + connection_t *conn = TO_CONN(dir_conn); + + if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) { + /* It's a directory connection and connecting or fetching + * failed: forget about this router, and maybe try again. */ + connection_dir_client_request_failed(dir_conn); + } + + connection_dir_client_refetch_hsdesc_if_needed(dir_conn); +} + +/** Write handler for directory connections; called when all data has + * been flushed. Close the connection or wait for a response as + * appropriate. + */ +int +connection_dir_finished_flushing(dir_connection_t *conn) +{ + tor_assert(conn); + tor_assert(conn->base_.type == CONN_TYPE_DIR); + + if (conn->base_.marked_for_close) + return 0; + + /* Note that we have finished writing the directory response. For direct + * connections this means we're done; for tunneled connections it's only + * an intermediate step. */ + if (conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + switch (conn->base_.state) { + case DIR_CONN_STATE_CONNECTING: + case DIR_CONN_STATE_CLIENT_SENDING: + log_debug(LD_DIR,"client finished sending command."); + conn->base_.state = DIR_CONN_STATE_CLIENT_READING; + return 0; + case DIR_CONN_STATE_SERVER_WRITING: + if (conn->spool) { + log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!"); + connection_mark_for_close(TO_CONN(conn)); + } else { + log_debug(LD_DIRSERV, "Finished writing server response. Closing."); + connection_mark_for_close(TO_CONN(conn)); + } + return 0; + default: + log_warn(LD_BUG,"called in unexpected state %d.", + conn->base_.state); + tor_fragile_assert(); + return -1; + } + return 0; +} + +/** Connected handler for directory connections: begin sending data to the + * server, and return 0. + * Only used when connections don't immediately connect. */ +int +connection_dir_finished_connecting(dir_connection_t *conn) +{ + tor_assert(conn); + tor_assert(conn->base_.type == CONN_TYPE_DIR); + tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING); + + log_debug(LD_HTTP,"Dir connection to router %s:%u established.", + conn->base_.address,conn->base_.port); + + /* start flushing conn */ + conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; + return 0; +} + +/** Helper. Compare two fp_pair_t objects, and return negative, 0, or + * positive as appropriate. */ +static int +compare_pairs_(const void **a, const void **b) +{ + const fp_pair_t *fp1 = *a, *fp2 = *b; + int r; + if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN))) + return r; + else + return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN); +} + +/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each + * FP is a hex-encoded fingerprint, into a sequence of distinct sorted + * fp_pair_t. Skip malformed pairs. On success, return 0 and add those + * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */ +int +dir_split_resource_into_fingerprint_pairs(const char *res, + smartlist_t *pairs_out) +{ + smartlist_t *pairs_tmp = smartlist_new(); + smartlist_t *pairs_result = smartlist_new(); + + smartlist_split_string(pairs_tmp, res, "+", 0, 0); + if (smartlist_len(pairs_tmp)) { + char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1); + size_t last_len = strlen(last); + if (last_len > 2 && !strcmp(last+last_len-2, ".z")) { + last[last_len-2] = '\0'; + } + } + SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) { + if (strlen(cp) != HEX_DIGEST_LEN*2+1) { + log_info(LD_DIR, + "Skipping digest pair %s with non-standard length.", escaped(cp)); + } else if (cp[HEX_DIGEST_LEN] != '-') { + log_info(LD_DIR, + "Skipping digest pair %s with missing dash.", escaped(cp)); + } else { + fp_pair_t pair; + if (base16_decode(pair.first, DIGEST_LEN, + cp, HEX_DIGEST_LEN) != DIGEST_LEN || + base16_decode(pair.second,DIGEST_LEN, + cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) { + log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp)); + } else { + smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair))); + } + } + tor_free(cp); + } SMARTLIST_FOREACH_END(cp); + smartlist_free(pairs_tmp); + + /* Uniq-and-sort */ + smartlist_sort(pairs_result, compare_pairs_); + smartlist_uniq(pairs_result, compare_pairs_, tor_free_); + + smartlist_add_all(pairs_out, pairs_result); + smartlist_free(pairs_result); + return 0; +} + +/** Given a directory <b>resource</b> request, containing zero + * or more strings separated by plus signs, followed optionally by ".z", store + * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. + */ +int +dir_split_resource_into_fingerprints(const char *resource, + smartlist_t *fp_out, int *compressed_out, + int flags) +{ + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; + smartlist_t *fp_tmp = smartlist_new(); + + tor_assert(!(decode_hex && decode_base64)); + tor_assert(fp_out); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); + if (compressed_out) + *compressed_out = 0; + if (smartlist_len(fp_tmp)) { + char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1); + size_t last_len = strlen(last); + if (last_len > 2 && !strcmp(last+last_len-2, ".z")) { + last[last_len-2] = '\0'; + if (compressed_out) + *compressed_out = 1; + } + } + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; + int i; + char *cp, *d = NULL; + for (i = 0; i < smartlist_len(fp_tmp); ++i) { + cp = smartlist_get(fp_tmp, i); + if (strlen(cp) != encoded_len) { + log_info(LD_DIR, + "Skipping digest %s with non-standard length.", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; + } + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) : + (base64_decode(d, digest_len, cp, base64_digest_len) + != digest_len)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; + } + smartlist_set(fp_tmp, i, d); + d = NULL; + again: + tor_free(cp); + tor_free(d); + } + } + if (sort_uniq) { + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { + smartlist_sort_strings(fp_tmp); + smartlist_uniq_strings(fp_tmp); + } + } + smartlist_add_all(fp_out, fp_tmp); + smartlist_free(fp_tmp); + return 0; +} + +/** As dir_split_resource_into_fingerprints, but instead fills + * <b>spool_out</b> with a list of spoolable_resource_t for the resource + * identified through <b>source</b>. */ +int +dir_split_resource_into_spoolable(const char *resource, + dir_spool_source_t source, + smartlist_t *spool_out, + int *compressed_out, + int flags) +{ + smartlist_t *fingerprints = smartlist_new(); + + tor_assert(flags & (DSR_HEX|DSR_BASE64)); + const size_t digest_len = + (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN; + + int r = dir_split_resource_into_fingerprints(resource, fingerprints, + compressed_out, flags); + /* This is not a very efficient implementation XXXX */ + SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) { + spooled_resource_t *spooled = + spooled_resource_new(source, digest, digest_len); + if (spooled) + smartlist_add(spool_out, spooled); + tor_free(digest); + } SMARTLIST_FOREACH_END(digest); + + smartlist_free(fingerprints); + return r; +} diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h new file mode 100644 index 000000000..ec95573f5 --- /dev/null +++ b/src/feature/dircommon/directory.h @@ -0,0 +1,129 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file directory.h + * \brief Header file for directory.c. + **/ + +#ifndef TOR_DIRECTORY_H +#define TOR_DIRECTORY_H + +dir_connection_t *TO_DIR_CONN(connection_t *c); + +#define DIR_CONN_STATE_MIN_ 1 +/** State for connection to directory server: waiting for connect(). */ +#define DIR_CONN_STATE_CONNECTING 1 +/** State for connection to directory server: sending HTTP request. */ +#define DIR_CONN_STATE_CLIENT_SENDING 2 +/** State for connection to directory server: reading HTTP response. */ +#define DIR_CONN_STATE_CLIENT_READING 3 +/** State for connection to directory server: happy and finished. */ +#define DIR_CONN_STATE_CLIENT_FINISHED 4 +/** State for connection at directory server: waiting for HTTP request. */ +#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5 +/** State for connection at directory server: sending HTTP response. */ +#define DIR_CONN_STATE_SERVER_WRITING 6 +#define DIR_CONN_STATE_MAX_ 6 + +#define DIR_PURPOSE_MIN_ 4 +/** A connection to a directory server: set after a v2 rendezvous + * descriptor is downloaded. */ +#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4 +/** A connection to a directory server: download one or more server + * descriptors. */ +#define DIR_PURPOSE_FETCH_SERVERDESC 6 +/** A connection to a directory server: download one or more extra-info + * documents. */ +#define DIR_PURPOSE_FETCH_EXTRAINFO 7 +/** A connection to a directory server: upload a server descriptor. */ +#define DIR_PURPOSE_UPLOAD_DIR 8 +/** A connection to a directory server: upload a v3 networkstatus vote. */ +#define DIR_PURPOSE_UPLOAD_VOTE 10 +/** A connection to a directory server: upload a v3 consensus signature */ +#define DIR_PURPOSE_UPLOAD_SIGNATURES 11 +/** A connection to a directory server: download one or more v3 networkstatus + * votes. */ +#define DIR_PURPOSE_FETCH_STATUS_VOTE 12 +/** A connection to a directory server: download a v3 detached signatures + * object for a consensus. */ +#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13 +/** A connection to a directory server: download a v3 networkstatus + * consensus. */ +#define DIR_PURPOSE_FETCH_CONSENSUS 14 +/** A connection to a directory server: download one or more directory + * authority certificates. */ +#define DIR_PURPOSE_FETCH_CERTIFICATE 15 + +/** Purpose for connection at a directory server. */ +#define DIR_PURPOSE_SERVER 16 +/** A connection to a hidden service directory server: upload a v2 rendezvous + * descriptor. */ +#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17 +/** A connection to a hidden service directory server: download a v2 rendezvous + * descriptor. */ +#define DIR_PURPOSE_FETCH_RENDDESC_V2 18 +/** A connection to a directory server: download a microdescriptor. */ +#define DIR_PURPOSE_FETCH_MICRODESC 19 +/** A connection to a hidden service directory: upload a v3 descriptor. */ +#define DIR_PURPOSE_UPLOAD_HSDESC 20 +/** A connection to a hidden service directory: fetch a v3 descriptor. */ +#define DIR_PURPOSE_FETCH_HSDESC 21 +/** A connection to a directory server: set after a hidden service descriptor + * is downloaded. */ +#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22 +#define DIR_PURPOSE_MAX_ 22 + +/** True iff <b>p</b> is a purpose corresponding to uploading + * data to a directory server. */ +#define DIR_PURPOSE_IS_UPLOAD(p) \ + ((p)==DIR_PURPOSE_UPLOAD_DIR || \ + (p)==DIR_PURPOSE_UPLOAD_VOTE || \ + (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \ + (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 || \ + (p)==DIR_PURPOSE_UPLOAD_HSDESC) + +enum compress_method_t; +int parse_http_response(const char *headers, int *code, time_t *date, + enum compress_method_t *compression, char **response); +int parse_http_command(const char *headers, + char **command_out, char **url_out); +char *http_get_header(const char *headers, const char *which); + +int connection_dir_is_encrypted(const dir_connection_t *conn); +int connection_dir_reached_eof(dir_connection_t *conn); +int connection_dir_process_inbuf(dir_connection_t *conn); +int connection_dir_finished_flushing(dir_connection_t *conn); +int connection_dir_finished_connecting(dir_connection_t *conn); +void connection_dir_about_to_close(dir_connection_t *dir_conn); + +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) +int dir_split_resource_into_fingerprints(const char *resource, + smartlist_t *fp_out, int *compressed_out, + int flags); +enum dir_spool_source_t; +int dir_split_resource_into_spoolable(const char *resource, + enum dir_spool_source_t source, + smartlist_t *spool_out, + int *compressed_out, + int flags); +int dir_split_resource_into_fingerprint_pairs(const char *res, + smartlist_t *pairs_out); +char *directory_dump_request_log(void); +void note_request(const char *key, size_t bytes); + +int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource); + +char *authdir_type_to_string(dirinfo_type_t auth); + +#define X_ADDRESS_HEADER "X-Your-Address-Is: " +#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: " + +#endif /* !defined(TOR_DIRECTORY_H) */ diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 563f6c6b3..1ba0428c5 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -19,7 +19,8 @@ #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/hs/hs_cache.h" #include "feature/hs/hs_cell.h" #include "feature/hs/hs_circuit.h" diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 643240fb6..dc7bb41ee 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -19,7 +19,8 @@ #include "lib/crypt_ops/crypto_util.h" #include "lib/crypt_ops/crypto_ope.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c index 74a6c5b5f..e070cb142 100644 --- a/src/feature/nodelist/authcert.c +++ b/src/feature/nodelist/authcert.c @@ -23,7 +23,9 @@ #include "core/mainloop/mainloop.h" #include "core/or/policies.h" #include "feature/client/bridges.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dircommon/fp_pair.h" #include "feature/nodelist/authcert.h" #include "feature/nodelist/dirlist.h" diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index 8f78df521..0935145d8 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -29,7 +29,7 @@ #include "app/config/config.h" #include "core/or/policies.h" #include "feature/control/control.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 203d079b1..96696b603 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -14,8 +14,9 @@

#include "core/or/circuitbuild.h" #include "app/config/config.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "feature/dircache/dirserv.h" +#include "feature/dirclient/dlstatus.h" #include "feature/client/entrynodes.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index c64c5176e..c6a51aefc 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -51,7 +51,9 @@ #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/reachability.h" #include "core/or/dos.h" diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c index a9f045d45..aab0bbe9f 100644 --- a/src/feature/nodelist/node_select.c +++ b/src/feature/nodelist/node_select.c @@ -18,7 +18,8 @@ #include "core/or/policies.h" #include "core/or/reasons.h" #include "feature/client/entrynodes.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 9a4e48b07..5c6a10477 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -69,8 +69,10 @@ #include "feature/client/bridges.h" #include "feature/control/control.h" #include "feature/dirauth/mode.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/dircache/dirserv.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dirauth/reachability.h" #include "feature/dirauth/process_descs.h" #include "feature/nodelist/authcert.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 447c21e89..622cfeb86 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -16,7 +16,8 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/crypt_ops/crypto_curve25519.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/process_descs.h" #include "feature/relay/dns.h" diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index a2b494bd2..d1310699f 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -19,7 +19,8 @@ #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/hs/hs_circuit.h" #include "feature/hs/hs_client.h" #include "feature/hs/hs_common.h" diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index ceb4d958d..7966ab846 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -19,7 +19,8 @@ #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/hs/hs_common.h" #include "feature/hs/hs_config.h" #include "core/mainloop/mainloop.h" diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 2fbb27561..06483282b 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -4,14 +4,14 @@ #include "orconfig.h"

#define BUFFERS_PRIVATE -#define DIRECTORY_PRIVATE +#define DIRCACHE_PRIVATE

#include "core/or/or.h" #include "lib/err/backtrace.h" #include "lib/container/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" -#include "feature/dircache/directory.h" +#include "feature/dircache/dircache.h" #include "lib/log/log.h"

#include "feature/dircommon/dir_connection_st.h" @@ -132,4 +132,3 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)

return 0; } - diff --git a/src/test/test_connection.c b/src/test/test_connection.c index d85ede61b..0013f47fb 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -18,7 +18,7 @@ #include "feature/nodelist/nodelist.h" #include "feature/nodelist/networkstatus.h" #include "feature/rend/rendcache.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "core/or/connection_or.h" #include "lib/net/resolve.h"

diff --git a/src/test/test_dir.c b/src/test/test_dir.c index b2c3f4426..686b03949 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -9,8 +9,11 @@ #define BWAUTH_PRIVATE #define CONFIG_PRIVATE #define CONTROL_PRIVATE +#define DIRCACHE_PRIVATE +#define DIRCLIENT_PRIVATE #define DIRSERV_PRIVATE #define DIRVOTE_PRIVATE +#define DLSTATUS_PRIVATE #define HIBERNATE_PRIVATE #define NETWORKSTATUS_PRIVATE #define NODE_SELECT_PRIVATE @@ -30,8 +33,11 @@ #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dirauth/bwauth.h" +#include "feature/dircache/dircache.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/process_descs.h" #include "feature/dirauth/dirvote.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 2cfed16b5..f05b0fc79 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -8,12 +8,14 @@ #define CONNECTION_PRIVATE #define CONFIG_PRIVATE #define RENDCACHE_PRIVATE +#define DIRCACHE_PRIVATE

#include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/dircache/consdiffmgr.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dircache/dircache.h" #include "test/test.h" #include "lib/compress/compress.h" #include "feature/rend/rendcommon.h" diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index cb694106c..ebe9accf5 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -8,7 +8,7 @@ #define STATEFILE_PRIVATE #define ENTRYNODES_PRIVATE #define ROUTERLIST_PRIVATE -#define DIRECTORY_PRIVATE +#define DIRCLIENT_PRIVATE

#include "core/or/or.h" #include "test/test.h" @@ -19,7 +19,8 @@ #include "app/config/config.h" #include "app/config/confparse.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/client/entrynodes.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 728bb4a2f..203a1d703 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -7,13 +7,15 @@ */

#define CONNECTION_PRIVATE -#define DIRECTORY_PRIVATE +#define DIRCACHE_PRIVATE +#define DIRCLIENT_PRIVATE #define HS_CACHE_PRIVATE

#include "trunnel/ed25519_cert.h" #include "feature/hs/hs_cache.h" #include "feature/rend/rendcache.h" -#include "feature/dircache/directory.h" +#include "feature/dircache/dircache.h" +#include "feature/dirclient/dirclient.h" #include "feature/nodelist/networkstatus.h" #include "core/mainloop/connection.h" #include "core/proto/proto_http.h" diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 31a2816c1..25cb991a7 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -25,7 +25,7 @@ #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_dh.h" #include "core/or/channeltls.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerset.h" diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index c60d6e264..95f7ed14b 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -24,7 +24,7 @@ #include "feature/hs/hs_service.h" #include "app/config/config.h" #include "feature/nodelist/networkstatus.h" -#include "feature/dircache/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/dirauth/dirvote.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" diff --git a/src/test/test_oos.c b/src/test/test_oos.c index b8ff3030a..fb0daa7a8 100644 --- a/src/test/test_oos.c +++ b/src/test/test_oos.c @@ -9,7 +9,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" #include "core/mainloop/mainloop.h" #include "test/test.h"

diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index f629596c5..1086b9861 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -6,7 +6,7 @@ #include <time.h>

#define CONNECTION_PRIVATE -#define DIRECTORY_PRIVATE +#define DIRCLIENT_PRIVATE #define DIRVOTE_PRIVATE #define ENTRYNODES_PRIVATE #define HIBERNATE_PRIVATE @@ -19,7 +19,8 @@ #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/dircache/directory.h" +#include "feature/dircommon/directory.h" +#include "feature/dirclient/dirclient.h" #include "feature/dirauth/dirvote.h" #include "feature/client/entrynodes.h" #include "feature/hibernate/hibernate.h"