commit 34d9d02150fc41107887c27900a62c4552578789 Author: Andrea Shepard <andrea@torproject.org> Date: Thu Jun 30 10:43:44 2016 +0000 Stub out connection_handle_oos() and call it from places we can change the socket count or thresholds --- src/or/config.c | 3 ++ src/or/connection.c | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/or/connection.h | 2 + 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/src/or/config.c b/src/or/config.c index 33c80e5..f7a7a53 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1358,6 +1358,9 @@ options_act_reversible(const or_options_t *old_options, char **msg) options->ConnLimit, options->ConnLimit_, options->ConnLimit_high_thresh, options->ConnLimit_low_thresh); + + /* Give the OOS handler a chance with the new thresholds */ + connection_handle_oos(get_n_open_sockets(), 0); } goto done; diff --git a/src/or/connection.c b/src/or/connection.c index 68e442d..3939531 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1090,6 +1090,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, int start_reading = 0; static int global_next_session_group = SESSION_GROUP_FIRST_AUTO; tor_addr_t addr; + int exhaustion = 0; if (listensockaddr->sa_family == AF_INET || listensockaddr->sa_family == AF_INET6) { @@ -1108,6 +1109,11 @@ connection_listener_new(const struct sockaddr *listensockaddr, int e = tor_socket_errno(s); if (ERRNO_IS_RESOURCE_LIMIT(e)) { warn_too_many_conns(); + /* + * We'll call the OOS handler at the error exit, so set the + * exhaustion flag for it. + */ + exhaustion = 1; } else { log_warn(LD_NET, "Socket creation failed: %s", tor_socket_strerror(e)); @@ -1226,6 +1232,11 @@ connection_listener_new(const struct sockaddr *listensockaddr, int e = tor_socket_errno(s); if (ERRNO_IS_RESOURCE_LIMIT(e)) { warn_too_many_conns(); + /* + * We'll call the OOS handler at the error exit, so set the + * exhaustion flag for it. + */ + exhaustion = 1; } else { log_warn(LD_NET,"Socket creation failed: %s.", strerror(e)); } @@ -1344,6 +1355,12 @@ connection_listener_new(const struct sockaddr *listensockaddr, dnsserv_configure_listener(conn); } + /* + * Normal exit; call the OOS handler since connection count just changed; + * the exhaustion flag will always be zero here though. + */ + connection_handle_oos(get_n_open_sockets(), 0); + return conn; err: @@ -1352,6 +1369,9 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (conn) connection_free(conn); + /* Call the OOS handler, indicate if we saw an exhaustion-related error */ + connection_handle_oos(get_n_open_sockets(), exhaustion); + return NULL; } @@ -1442,21 +1462,34 @@ connection_handle_listener_read(connection_t *conn, int new_type) if (!SOCKET_OK(news)) { /* accept() error */ int e = tor_socket_errno(conn->s); if (ERRNO_IS_ACCEPT_EAGAIN(e)) { - return 0; /* they hung up before we could accept(). that's fine. */ + /* + * they hung up before we could accept(). that's fine. + * + * give the OOS handler a chance to run though + */ + connection_handle_oos(get_n_open_sockets(), 0); + return 0; } else if (ERRNO_IS_RESOURCE_LIMIT(e)) { warn_too_many_conns(); + /* Exhaustion; tell the OOS handler */ + connection_handle_oos(get_n_open_sockets(), 1); return 0; } /* else there was a real error. */ log_warn(LD_NET,"accept() failed: %s. Closing listener.", tor_socket_strerror(e)); connection_mark_for_close(conn); + /* Tell the OOS handler about this too */ + connection_handle_oos(get_n_open_sockets(), 0); return -1; } log_debug(LD_NET, "Connection accepted on socket %d (child of fd %d).", (int)news,(int)conn->s); + /* We accepted a new conn; run OOS handler */ + connection_handle_oos(get_n_open_sockets(), 0); + if (make_socket_reuseable(news) < 0) { if (tor_socket_errno(news) == EINVAL) { /* This can happen on OSX if we get a badly timed shutdown. */ @@ -1661,12 +1694,18 @@ connection_connect_sockaddr,(connection_t *conn, s = tor_open_socket_nonblocking(protocol_family, SOCK_STREAM, proto); if (! SOCKET_OK(s)) { + /* + * Early OOS handler calls; it matters if it's an exhaustion-related + * error or not. + */ *socket_error = tor_socket_errno(s); if (ERRNO_IS_RESOURCE_LIMIT(*socket_error)) { warn_too_many_conns(); + connection_handle_oos(get_n_open_sockets(), 1); } else { log_warn(LD_NET,"Error creating network socket: %s", tor_socket_strerror(*socket_error)); + connection_handle_oos(get_n_open_sockets(), 0); } return -1; } @@ -1676,6 +1715,13 @@ connection_connect_sockaddr,(connection_t *conn, tor_socket_strerror(errno)); } + /* + * We've got the socket open; give the OOS handler a chance to check + * against configuured maximum socket number, but tell it no exhaustion + * failure. + */ + connection_handle_oos(get_n_open_sockets(), 0); + if (bindaddr && bind(s, bindaddr, bindaddr_len) < 0) { *socket_error = tor_socket_errno(s); log_warn(LD_NET,"Error binding network socket: %s", @@ -4454,6 +4500,66 @@ connection_reached_eof(connection_t *conn) } } +/** Out-of-Sockets handler; n_socks is the current number of open + * sockets, and failed is non-zero if a socket exhaustion related + * error immediately preceded this call. This is where to do + * circuit-killing heuristics as needed. + */ +void +connection_handle_oos(int n_socks, int failed) +{ + int target_n_socks = 0; + + /* Sanity-check args */ + tor_assert(n_socks >= 0); + + /* + * Make some log noise; keep it at debug level since this gets a chance + * to run on every connection attempt. + */ + log_debug(LD_NET, + "Running the OOS handler (%d open sockets, %s)", + n_socks, (failed != 0) ? "exhaustion seen" : "no exhaustion"); + + /* + * Check if we're really handling an OOS condition, and if so decide how + * many sockets we want to get down to. + */ + if (n_socks > get_options()->ConnLimit_high_thresh) { + /* Try to get down to the low threshold */ + target_n_socks = get_options()->ConnLimit_low_thresh; + log_notice(LD_NET, + "Current number of sockets %d is greater than configured " + "limit %d; OOS handler trying to get down to %d", + n_socks, get_options()->ConnLimit_high_thresh, + target_n_socks); + } else if (failed) { + /* + * If we're not at the limit but we hit a socket exhaustion error, try to + * drop some (but not as aggressively as ConnLimit_low_threshold, which is + * 3/4 of ConnLimit_) + */ + target_n_socks = (n_socks * 9) / 10; + log_notice(LD_NET, + "We saw socket exhaustion at %d open sockets; OOS handler " + "trying to get down to %d", + n_socks, target_n_socks); + } + + if (target_n_socks > 0) { + /* + * It's an OOS! + * + * TODO count moribund sockets; it'll be important that anything we decide + * to get rid of here but don't immediately close get counted as moribund + * on subsequent invocations so we don't try to kill too many things if + * this gets called multiple times. + */ + + /* TODO pick what to try to close */ + } +} + /** Log how many bytes are used by buffers of different kinds and sizes. */ void connection_dump_buffer_mem_stats(int severity) diff --git a/src/or/connection.h b/src/or/connection.h index f8e0f73..4e8a740 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -247,6 +247,8 @@ void clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted, log_domain_mask_t domain, const char *received, const char *source); +void connection_handle_oos(int n_socks, int failed); + #ifdef CONNECTION_PRIVATE STATIC void connection_free_(connection_t *conn);