March 2011 - tor-commits - lists.torproject.org

[tor/master] Handle negative run lengths in wfu/mtbf calculations
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 5a9903b9e09697ac131c841310dd82eebcca02e0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Nov 22 12:39:22 2010 -0500 Handle negative run lengths in wfu/mtbf calculations --- changes/bug1035 | 5 +++++ src/or/rephist.c | 22 +++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/changes/bug1035 b/changes/bug1035 index 4b62b46..041e3b3 100644 --- a/changes/bug1035 +++ b/changes/bug1035 @@ -4,4 +4,9 @@ treat it as having any downtime for the purposes of stability calculation, whereas clients would experience downtime since the IP could take a while to propagate to them. Resolves issue 1035. + o Minor bugfixes (authorities) + - Try to be more robust to hops back in time when calculating + router stability. Previously, if a run of uptime or downtime + appeared to be negative, the calculation could give incorrect + results. Bugfix on 0.2.0.6-alpha. diff --git a/src/or/rephist.c b/src/or/rephist.c index a0c9c9f..e59fcb5 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -380,12 +380,20 @@ rep_hist_note_router_unreachable(const char *id, time_t when) long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; + if (run_length < 0) { + unsigned long penalty = -run_length; +#define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@ -458,7 +466,7 @@ rep_hist_downrate_old_runs(time_t now) static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@ -494,8 +502,8 @@ get_total_weighted_time(or_history_t *hist, time_t when) static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run);

1 0

[tor/master] Merge remote branch 'origin/maint-0.2.2'
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit f9bb3ced51422bc8f6c3702a169a799614bbce02 Merge: d5907e7 0d78a16 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 8 16:10:40 2011 -0500 Merge remote branch 'origin/maint-0.2.2' Resolved trivial one-line conflicts. Conflicts: src/or/dirserv.c src/or/rephist.c changes/bug1035 | 13 +++++++++ doc/HACKING | 13 ++++----- src/or/connection_or.c | 3 -- src/or/dirserv.c | 19 +++++++++---- src/or/rephist.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/or/rephist.h | 3 +- 6 files changed, 91 insertions(+), 25 deletions(-) diff --combined src/or/connection_or.c index 5e28f94,2d24444..01cc160 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@@ -22,17 -22,12 +22,17 @@@ #include "geoip.h" #include "main.h" #include "networkstatus.h" +#include "nodelist.h" #include "reasons.h" #include "relay.h" #include "rephist.h" #include "router.h" #include "routerlist.h" +#ifdef USE_BUFFEREVENTS +#include <event2/bufferevent_ssl.h> +#endif + static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); static int connection_or_send_versions(or_connection_t *conn); @@@ -42,14 -37,6 +42,14 @@@ static int connection_or_check_valid_tl int started_here, char *digest_rcvd_out); +static void connection_or_tls_renegotiated_cb(tor_tls_t *tls, void *_conn); + +#ifdef USE_BUFFEREVENTS +static void connection_or_handle_event_cb(struct bufferevent *bufev, + short event, void *arg); +#include <event2/buffer.h>/*XXXX REMOVE */ +#endif + /**************************************************************/ /** Map from identity digest of connected OR or desired OR to a connection_t @@@ -191,8 -178,7 +191,8 @@@ var_cell_pack_header(const var_cell_t * var_cell_t * var_cell_new(uint16_t payload_len) { - var_cell_t *cell = tor_malloc(sizeof(var_cell_t)+payload_len-1); + size_t size = STRUCT_OFFSET(var_cell_t, payload) + payload_len; + var_cell_t *cell = tor_malloc(size); cell->payload_len = payload_len; cell->command = 0; cell->circ_id = 0; @@@ -241,14 -227,6 +241,14 @@@ connection_or_process_inbuf(or_connecti } return ret; +#ifdef USE_BUFFEREVENTS + case OR_CONN_STATE_TLS_SERVER_RENEGOTIATING: + if (tor_tls_server_got_renegotiate(conn->tls)) + connection_or_tls_renegotiated_cb(conn->tls, conn); + if (conn->_base.marked_for_close) + return 0; + /* fall through. */ +#endif case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@@ -270,7 -248,7 +270,7 @@@ int connection_or_flushed_some(or_connection_t *conn) { - size_t datalen = buf_datalen(conn->_base.outbuf); + size_t datalen = connection_get_outbuf_len(TO_CONN(conn)); /* If we're under the low water mark, add cells until we're just over the * high water mark. */ if (datalen < OR_CONN_LOWWATER) { @@@ -303,6 -281,7 +303,6 @@@ connection_or_finished_flushing(or_conn case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: - connection_stop_writing(TO_CONN(conn)); break; default: log_err(LD_BUG,"Called in unexpected state %d.", conn->_base.state); @@@ -363,7 -342,7 +363,7 @@@ connection_or_digest_is_known_relay(con { if (router_get_consensus_status_by_id(id_digest)) return 1; /* It's in the consensus: "yes" */ - if (router_get_by_digest(id_digest)) + if (router_get_by_id_digest(id_digest)) return 1; /* Not in the consensus, but we have a descriptor for * it. Probably it was in a recent consensus. "Yes". */ return 0; @@@ -400,22 -379,6 +400,22 @@@ connection_or_update_token_buckets_help conn->bandwidthrate = rate; conn->bandwidthburst = burst; +#ifdef USE_BUFFEREVENTS + { + const struct timeval *tick = tor_libevent_get_one_tick_timeout(); + struct ev_token_bucket_cfg *cfg, *old_cfg; + int rate_per_tick = rate / TOR_LIBEVENT_TICKS_PER_SECOND; + cfg = ev_token_bucket_cfg_new(rate_per_tick, burst, rate_per_tick, + burst, tick); + old_cfg = conn->bucket_cfg; + if (conn->_base.bufev) + bufferevent_set_rate_limit(conn->_base.bufev, cfg); + if (old_cfg) + ev_token_bucket_cfg_free(old_cfg); + conn->bucket_cfg = cfg; + (void) reset; /* No way to do this with libevent yet. */ + } +#else if (reset) { /* set up the token buckets to be full */ conn->read_bucket = conn->write_bucket = burst; return; @@@ -426,7 -389,6 +426,7 @@@ conn->read_bucket = burst; if (conn->write_bucket > burst) conn->write_bucket = burst; +#endif } /** Either our set of relays or our per-conn rate limits have changed. @@@ -450,7 -412,7 +450,7 @@@ connection_or_init_conn_from_address(or const char *id_digest, int started_here) { - routerinfo_t *r = router_get_by_digest(id_digest); + const node_t *r = node_get_by_id(id_digest); connection_or_set_identity_digest(conn, id_digest); connection_or_update_token_buckets_helper(conn, 1, get_options()); @@@ -458,10 -420,8 +458,10 @@@ tor_addr_copy(&conn->_base.addr, addr); tor_addr_copy(&conn->real_addr, addr); if (r) { + tor_addr_t node_addr; + node_get_addr(r, &node_addr); /* XXXX proposal 118 will make this more complex. */ - if (tor_addr_eq_ipv4h(&conn->_base.addr, r->addr)) + if (tor_addr_eq(&conn->_base.addr, &node_addr)) conn->is_canonical = 1; if (!started_here) { /* Override the addr/port, so our log messages will make sense. @@@ -474,12 -434,12 +474,12 @@@ * right IP address and port 56244, that wouldn't be as helpful. now we * log the "right" port too, so we know if it's moria1 or moria2. */ - tor_addr_from_ipv4h(&conn->_base.addr, r->addr); - conn->_base.port = r->or_port; + tor_addr_copy(&conn->_base.addr, &node_addr); + conn->_base.port = node_get_orport(r); } - conn->nickname = tor_strdup(r->nickname); + conn->nickname = tor_strdup(node_get_nickname(r)); tor_free(conn->_base.address); - conn->_base.address = tor_strdup(r->address); + conn->_base.address = tor_dup_addr(&node_addr); } else { const char *n; /* If we're an authoritative directory server, we may know a @@@ -899,7 -859,6 +899,7 @@@ in connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; + tor_assert(!conn->tls); conn->tls = tor_tls_new(conn->_base.s, receiving); tor_tls_set_logged_address(conn->tls, // XXX client and relay? escaped_safe_str(conn->_base.address)); @@@ -907,38 -866,12 +907,38 @@@ log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } +#ifdef USE_BUFFEREVENTS + if (connection_type_uses_bufferevent(TO_CONN(conn))) { + const int filtering = get_options()->_UseFilteringSSLBufferevents; + struct bufferevent *b = + tor_tls_init_bufferevent(conn->tls, conn->_base.bufev, conn->_base.s, + receiving, filtering); + if (!b) { + log_warn(LD_BUG,"tor_tls_init_bufferevent failed. Closing."); + return -1; + } + conn->_base.bufev = b; + if (conn->bucket_cfg) + bufferevent_set_rate_limit(conn->_base.bufev, conn->bucket_cfg); + connection_enable_rate_limiting(TO_CONN(conn)); + + connection_configure_bufferevent_callbacks(TO_CONN(conn)); + bufferevent_setcb(b, + connection_handle_read_cb, + connection_handle_write_cb, + connection_or_handle_event_cb,/* overriding this one*/ + TO_CONN(conn)); + } +#endif connection_start_reading(TO_CONN(conn)); log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); - if (connection_tls_continue_handshake(conn) < 0) { - return -1; + IF_HAS_BUFFEREVENT(TO_CONN(conn), { + /* ???? */; + }) ELSE_IF_NO_BUFFEREVENT { + if (connection_tls_continue_handshake(conn) < 0) + return -1; } return 0; } @@@ -1023,78 -956,6 +1023,78 @@@ connection_tls_continue_handshake(or_co return 0; } +#ifdef USE_BUFFEREVENTS +static void +connection_or_handle_event_cb(struct bufferevent *bufev, short event, + void *arg) +{ + struct or_connection_t *conn = TO_OR_CONN(arg); + + /* XXXX cut-and-paste code; should become a function. */ + if (event & BEV_EVENT_CONNECTED) { + if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) { + if (tor_tls_finish_handshake(conn->tls) < 0) { + log_warn(LD_OR, "Problem finishing handshake"); + connection_mark_for_close(TO_CONN(conn)); + return; + } + } + + if (! tor_tls_used_v1_handshake(conn->tls)) { + if (!tor_tls_is_server(conn->tls)) { + if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) { + conn->_base.state = OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING; + tor_tls_unblock_renegotiation(conn->tls); + if (bufferevent_ssl_renegotiate(conn->_base.bufev)<0) { + log_warn(LD_OR, "Start_renegotiating went badly."); + connection_mark_for_close(TO_CONN(conn)); + } + tor_tls_unblock_renegotiation(conn->tls); + return; /* ???? */ + } + } else if (tor_tls_get_num_server_handshakes(conn->tls) == 1) { + /* improved handshake, as a server. Only got one handshake, so + * wait for the next one. */ + tor_tls_set_renegotiate_callback(conn->tls, + connection_or_tls_renegotiated_cb, + conn); + conn->_base.state = OR_CONN_STATE_TLS_SERVER_RENEGOTIATING; + /* return 0; */ + return; /* ???? */ + } else { + const int handshakes = tor_tls_get_num_server_handshakes(conn->tls); + tor_assert(handshakes >= 2); + if (handshakes == 2) { + /* improved handshake, as a server. Two handshakes happened already, + * so we treat renegotiation as done. + */ + connection_or_tls_renegotiated_cb(conn->tls, conn); + } else { + log_warn(LD_OR, "More than two handshakes done on connection. " + "Closing."); + connection_mark_for_close(TO_CONN(conn)); + } + return; + } + } + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); + if (connection_tls_finish_handshake(conn) < 0) + connection_mark_for_close(TO_CONN(conn)); /* ???? */ + return; + } + + if (event & BEV_EVENT_ERROR) { + unsigned long err; + while ((err = bufferevent_get_openssl_error(bufev))) { + tor_tls_log_one_error(conn->tls, err, LOG_WARN, LD_OR, + "handshaking (with bufferevent)"); + } + } + + connection_handle_event_cb(bufev, event, arg); +} +#endif + /** Return 1 if we initiated this connection, or 0 if it started * out as an incoming connection. */ @@@ -1237,9 -1098,6 +1237,6 @@@ connection_or_check_valid_tls_handshake as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@@ -1266,9 -1124,7 +1263,9 @@@ connection_tls_finish_handshake(or_conn char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + log_debug(LD_HANDSHAKE,"%s tls handshake on %p with %s done. verifying.", + started_here?"outgoing":"incoming", + conn, safe_str_client(conn->_base.address)); directory_set_dirty(); @@@ -1349,7 -1205,7 +1346,7 @@@ connection_or_set_state_open(or_connect router_set_status(conn->identity_digest, 1); } else { /* only report it to the geoip module if it's not a known router */ - if (!router_get_by_digest(conn->identity_digest)) { + if (!router_get_by_id_digest(conn->identity_digest)) { if (tor_addr_family(&TO_CONN(conn)->addr) == AF_INET) { /*XXXX IP6 support ipv6 geoip.*/ uint32_t a = tor_addr_to_ipv4h(&TO_CONN(conn)->addr); @@@ -1360,12 -1216,8 +1357,12 @@@ or_handshake_state_free(conn->handshake_state); conn->handshake_state = NULL; + IF_HAS_BUFFEREVENT(TO_CONN(conn), { + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); + }) ELSE_IF_NO_BUFFEREVENT { + connection_start_reading(TO_CONN(conn)); + } - connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ return 0; @@@ -1410,18 -1262,12 +1407,18 @@@ connection_or_write_var_cell_to_buf(con conn->timestamp_last_added_nonpadding = approx_time(); } -/** See whether there's a variable-length cell waiting on conn's +/** See whether there's a variable-length cell waiting on or_conn's * inbuf. Return values as for fetch_var_cell_from_buf(). */ static int -connection_fetch_var_cell_from_buf(or_connection_t *conn, var_cell_t **out) +connection_fetch_var_cell_from_buf(or_connection_t *or_conn, var_cell_t **out) { - return fetch_var_cell_from_buf(conn->_base.inbuf, out, conn->link_proto); + connection_t *conn = TO_CONN(or_conn); + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_var_cell_from_evbuffer(input, out, or_conn->link_proto); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_var_cell_from_buf(conn->inbuf, out, or_conn->link_proto); + } } /** Process cells from conn's inbuf. @@@ -1439,7 -1285,7 +1436,7 @@@ connection_or_process_cells_from_inbuf( while (1) { log_debug(LD_OR, "%d: starting, inbuf_datalen %d (%d pending in tls object).", - conn->_base.s,(int)buf_datalen(conn->_base.inbuf), + conn->_base.s,(int)connection_get_inbuf_len(TO_CONN(conn)), tor_tls_get_pending_bytes(conn->tls)); if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) @@@ -1450,8 -1296,8 +1447,8 @@@ } else { char buf[CELL_NETWORK_SIZE]; cell_t cell; - if (buf_datalen(conn->_base.inbuf) < CELL_NETWORK_SIZE) /* whole response - available? */ + if (connection_get_inbuf_len(TO_CONN(conn)) + < CELL_NETWORK_SIZE) /* whole response available? */ return 0; /* not yet */ circuit_build_times_network_is_live(&circ_times); @@@ -1538,7 -1384,7 +1535,7 @@@ connection_or_send_netinfo(or_connectio { cell_t cell; time_t now = time(NULL); - routerinfo_t *me; + const routerinfo_t *me; int len; uint8_t *out; diff --combined src/or/dirserv.c index bf61059,aeeab45..4f06de1 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@@ -16,7 -16,6 +16,7 @@@ #include "hibernate.h" #include "microdesc.h" #include "networkstatus.h" +#include "nodelist.h" #include "policies.h" #include "rephist.h" #include "router.h" @@@ -66,6 -65,8 +66,6 @@@ static char *format_versions_list(confi struct authdir_config_t; static int add_fingerprint_to_dir(const char *nickname, const char *fp, struct authdir_config_t *list); -static uint32_t dirserv_router_get_status(const routerinfo_t *router, - const char **msg); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, const char *address, @@@ -73,8 -74,7 +73,8 @@@ const char *platform, const char *contact, const char **msg, int should_log); static void clear_cached_dir(cached_dir_t *d); -static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, +static const signed_descriptor_t *get_signed_descriptor_by_fp( + const char *fp, int extrainfo, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); @@@ -303,7 -303,7 +303,7 @@@ dirserv_load_fingerprint_file(void * * If the status is 'FP_REJECT' and msg is provided, set * *msg to an explanation of why. */ -static uint32_t +uint32_t dirserv_router_get_status(const routerinfo_t *router, const char **msg) { char d[DIGEST_LEN]; @@@ -325,7 -325,7 +325,7 @@@ /** Return true if there is no point in downloading the router described by * rs because this directory would reject it. */ int -dirserv_would_reject_router(routerstatus_t *rs) +dirserv_would_reject_router(const routerstatus_t *rs) { uint32_t res; @@@ -360,7 -360,7 +360,7 @@@ dirserv_get_name_status(const char *id_ return 0; } -/** Helper: As dirserv_get_router_status, but takes the router fingerprint +/** Helper: As dirserv_router_get_status, but takes the router fingerprint * (hex, no spaces), nickname, address (used for logging only), IP address, OR * port, platform (logging only) and contact info (logging only) as arguments. * @@@ -375,7 -375,7 +375,7 @@@ dirserv_get_status_impl(const char *id_ const char **msg, int should_log) { int reject_unlisted = get_options()->AuthDirRejectUnlisted; - uint32_t result = 0; + uint32_t result; router_status_t *status_by_digest; if (!fingerprint_list) @@@ -539,7 -539,7 +539,7 @@@ dirserv_router_has_valid_address(router */ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, - int complain) + int complain, int *valid_out) { /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); @@@ -580,24 -580,15 +580,24 @@@ *msg = "Rejected: Address is not an IP, or IP is a private address."; return -1; } - /* Okay, looks like we're willing to accept this one. */ - ri->is_named = (status & FP_NAMED) ? 1 : 0; - ri->is_valid = (status & FP_INVALID) ? 0 : 1; - ri->is_bad_directory = (status & FP_BADDIR) ? 1 : 0; - ri->is_bad_exit = (status & FP_BADEXIT) ? 1 : 0; + + *valid_out = ! (status & FP_INVALID); return 0; } +/** Update the relevant flags of node based on our opinion as a + * directory authority in authstatus, as returned by + * dirserv_router_get_status or equivalent. */ +void +dirserv_set_node_flags_from_authoritative_status(node_t *node, + uint32_t authstatus) +{ + node->is_valid = (authstatus & FP_INVALID) ? 0 : 1; + node->is_bad_directory = (authstatus & FP_BADDIR) ? 1 : 0; + node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0; +} + /** True iff a is more severe than b. */ static int WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b) @@@ -722,7 -713,7 +722,7 @@@ dirserv_add_descriptor(routerinfo_t *ri * from this server. (We do this here and not in router_add_to_routerlist * because we want to be able to accept the newest router descriptor that * another authority has, so we all converge on the same one.) */ - ri_old = router_get_by_digest(ri->cache_info.identity_digest); + ri_old = router_get_mutable_by_digest(ri->cache_info.identity_digest); if (ri_old && ri_old->cache_info.published_on < ri->cache_info.published_on && router_differences_are_cosmetic(ri_old, ri) && !router_is_me(ri)) { @@@ -766,7 -757,8 +766,7 @@@ routerlist_descriptors_added(changed, 0); smartlist_free(changed); if (!*msg) { - *msg = ri->is_valid ? "Descriptor for valid server accepted" : - "Descriptor for invalid server accepted"; + *msg = "Descriptor accepted"; } log_info(LD_DIRSERV, "Added descriptor from '%s' (source: %s): %s.", @@@ -781,12 -773,12 +781,12 @@@ static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) { - routerinfo_t *ri; + const routerinfo_t *ri; int r; tor_assert(msg); *msg = NULL; - ri = router_get_by_digest(ei->cache_info.identity_digest); + ri = router_get_by_id_digest(ei->cache_info.identity_digest); if (!ri) { *msg = "No corresponding router descriptor for extra-info descriptor"; extrainfo_free(ei); @@@ -821,65 -813,54 +821,65 @@@ static void directory_remove_invalid(void) { - int i; int changed = 0; routerlist_t *rl = router_get_routerlist(); + smartlist_t *nodes = smartlist_create(); + smartlist_add_all(nodes, nodelist_get_list()); - routerlist_assert_ok(rl); - - for (i = 0; i < smartlist_len(rl->routers); ++i) { + SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) { const char *msg; - routerinfo_t *ent = smartlist_get(rl->routers, i); - uint32_t r = dirserv_router_get_status(ent, &msg); + routerinfo_t *ent = node->ri; + uint32_t r; + if (!ent) + continue; + r = dirserv_router_get_status(ent, &msg); if (r & FP_REJECT) { log_info(LD_DIRSERV, "Router '%s' is now rejected: %s", ent->nickname, msg?msg:""); routerlist_remove(rl, ent, 0, time(NULL)); changed = 1; continue; } - if (bool_neq((r & FP_NAMED), ent->is_named)) { +#if 0 + if (bool_neq((r & FP_NAMED), ent->auth_says_is_named)) { log_info(LD_DIRSERV, "Router '%s' is now %snamed.", ent->nickname, (r&FP_NAMED)?"":"un"); ent->is_named = (r&FP_NAMED)?1:0; changed = 1; } - if (bool_neq((r & FP_INVALID), !ent->is_valid)) { + if (bool_neq((r & FP_UNNAMED), ent->auth_says_is_unnamed)) { + log_info(LD_DIRSERV, + "Router '%s' is now %snamed. (FP_UNNAMED)", ent->nickname, + (r&FP_NAMED)?"":"un"); + ent->is_named = (r&FP_NUNAMED)?0:1; + changed = 1; + } +#endif + if (bool_neq((r & FP_INVALID), !node->is_valid)) { log_info(LD_DIRSERV, "Router '%s' is now %svalid.", ent->nickname, (r&FP_INVALID) ? "in" : ""); - ent->is_valid = (r&FP_INVALID)?0:1; + node->is_valid = (r&FP_INVALID)?0:1; changed = 1; } - if (bool_neq((r & FP_BADDIR), ent->is_bad_directory)) { + if (bool_neq((r & FP_BADDIR), node->is_bad_directory)) { log_info(LD_DIRSERV, "Router '%s' is now a %s directory", ent->nickname, (r & FP_BADDIR) ? "bad" : "good"); - ent->is_bad_directory = (r&FP_BADDIR) ? 1: 0; + node->is_bad_directory = (r&FP_BADDIR) ? 1: 0; changed = 1; } - if (bool_neq((r & FP_BADEXIT), ent->is_bad_exit)) { + if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) { log_info(LD_DIRSERV, "Router '%s' is now a %s exit", ent->nickname, (r & FP_BADEXIT) ? "bad" : "good"); - ent->is_bad_exit = (r&FP_BADEXIT) ? 1: 0; + node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0; changed = 1; } - } + } SMARTLIST_FOREACH_END(node); if (changed) directory_set_dirty(); routerlist_assert_ok(rl); + smartlist_free(nodes); } /** Mark the directory as dirty -- when we're next asked for a @@@ -918,11 -899,10 +918,11 @@@ directory_set_dirty(void * as running iff is_live is true. */ static char * -list_single_server_status(routerinfo_t *desc, int is_live) +list_single_server_status(const routerinfo_t *desc, int is_live) { char buf[MAX_NICKNAME_LEN+HEX_DIGEST_LEN+4]; /* !nickname=$hexdigest\0 */ char *cp; + const node_t *node; tor_assert(desc); @@@ -930,8 -910,7 +930,8 @@@ if (!is_live) { *cp++ = '!'; } - if (desc->is_valid) { + node = node_get_by_id(desc->cache_info.identity_digest); + if (node && node->is_valid) { strlcpy(cp, desc->nickname, sizeof(buf)-(cp-buf)); cp += strlen(cp); *cp++ = '='; @@@ -970,8 -949,6 +970,8 @@@ dirserv_set_router_is_running(routerinf unreachable. */ int answer; + node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest); + tor_assert(node); if (router_is_me(router)) { /* We always know if we are down ourselves. */ @@@ -996,7 -973,7 +996,7 @@@ rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); } - router->is_running = answer; + node->is_running = answer; } /** Based on the routerinfo_ts in routers, allocate the @@@ -1024,8 -1001,6 +1024,8 @@@ list_server_status_v1(smartlist_t *rout rs_entries = smartlist_create(); SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + const node_t *node = node_get_by_id(ri->cache_info.identity_digest); + tor_assert(node); if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); @@@ -1033,13 -1008,12 +1033,13 @@@ if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; - if (!ri->is_running) + if (!node->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); } else if (ri->cache_info.published_on >= cutoff) { - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); + smartlist_add(rs_entries, list_single_server_status(ri, + node->is_running)); } } SMARTLIST_FOREACH_END(ri); @@@ -1077,12 -1051,12 +1077,12 @@@ format_versions_list(config_line_t *ln * not hibernating, and not too old. Else return 0. */ static int -router_is_active(routerinfo_t *ri, time_t now) +router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) { time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; if (ri->cache_info.published_on < cutoff) return 0; - if (!ri->is_running || !ri->is_valid || ri->is_hibernating) + if (!node->is_running || !node->is_valid || ri->is_hibernating) return 0; return 1; } @@@ -1183,7 -1157,7 +1183,7 @@@ dirserv_dump_directory_to_string(char * int directory_fetches_from_authorities(or_options_t *options) { - routerinfo_t *me; + const routerinfo_t *me; uint32_t addr; int refuseunknown; if (options->FetchDirInfoEarly) @@@ -1295,8 -1269,7 +1295,8 @@@ static cached_dir_t cached_runningroute * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** Map from flavor name to the v3 consensuses that we're currently serving. */ +/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're + * currently serving. */ static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of d with the value of @@@ -1749,7 -1722,7 +1749,7 @@@ static uint64_t total_exit_bandwidth = /** Helper: estimate the uptime of a router given its stated uptime and the * amount of time since it last stated its stated uptime. */ static INLINE long -real_uptime(routerinfo_t *router, time_t now) +real_uptime(const routerinfo_t *router, time_t now) { if (now < router->cache_info.published_on) return router->uptime; @@@ -1800,8 -1773,7 +1800,8 @@@ dirserv_thinks_router_is_unreliable(tim * been set. */ static int -dirserv_thinks_router_is_hs_dir(routerinfo_t *router, time_t now) +dirserv_thinks_router_is_hs_dir(const routerinfo_t *router, + const node_t *node, time_t now) { long uptime = real_uptime(router, now); @@@ -1811,7 -1783,7 +1811,7 @@@ * version is too old. */ return (router->wants_to_be_hs_dir && router->dir_port && uptime > get_options()->MinUptimeHidServDirectoryV2 && - router->is_running); + node->is_running); } /** Look through the routerlist, the Mean Time Between Failure history, and @@@ -1859,22 -1831,19 +1859,22 @@@ dirserv_compute_performance_thresholds( /* Weighted fractional uptime for each active router. */ wfus = tor_malloc(sizeof(double)*smartlist_len(rl->routers)); + nodelist_assert_ok(); + /* Now, fill in the arrays. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - if (router_is_active(ri, now)) { + SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + routerinfo_t *ri = node->ri; + if (ri && router_is_active(ri, node, now)) { const char *id = ri->cache_info.identity_digest; uint32_t bw; - ri->is_exit = (!router_exit_policy_rejects_all(ri) && - exit_policy_is_general_exit(ri->exit_policy)); + node->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); bandwidths[n_active] = bw = router_get_advertised_bandwidth(ri); total_bandwidth += bw; - if (ri->is_exit && !ri->is_bad_exit) { + if (node->is_exit && !node->is_bad_exit) { total_exit_bandwidth += bw; } else { bandwidths_excluding_exits[n_active_nonexit] = bw; @@@ -1882,7 -1851,7 +1882,7 @@@ } ++n_active; } - }); + } SMARTLIST_FOREACH_END(node); /* Now, compute thresholds. */ if (n_active) { @@@ -1908,17 -1877,15 +1908,17 @@@ /* Now that we have a time-known that 7/8 routers are known longer than, * fill wfus with the wfu of every such "familiar" router. */ n_familiar = 0; - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - if (router_is_active(ri, now)) { + + SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + routerinfo_t *ri = node->ri; + if (ri && router_is_active(ri, node, now)) { const char *id = ri->cache_info.identity_digest; long tk = rep_hist_get_weighted_time_known(id, now); if (tk < guard_tk) continue; wfus[n_familiar++] = rep_hist_get_weighted_fractional_uptime(id, now); } - }); + } SMARTLIST_FOREACH_END(node); if (n_familiar) guard_wfu = median_double(wfus, n_familiar); if (guard_wfu > WFU_TO_GUARANTEE_GUARD) @@@ -1989,20 -1956,24 +1989,20 @@@ version_from_platform(const char *platf */ int routerstatus_format_entry(char *buf, size_t buf_len, - routerstatus_t *rs, const char *version, + const routerstatus_t *rs, const char *version, routerstatus_format_type_t format) { int r; - struct in_addr in; char *cp; char *summary; char published[ISO_TIME_LEN+1]; - char ipaddr[INET_NTOA_BUF_LEN]; char identity64[BASE64_DIGEST_LEN+1]; char digest64[BASE64_DIGEST_LEN+1]; format_iso_time(published, rs->published_on); digest_to_base64(identity64, rs->identity_digest); digest_to_base64(digest64, rs->descriptor_digest); - in.s_addr = htonl(rs->addr); - tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, "r %s %s %s%s%s %s %d %d\n", @@@ -2011,7 -1982,7 +2011,7 @@@ (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, - ipaddr, + fmt_addr32(rs->addr), (int)rs->or_port, (int)rs->dir_port); if (r<0) { @@@ -2039,7 -2010,7 +2039,7 @@@ rs->is_possible_guard?" Guard":"", rs->is_hs_dir?" HSDir":"", rs->is_named?" Named":"", - rs->is_running?" Running":"", + rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", rs->is_unnamed?" Unnamed":"", rs->is_v2_dir?" V2Dir":"", @@@ -2061,7 -2032,7 +2061,7 @@@ } if (format != NS_V2) { - routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + const routerinfo_t* desc = router_get_by_id_digest(rs->identity_digest); uint32_t bw; if (format != NS_CONTROL_PORT) { @@@ -2157,8 -2128,6 +2157,8 @@@ _compare_routerinfo_by_ip_and_bw(const routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; int first_is_auth, second_is_auth; uint32_t bw_first, bw_second; + const node_t *node_first, *node_second; + int first_is_running, second_is_running; /* we return -1 if first should appear before second... that is, * if first is a better router. */ @@@ -2181,14 -2150,9 +2181,14 @@@ else if (!first_is_auth && second_is_auth) return 1; - else if (first->is_running && !second->is_running) + node_first = node_get_by_id(first->cache_info.identity_digest); + node_second = node_get_by_id(second->cache_info.identity_digest); + first_is_running = node_first && node_first->is_running; + second_is_running = node_second && node_second->is_running; + + if (first_is_running && !second_is_running) return -1; - else if (!first->is_running && second->is_running) + else if (!first_is_running && second_is_running) return 1; bw_first = router_get_advertised_bandwidth(first); @@@ -2257,9 -2221,7 +2257,9 @@@ get_possible_sybil_list(const smartlist */ void set_routerstatus_from_routerinfo(routerstatus_t *rs, - routerinfo_t *ri, time_t now, + node_t *node, + routerinfo_t *ri, + time_t now, int naming, int listbadexits, int listbaddirs) { @@@ -2271,46 -2233,48 +2271,46 @@@ router_digest_is_trusted_dir(ri->cache_info.identity_digest); /* Already set by compute_performance_thresholds. */ - rs->is_exit = ri->is_exit; - rs->is_stable = ri->is_stable = - router_is_active(ri, now) && + rs->is_exit = node->is_exit; + rs->is_stable = node->is_stable = + router_is_active(ri, node, now) && !dirserv_thinks_router_is_unreliable(now, ri, 1, 0) && !unstable_version; - rs->is_fast = ri->is_fast = - router_is_active(ri, now) && + rs->is_fast = node->is_fast = + router_is_active(ri, node, now) && !dirserv_thinks_router_is_unreliable(now, ri, 0, 1); - rs->is_running = ri->is_running; /* computed above */ + rs->is_flagged_running = node->is_running; /* computed above */ if (naming) { uint32_t name_status = dirserv_get_name_status( - ri->cache_info.identity_digest, ri->nickname); + node->identity, ri->nickname); rs->is_named = (naming && (name_status & FP_NAMED)) ? 1 : 0; rs->is_unnamed = (naming && (name_status & FP_UNNAMED)) ? 1 : 0; } - rs->is_valid = ri->is_valid; + rs->is_valid = node->is_valid; - if (rs->is_fast && + if (node->is_fast && (router_get_advertised_bandwidth(ri) >= BANDWIDTH_TO_GUARANTEE_GUARD || router_get_advertised_bandwidth(ri) >= MIN(guard_bandwidth_including_exits, guard_bandwidth_excluding_exits))) { - long tk = rep_hist_get_weighted_time_known( - ri->cache_info.identity_digest, now); - double wfu = rep_hist_get_weighted_fractional_uptime( - ri->cache_info.identity_digest, now); + long tk = rep_hist_get_weighted_time_known(node->identity, now); + double wfu = rep_hist_get_weighted_fractional_uptime(node->identity, now); rs->is_possible_guard = (wfu >= guard_wfu && tk >= guard_tk) ? 1 : 0; } else { rs->is_possible_guard = 0; } - rs->is_bad_directory = listbaddirs && ri->is_bad_directory; - rs->is_bad_exit = listbadexits && ri->is_bad_exit; - ri->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, now); - rs->is_hs_dir = ri->is_hs_dir; + rs->is_bad_directory = listbaddirs && node->is_bad_directory; + rs->is_bad_exit = listbadexits && node->is_bad_exit; + node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now); + rs->is_hs_dir = node->is_hs_dir; rs->is_v2_dir = ri->dir_port != 0; if (!strcasecmp(ri->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = rs->is_unnamed = 0; rs->published_on = ri->cache_info.published_on; - memcpy(rs->identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); + memcpy(rs->identity_digest, node->identity, DIGEST_LEN); memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest, DIGEST_LEN); rs->addr = ri->addr; @@@ -2327,7 -2291,7 +2327,7 @@@ static voi clear_status_flags_on_sybil(routerstatus_t *rs) { rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_named = rs->is_valid = rs->is_v2_dir = + rs->is_flagged_running = rs->is_named = rs->is_valid = rs->is_v2_dir = rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = rs->is_bad_directory = 0; /* FFFF we might want some mechanism to check later on if we @@@ -2335,6 -2299,18 +2335,6 @@@ * forget to add it to this clause. */ } -/** Clear all the status flags in routerinfo router. We put this - * function here because it's eerily similar to - * clear_status_flags_on_sybil() above. One day we should merge them. */ -void -router_clear_status_flags(routerinfo_t *router) -{ - router->is_valid = router->is_running = router->is_hs_dir = - router->is_fast = router->is_stable = - router->is_possible_guard = router->is_exit = - router->is_bad_exit = router->is_bad_directory = 0; -} - /** * Helper function to parse out a line in the measured bandwidth file * into a measured_bw_line_t output structure. Returns -1 on failure @@@ -2452,7 -2428,7 +2452,7 @@@ dirserv_read_measured_bandwidths(const smartlist_t *routerstatuses) { char line[256]; - FILE *fp = fopen(from_file, "r"); + FILE *fp = tor_fopen_cloexec(from_file, "r"); int applied_lines = 0; time_t file_time; int ok; @@@ -2582,20 -2558,17 +2582,20 @@@ dirserv_generate_networkstatus_vote_obj routerstatus_t *rs; vote_routerstatus_t *vrs; microdesc_t *md; + node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); + if (!node) + continue; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; - set_routerstatus_from_routerinfo(rs, ri, now, + set_routerstatus_from_routerinfo(rs, node, ri, now, naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(rs); if (!vote_on_reachability) - rs->is_running = 0; + rs->is_flagged_running = 0; vrs->version = version_from_platform(ri->platform); md = dirvote_create_microdescriptor(ri); @@@ -2726,8 -2699,10 +2726,8 @@@ generate_v2_networkstatus_opinion(void char *outp, *endp; or_options_t *options = get_options(); char fingerprint[FINGERPRINT_LEN+1]; - char ipaddr[INET_NTOA_BUF_LEN]; char published[ISO_TIME_LEN+1]; char digest[DIGEST_LEN]; - struct in_addr in; uint32_t addr; crypto_pk_env_t *private_key; routerlist_t *rl = router_get_routerlist(); @@@ -2748,6 -2723,8 +2748,6 @@@ log_warn(LD_NET, "Couldn't resolve my hostname"); goto done; } - in.s_addr = htonl(addr); - tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); format_iso_time(published, now); @@@ -2793,7 -2770,7 +2793,7 @@@ "dir-options%s%s%s%s\n" "%s" /* client version line, server version line. */ "dir-signing-key\n%s", - hostname, ipaddr, (int)options->DirPort, + hostname, fmt_addr32(addr), (int)options->DirPort, fingerprint, contact, published, @@@ -2824,12 -2801,8 +2824,12 @@@ if (ri->cache_info.published_on >= cutoff) { routerstatus_t rs; char *version = version_from_platform(ri->platform); - - set_routerstatus_from_routerinfo(&rs, ri, now, + node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); + if (!node) { + tor_free(version); + continue; + } + set_routerstatus_from_routerinfo(&rs, node, ri, now, naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) @@@ -2912,7 -2885,7 +2912,7 @@@ dirserv_get_networkstatus_v2_fingerprin if (!strcmp(key,"authority")) { if (authdir_mode_v2(get_options())) { - routerinfo_t *me = router_get_my_routerinfo(); + const routerinfo_t *me = router_get_my_routerinfo(); if (me) smartlist_add(result, tor_memdup(me->cache_info.identity_digest, DIGEST_LEN)); @@@ -3000,7 -2973,7 +3000,7 @@@ dirserv_get_routerdesc_fingerprints(sma /* Treat "all" requests as if they were unencrypted */ for_unencrypted_conn = 1; } else if (!strcmp(key, "authority")) { - routerinfo_t *ri = router_get_my_routerinfo(); + const routerinfo_t *ri = router_get_my_routerinfo(); if (ri) smartlist_add(fps_out, tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN)); @@@ -3020,8 -2993,8 +3020,8 @@@ if (for_unencrypted_conn) { /* Remove anything that insists it not be sent unencrypted. */ - SMARTLIST_FOREACH(fps_out, char *, cp, { - signed_descriptor_t *sd; + SMARTLIST_FOREACH_BEGIN(fps_out, char *, cp) { + const signed_descriptor_t *sd; if (by_id) sd = get_signed_descriptor_by_fp(cp,is_extrainfo,0); else if (is_extrainfo) @@@ -3032,7 -3005,7 +3032,7 @@@ tor_free(cp); SMARTLIST_DEL_CURRENT(fps_out, cp); } - }); + } SMARTLIST_FOREACH_END(cp); } if (!smartlist_len(fps_out)) { @@@ -3071,9 -3044,9 +3071,9 @@@ dirserv_get_routerdescs(smartlist_t *de SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, smartlist_add(descs_out, &(r->cache_info))); } else if (!strcmp(key, "/tor/server/authority")) { - routerinfo_t *ri = router_get_my_routerinfo(); + const routerinfo_t *ri = router_get_my_routerinfo(); if (ri) - smartlist_add(descs_out, &(ri->cache_info)); + smartlist_add(descs_out, (void*) &(ri->cache_info)); } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); @@@ -3097,17 -3070,17 +3097,17 @@@ { if (router_digest_is_me(d)) { /* make sure desc_routerinfo exists */ - routerinfo_t *ri = router_get_my_routerinfo(); + const routerinfo_t *ri = router_get_my_routerinfo(); if (ri) - smartlist_add(descs_out, &(ri->cache_info)); + smartlist_add(descs_out, (void*) &(ri->cache_info)); } else { - routerinfo_t *ri = router_get_by_digest(d); + const routerinfo_t *ri = router_get_by_id_digest(d); /* Don't actually serve a descriptor that everyone will think is * expired. This is an (ugly) workaround to keep buggy 0.1.1.10 * Tors from downloading descriptors that they will throw away. */ if (ri && ri->cache_info.published_on > cutoff) - smartlist_add(descs_out, &(ri->cache_info)); + smartlist_add(descs_out, (void*) &(ri->cache_info)); } }); SMARTLIST_FOREACH(digests, char *, d, tor_free(d)); @@@ -3143,20 -3116,27 +3143,27 @@@ dirserv_orconn_tls_done(const char *add tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); - + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ @@@ -3166,8 -3146,7 +3173,8 @@@ * an upload or a download. Used to decide whether to relaunch reachability * testing for the server. */ int -dirserv_should_launch_reachability_test(routerinfo_t *ri, routerinfo_t *ri_old) +dirserv_should_launch_reachability_test(const routerinfo_t *ri, + const routerinfo_t *ri_old) { if (!authdir_mode_handles_descs(get_options(), ri->purpose)) return 0; @@@ -3291,7 -3270,7 +3298,7 @@@ dirserv_remove_old_statuses(smartlist_ * its extra-info document if extrainfo is true. Return * NULL if not found or if the descriptor is older than * publish_cutoff. */ -static signed_descriptor_t * +static const signed_descriptor_t * get_signed_descriptor_by_fp(const char *fp, int extrainfo, time_t publish_cutoff) { @@@ -3301,7 -3280,7 +3308,7 @@@ else return &(router_get_my_routerinfo()->cache_info); } else { - routerinfo_t *ri = router_get_by_digest(fp); + const routerinfo_t *ri = router_get_by_id_digest(fp); if (ri && ri->cache_info.published_on > publish_cutoff) { if (extrainfo) @@@ -3369,7 -3348,7 +3376,7 @@@ dirserv_estimate_data_size(smartlist_t tor_assert(fps); if (is_serverdescs) { int n = smartlist_len(fps); - routerinfo_t *me = router_get_my_routerinfo(); + const routerinfo_t *me = router_get_my_routerinfo(); result = (me?me->cache_info.signed_descriptor_len:2048) * n; if (compressed) result /= 2; /* observed compressibility is between 35 and 55%. */ @@@ -3433,10 -3412,10 +3440,10 @@@ connection_dirserv_add_servers_to_outbu time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH; while (smartlist_len(conn->fingerprint_stack) && - buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { const char *body; char *fp = smartlist_pop_last(conn->fingerprint_stack); - signed_descriptor_t *sd = NULL; + const signed_descriptor_t *sd = NULL; if (by_fp) { sd = get_signed_descriptor_by_fp(fp, extra, publish_cutoff); } else { @@@ -3494,7 -3473,7 +3501,7 @@@ connection_dirserv_add_microdescs_to_ou { microdesc_cache_t *cache = get_microdesc_cache(); while (smartlist_len(conn->fingerprint_stack) && - buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { char *fp256 = smartlist_pop_last(conn->fingerprint_stack); microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); tor_free(fp256); @@@ -3533,7 -3512,7 +3540,7 @@@ connection_dirserv_add_dir_bytes_to_out ssize_t bytes; int64_t remaining; - bytes = DIRSERV_BUFFER_MIN - buf_datalen(conn->_base.outbuf); + bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn)); tor_assert(bytes > 0); tor_assert(conn->cached_dir); if (bytes < 8192) @@@ -3572,7 -3551,7 +3579,7 @@@ static in connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) { - while (buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { if (conn->cached_dir) { int uncompressing = (conn->zlib_state != NULL); int r = connection_dirserv_add_dir_bytes_to_outbuf(conn); @@@ -3618,7 -3597,7 +3625,7 @@@ connection_dirserv_flushed_some(dir_con { tor_assert(conn->_base.state == DIR_CONN_STATE_SERVER_WRITING); - if (buf_datalen(conn->_base.outbuf) >= DIRSERV_BUFFER_MIN) + if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN) return 0; switch (conn->dir_spool_src) { diff --combined src/or/rephist.c index 0c5f937,53214d6..be91422 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@@ -7,14 -7,14 +7,15 @@@ * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've * been using, which ports we tend to want, and so on; further, - * exit port statistics and cell statistics. + * exit port statistics, cell statistics, and connection statistics. **/ #include "or.h" #include "circuitlist.h" #include "circuituse.h" #include "config.h" + #include "networkstatus.h" +#include "nodelist.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@@ -74,6 -74,13 +75,13 @@@ typedef struct or_history_t /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@@ -120,6 -127,7 +128,7 @@@ get_or_history(const char* id rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@@ -290,13 -298,20 +299,20 @@@ rep_hist_note_connection_died(const cha /** We have just decided that this router with identity digest id is * reachable, meaning we will give it a "Running" flag for the next while. */ void - rep_hist_note_router_reachable(const char *id, time_t when) + rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@@ -316,6 -331,27 +332,27 @@@ down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@@ -325,6 -361,10 +362,10 @@@ log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@@ -345,12 -385,20 +386,20 @@@ rep_hist_note_router_unreachable(const long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; - + if (run_length < 0) { + unsigned long penalty = -run_length; + #define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@@ -423,7 -471,7 +472,7 @@@ rep_hist_downrate_old_runs(time_t now static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@@ -459,8 -507,8 +508,8 @@@ get_total_weighted_time(or_history_t *h static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@@ -580,7 -628,7 +629,7 @@@ rep_hist_dump_stats(time_t now, int sev size_t len; int ret; unsigned long upt, downt; - routerinfo_t *r; + const node_t *node; rep_history_clean(now - get_options()->RephistTrackTime); @@@ -594,8 -642,8 +643,8 @@@ digestmap_iter_get(orhist_it, &digest1, &or_history_p); or_history = (or_history_t*) or_history_p; - if ((r = router_get_by_digest(digest1))) - name1 = r->nickname; + if ((node = node_get_by_id(digest1)) && node_get_nickname(node)) + name1 = node_get_nickname(node); else name1 = "(unknown)"; base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN); @@@ -625,8 -673,8 +674,8 @@@ lhist_it = digestmap_iter_next(or_history->link_history_map, lhist_it)) { digestmap_iter_get(lhist_it, &digest2, &link_history_p); - if ((r = router_get_by_digest(digest2))) - name2 = r->nickname; + if ((node = node_get_by_id(digest2)) && node_get_nickname(node)) + name2 = node_get_nickname(node); else name2 = "(unknown)"; @@@ -757,7 -805,7 +806,7 @@@ rep_hist_record_mtbf_data(time_t now, i base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN); if (missing_means_down && hist->start_of_run && - !router_get_by_digest(digest)) { + !router_get_by_id_digest(digest)) { /* We think this relay is running, but it's not listed in our * routerlist. Somehow it fell out without telling us it went * down. Complain and also correct it. */ @@@ -872,32 -920,28 +921,32 @@@ rep_hist_get_router_stability_doc(time_ } DIGESTMAP_FOREACH(history_map, id, or_history_t *, hist) { - routerinfo_t *ri; + const node_t *node; char dbuf[BASE64_DIGEST_LEN+1]; char header_buf[512]; char *info; digest_to_base64(dbuf, id); - ri = router_get_by_digest(id); - if (ri) { - char *ip = tor_dup_ip(ri->addr); + node = node_get_by_id(id); + if (node) { + char ip[INET_NTOA_BUF_LEN+1]; char tbuf[ISO_TIME_LEN+1]; - format_iso_time(tbuf, ri->cache_info.published_on); + time_t published = node_get_published_on(node); + node_get_address_string(node,ip,sizeof(ip)); + if (published > 0) + format_iso_time(tbuf, published); + else + strlcpy(tbuf, "???", sizeof(tbuf)); tor_snprintf(header_buf, sizeof(header_buf), "router %s %s %s\n" "published %s\n" "relevant-flags %s%s%s\n" "declared-uptime %ld\n", - dbuf, ri->nickname, ip, + dbuf, node_get_nickname(node), ip, tbuf, - ri->is_running ? "Running " : "", - ri->is_valid ? "Valid " : "", - ri->is_hibernating ? "Hibernating " : "", - ri->uptime); - tor_free(ip); + node->is_running ? "Running " : "", + node->is_valid ? "Valid " : "", + node->ri && node->ri->is_hibernating ? "Hibernating " : "", + node_get_declared_uptime(node)); } else { tor_snprintf(header_buf, sizeof(header_buf), "router %s {no descriptor}\n", dbuf); @@@ -1682,13 -1726,10 +1731,13 @@@ rep_hist_load_state(or_state_t *state, /*********************************************************************/ +typedef struct predicted_port_t { + uint16_t port; + time_t time; +} predicted_port_t; + /** A list of port numbers that have been used recently. */ static smartlist_t *predicted_ports_list=NULL; -/** The corresponding most recently used time for each port. */ -static smartlist_t *predicted_ports_times=NULL; /** We just got an application request for a connection with * port port. Remember it for the future, so we can keep @@@ -1697,11 -1738,14 +1746,11 @@@ static void add_predicted_port(time_t now, uint16_t port) { - /* XXXX we could just use uintptr_t here, I think. */ - uint16_t *tmp_port = tor_malloc(sizeof(uint16_t)); - time_t *tmp_time = tor_malloc(sizeof(time_t)); - *tmp_port = port; - *tmp_time = now; - rephist_total_alloc += sizeof(uint16_t) + sizeof(time_t); - smartlist_add(predicted_ports_list, tmp_port); - smartlist_add(predicted_ports_times, tmp_time); + predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t)); + pp->port = port; + pp->time = now; + rephist_total_alloc += sizeof(*pp); + smartlist_add(predicted_ports_list, pp); } /** Initialize whatever memory and structs are needed for predicting @@@ -1712,6 -1756,7 +1761,6 @@@ static voi predicted_ports_init(void) { predicted_ports_list = smartlist_create(); - predicted_ports_times = smartlist_create(); add_predicted_port(time(NULL), 80); /* add one to kickstart us */ } @@@ -1721,11 -1766,12 +1770,11 @@@ static void predicted_ports_free(void) { - rephist_total_alloc -= smartlist_len(predicted_ports_list)*sizeof(uint16_t); - SMARTLIST_FOREACH(predicted_ports_list, char *, cp, tor_free(cp)); + rephist_total_alloc -= + smartlist_len(predicted_ports_list)*sizeof(predicted_port_t); + SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *, + pp, tor_free(pp)); smartlist_free(predicted_ports_list); - rephist_total_alloc -= smartlist_len(predicted_ports_times)*sizeof(time_t); - SMARTLIST_FOREACH(predicted_ports_times, char *, cp, tor_free(cp)); - smartlist_free(predicted_ports_times); } /** Remember that port has been asked for as of time now. @@@ -1735,17 -1781,24 +1784,17 @@@ void rep_hist_note_used_port(time_t now, uint16_t port) { - int i; - uint16_t *tmp_port; - time_t *tmp_time; - tor_assert(predicted_ports_list); - tor_assert(predicted_ports_times); if (!port) /* record nothing */ return; - for (i = 0; i < smartlist_len(predicted_ports_list); ++i) { - tmp_port = smartlist_get(predicted_ports_list, i); - tmp_time = smartlist_get(predicted_ports_times, i); - if (*tmp_port == port) { - *tmp_time = now; + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (pp->port == port) { + pp->time = now; return; } - } + } SMARTLIST_FOREACH_END(pp); /* it's not there yet; we need to add it */ add_predicted_port(now, port); } @@@ -1754,28 -1807,36 +1803,28 @@@ * we'll want to make connections to the same port in the future. */ #define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) -/** Return a pointer to the list of port numbers that +/** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. - * - * The caller promises not to mess with it. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { - int i; - uint16_t *tmp_port; - time_t *tmp_time; - + smartlist_t *out = smartlist_create(); tor_assert(predicted_ports_list); - tor_assert(predicted_ports_times); /* clean out obsolete entries */ - for (i = 0; i < smartlist_len(predicted_ports_list); ++i) { - tmp_time = smartlist_get(predicted_ports_times, i); - if (*tmp_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { - tmp_port = smartlist_get(predicted_ports_list, i); - log_debug(LD_CIRC, "Expiring predicted port %d", *tmp_port); - smartlist_del(predicted_ports_list, i); - smartlist_del(predicted_ports_times, i); - rephist_total_alloc -= sizeof(uint16_t)+sizeof(time_t); - tor_free(tmp_port); - tor_free(tmp_time); - i--; + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); + + rephist_total_alloc -= sizeof(predicted_port_t); + tor_free(pp); + SMARTLIST_DEL_CURRENT(predicted_ports_list, pp); + } else { + smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t))); } - } - return predicted_ports_list; + } SMARTLIST_FOREACH_END(pp); + return out; } /** The user asked us to do a resolve. Rather than keeping track of @@@ -2011,9 -2072,7 +2060,9 @@@ rep_hist_exit_stats_term(void tor_free(exit_streams); } -/** Helper for qsort: compare two ints. */ +/** Helper for qsort: compare two ints. Does not handle overflow properly, + * but works fine for sorting an array of port numbers, which is what we use + * it for. */ static int _compare_int(const void *x, const void *y) { @@@ -2261,6 -2320,7 +2310,6 @@@ typedef struct circ_buffer_stats_t uint32_t processed_cells; double mean_num_cells_in_queue; double mean_time_cells_in_queue; - uint32_t local_circ_id; } circ_buffer_stats_t; /** Holds stats. */ @@@ -2283,9 -2343,9 +2332,9 @@@ rep_hist_buffer_stats_add_circ(circuit_ return; if (!circuits_for_buffer_stats) circuits_for_buffer_stats = smartlist_create(); - start_of_interval = circ->timestamp_created > - start_of_buffer_stats_interval ? - circ->timestamp_created : + start_of_interval = (circ->timestamp_created.tv_sec > + start_of_buffer_stats_interval) ? + circ->timestamp_created.tv_sec : start_of_buffer_stats_interval; interval_length = (int) (end_of_interval - start_of_interval); stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); @@@ -2451,227 -2511,6 +2500,227 @@@ rep_hist_buffer_stats_write(time_t now return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } +/*** Connection statistics ***/ + +/** Start of the current connection stats interval or 0 if we're not + * collecting connection statistics. */ +static time_t start_of_conn_stats_interval; + +/** Initialize connection stats. */ +void +rep_hist_conn_stats_init(time_t now) +{ + start_of_conn_stats_interval = now; +} + +/* Count connections that we read and wrote less than these many bytes + * from/to as below threshold. */ +#define BIDI_THRESHOLD 20480 + +/* Count connections that we read or wrote at least this factor as many + * bytes from/to than we wrote or read to/from as mostly reading or + * writing. */ +#define BIDI_FACTOR 10 + +/* Interval length in seconds for considering read and written bytes for + * connection stats. */ +#define BIDI_INTERVAL 10 + +/* Start of next BIDI_INTERVAL second interval. */ +static time_t bidi_next_interval = 0; + +/* Number of connections that we read and wrote less than BIDI_THRESHOLD + * bytes from/to in BIDI_INTERVAL seconds. */ +static uint32_t below_threshold = 0; + +/* Number of connections that we read at least BIDI_FACTOR times more + * bytes from than we wrote to in BIDI_INTERVAL seconds. */ +static uint32_t mostly_read = 0; + +/* Number of connections that we wrote at least BIDI_FACTOR times more + * bytes to than we read from in BIDI_INTERVAL seconds. */ +static uint32_t mostly_written = 0; + +/* Number of connections that we read and wrote at least BIDI_THRESHOLD + * bytes from/to, but not BIDI_FACTOR times more in either direction in + * BIDI_INTERVAL seconds. */ +static uint32_t both_read_and_written = 0; + +/* Entry in a map from connection ID to the number of read and written + * bytes on this connection in a BIDI_INTERVAL second interval. */ +typedef struct bidi_map_entry_t { + HT_ENTRY(bidi_map_entry_t) node; + uint64_t conn_id; /**< Connection ID */ + size_t read; /**< Number of read bytes */ + size_t written; /**< Number of written bytes */ +} bidi_map_entry_t; + +/** Map of OR connections together with the number of read and written + * bytes in the current BIDI_INTERVAL second interval. */ +static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map = + HT_INITIALIZER(); + +static int +bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b) +{ + return a->conn_id == b->conn_id; +} + +static unsigned +bidi_map_ent_hash(const bidi_map_entry_t *entry) +{ + return (unsigned) entry->conn_id; +} + +HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq); +HT_GENERATE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq, 0.6, malloc, realloc, free); + +static void +bidi_map_free(void) +{ + bidi_map_entry_t **ptr, **next, *ent; + for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) { + ent = *ptr; + next = HT_NEXT_RMV(bidimap, &bidi_map, ptr); + tor_free(ent); + } + HT_CLEAR(bidimap, &bidi_map); +} + +/** Reset counters for conn statistics. */ +void +rep_hist_reset_conn_stats(time_t now) +{ + start_of_conn_stats_interval = now; + below_threshold = 0; + mostly_read = 0; + mostly_written = 0; + both_read_and_written = 0; + bidi_map_free(); +} + +/** Stop collecting connection stats in a way that we can re-start doing + * so in rep_hist_conn_stats_init(). */ +void +rep_hist_conn_stats_term(void) +{ + rep_hist_reset_conn_stats(0); +} + +/** We read num_read bytes and wrote num_written from/to OR + * connection conn_id in second when. If this is the first + * observation in a new interval, sum up the last observations. Add bytes + * for this connection. */ +void +rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read, + size_t num_written, time_t when) +{ + if (!start_of_conn_stats_interval) + return; + /* Initialize */ + if (bidi_next_interval == 0) + bidi_next_interval = when + BIDI_INTERVAL; + /* Sum up last period's statistics */ + if (when >= bidi_next_interval) { + bidi_map_entry_t **ptr, **next, *ent; + for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) { + ent = *ptr; + if (ent->read + ent->written < BIDI_THRESHOLD) + below_threshold++; + else if (ent->read >= ent->written * BIDI_FACTOR) + mostly_read++; + else if (ent->written >= ent->read * BIDI_FACTOR) + mostly_written++; + else + both_read_and_written++; + next = HT_NEXT_RMV(bidimap, &bidi_map, ptr); + tor_free(ent); + } + while (when >= bidi_next_interval) + bidi_next_interval += BIDI_INTERVAL; + log_info(LD_GENERAL, "%d below threshold, %d mostly read, " + "%d mostly written, %d both read and written.", + below_threshold, mostly_read, mostly_written, + both_read_and_written); + } + /* Add this connection's bytes. */ + if (num_read > 0 || num_written > 0) { + bidi_map_entry_t *entry, lookup; + lookup.conn_id = conn_id; + entry = HT_FIND(bidimap, &bidi_map, &lookup); + if (entry) { + entry->written += num_written; + entry->read += num_read; + } else { + entry = tor_malloc_zero(sizeof(bidi_map_entry_t)); + entry->conn_id = conn_id; + entry->written = num_written; + entry->read = num_read; + HT_INSERT(bidimap, &bidi_map, entry); + } + } +} + +/** Return a newly allocated string containing the connection statistics + * until now, or NULL if we're not collecting conn stats. */ +char * +rep_hist_format_conn_stats(time_t now) +{ + char *result, written[ISO_TIME_LEN+1]; + + if (!start_of_conn_stats_interval) + return NULL; /* Not initialized. */ + + format_iso_time(written, now); + tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n", + written, + (unsigned) (now - start_of_conn_stats_interval), + below_threshold, + mostly_read, + mostly_written, + both_read_and_written); + return result; +} + +/** If 24 hours have passed since the beginning of the current conn stats + * period, write conn stats to $DATADIR/stats/conn-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write conn stats or 0 if we never want to write. */ +time_t +rep_hist_conn_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *str = NULL; + + if (!start_of_conn_stats_interval) + return 0; /* Not initialized. */ + if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* Generate history string. */ + str = rep_hist_format_conn_stats(now); + + /* Reset counters. */ + rep_hist_reset_conn_stats(now); + + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; + } + filename = get_datadir_fname2("stats", "conn-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write conn stats to disk!"); + + done: + tor_free(str); + tor_free(filename); + tor_free(statsdir); + return start_of_conn_stats_interval + WRITE_STATS_INTERVAL; +} + /** Free all storage held by the OR/link history caches, by the * bandwidth history arrays, by the port history, or by statistics . */ void @@@ -2686,6 -2525,5 +2735,6 @@@ rep_hist_free_all(void tor_free(exit_streams); built_last_stability_doc_at = 0; predicted_ports_free(); + bidi_map_free(); } diff --combined src/or/rephist.h index cb0ebf0,5f6b9f9..cb70e15 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@@ -33,7 -33,8 +33,8 @@@ void rep_hist_update_state(or_state_t * int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); - void rep_hist_note_router_reachable(const char *id, time_t when); + void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now); @@@ -76,13 -77,5 +77,13 @@@ void rep_hist_buffer_stats_add_circ(cir time_t rep_hist_buffer_stats_write(time_t now); void rep_hist_buffer_stats_term(void); +void rep_hist_conn_stats_init(time_t now); +void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read, + size_t num_written, time_t when); +void rep_hist_reset_conn_stats(time_t now); +char *rep_hist_format_conn_stats(time_t now); +time_t rep_hist_conn_stats_write(time_t now); +void rep_hist_conn_stats_term(void); + #endif

1 0

[tor/master] Merge remote branch 'sebastian/bug1035' into maint-0.2.2
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 0d78a16c3642f7538266e007da79c39860aac332 Merge: 9c72324 5a4f7fa Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 8 15:52:43 2011 -0500 Merge remote branch 'sebastian/bug1035' into maint-0.2.2 changes/bug1035 | 13 +++++++++ src/or/connection_or.c | 3 -- src/or/dirserv.c | 18 +++++++++--- src/or/rephist.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/or/rephist.h | 3 +- 5 files changed, 85 insertions(+), 17 deletions(-) diff --combined src/or/connection_or.c index b93699c,6279c93..2d24444 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -370,11 -370,11 +370,11 @@@ connection_or_update_token_buckets_help * bandwidth parameters in the consensus, but allow local config * options to override. */ rate = options->PerConnBWRate ? (int)options->PerConnBWRate : - (int)networkstatus_get_param(NULL, "perconnbwrate", - (int)options->BandwidthRate); + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : - (int)networkstatus_get_param(NULL, "perconnbwburst", - (int)options->BandwidthBurst); + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); } conn->bandwidthrate = rate; @@@ -1098,9 -1098,6 +1098,6 @@@ connection_or_check_valid_tls_handshake as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@@ -1259,8 -1256,7 +1256,8 @@@ connection_or_write_var_cell_to_buf(con tor_assert(conn); var_cell_pack_header(cell, hdr); connection_write_to_buf(hdr, sizeof(hdr), TO_CONN(conn)); - connection_write_to_buf(cell->payload, cell->payload_len, TO_CONN(conn)); + connection_write_to_buf((char*)cell->payload, + cell->payload_len, TO_CONN(conn)); if (cell->command != CELL_PADDING) conn->timestamp_last_added_nonpadding = approx_time(); } @@@ -1389,7 -1385,7 +1386,7 @@@ connection_or_send_netinfo(or_connectio time_t now = time(NULL); routerinfo_t *me; int len; - char *out; + uint8_t *out; memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; diff --combined src/or/dirserv.c index 3c15c59,f426881..aeeab45 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@@ -1,6 -1,6 +1,6 @@@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@@ -386,19 -386,13 +386,19 @@@ dirserv_get_status_impl(const char *id_ strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, - * and there aren't any critical relay-side vulnerabilities. Once more - * of them die off, we should raise this minimum to 0.2.0.x. */ - if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@@ -1166,7 -1160,7 +1166,7 @@@ directory_fetches_from_authorities(or_o return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - refuseunknown = router_my_exit_policy_is_reject_star() && + refuseunknown = ! router_my_exit_policy_is_reject_star() && should_refuse_unknown_exits(options); if (options->DirPort == 0 && !refuseunknown) return 0; @@@ -2970,8 -2964,6 +2970,8 @@@ dirserv_get_routerdesc_fingerprints(sma SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, smartlist_add(fps_out, tor_memdup(r->cache_info.identity_digest, DIGEST_LEN))); + /* Treat "all" requests as if they were unencrypted */ + for_unencrypted_conn = 1; } else if (!strcmp(key, "authority")) { routerinfo_t *ri = router_get_my_routerinfo(); if (ri) @@@ -3116,19 -3108,27 +3116,27 @@@ dirserv_orconn_tls_done(const char *add tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ diff --combined src/or/rephist.c index 61ae2c3,207eb88..53214d6 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@@ -1,5 -1,5 +1,5 @@@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -14,6 -14,7 +14,7 @@@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" + #include "networkstatus.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@@ -73,6 -74,13 +74,13 @@@ typedef struct or_history_t /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@@ -119,6 -127,7 +127,7 @@@ get_or_history(const char* id rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@@ -289,13 -298,20 +298,20 @@@ rep_hist_note_connection_died(const cha /** We have just decided that this router with identity digest id is * reachable, meaning we will give it a "Running" flag for the next while. */ void - rep_hist_note_router_reachable(const char *id, time_t when) + rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@@ -315,6 -331,27 +331,27 @@@ down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@@ -324,6 -361,10 +361,10 @@@ log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@@ -344,12 -385,20 +385,20 @@@ rep_hist_note_router_unreachable(const long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; - + if (run_length < 0) { + unsigned long penalty = -run_length; + #define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@@ -422,7 -471,7 +471,7 @@@ rep_hist_downrate_old_runs(time_t now static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@@ -458,8 -507,8 +507,8 @@@ get_total_weighted_time(or_history_t *h static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@@ -1165,8 -1214,6 +1214,8 @@@ rep_hist_load_mtbf_data(time_t now * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@@ -1261,12 -1308,8 +1310,12 @@@ add_obs(bw_array_t *b, time_t when, uin /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@@ -1297,15 -1340,10 +1346,15 @@@ static bw_array_t *dir_read_array = NUL directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@@ -1403,7 -1441,7 +1452,7 @@@ rep_hist_bandwidth_assess(void * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@@ -1495,181 -1533,163 +1544,181 @@@ rep_hist_get_bandwidth_lines(void return buf; } +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + /** Update state with the newest bandwidth history. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values = NULL; - time_t *s_begins = NULL; - int *s_interval = NULL; - bw_array_t *b = NULL; +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = &state->BWHistoryWriteEnds; - s_interval = &state->BWHistoryWriteInterval; - s_values = &state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = &state->BWHistoryReadEnds; - s_interval = &state->BWHistoryReadInterval; - s_values = &state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = &state->BWHistoryDirWriteEnds; - s_interval = &state->BWHistoryDirWriteInterval; - s_values = &state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = &state->BWHistoryDirReadEnds; - s_interval = &state->BWHistoryDirReadInterval; - s_values = &state->BWHistoryDirReadValues; - break; - } - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins = 0, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval = 0; - smartlist_t *s_values = NULL; - bw_array_t *b = NULL; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = state->BWHistoryWriteEnds; - s_interval = state->BWHistoryWriteInterval; - s_values = state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = state->BWHistoryReadEnds; - s_interval = state->BWHistoryReadInterval; - s_values = state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = state->BWHistoryDirWriteEnds; - s_interval = state->BWHistoryDirWriteInterval; - s_values = state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = state->BWHistoryDirReadEnds; - s_interval = state->BWHistoryDirReadInterval; - s_values = state->BWHistoryDirReadValues; - break; - } - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += s_interval; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} +/** Set bandwidth history from our saved state. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@@ -1972,8 -1992,9 +2021,8 @@@ dump_pk_ops(int severity #define EXIT_STATS_ROUND_UP_STREAMS 4 /** Number of TCP ports */ #define EXIT_STATS_NUM_PORTS 65536 -/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to - * see in order to be included in exit stats. */ -#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at @@@ -2023,24 -2044,15 +2072,24 @@@ rep_hist_exit_stats_term(void tor_free(exit_streams); } +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) +{ + return (*(int*)x - *(int*)y); +} + /** Return a newly allocated string containing the exit port statistics * until now, or NULL if we're not collecting exit stats. */ char * rep_hist_format_exit_stats(time_t now) { - int i; - uint64_t total_bytes = 0, threshold_bytes, other_read = 0, - other_written = 0; - uint32_t other_streams = 0; + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; char *buf; smartlist_t *written_strings, *read_strings, *streams_strings; char *written_string, *read_string, *streams_string; @@@ -2050,101 -2062,52 +2099,101 @@@ if (!start_of_exit_stats_interval) return NULL; /* Not initialized. */ - /* Count total number of bytes, so that we can attribute observations - * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL - * of all bytes to a special port 'other'. */ + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - total_bytes += exit_bytes_read[i]; - total_bytes += exit_bytes_written[i]; + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } } - threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - /* Add observations of all ports above the threshold to smartlists and - * join them to single strings. Also count bytes and streams of ports - * below or equal to the threshold. */ + /* Add observations of top ports to smartlists. */ written_strings = smartlist_create(); read_strings = smartlist_create(); streams_strings = smartlist_create(); - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - if (exit_bytes_written[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(written_strings, buf); - } - if (exit_bytes_read[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(read_strings, buf); - } - if (exit_streams[i] > 0) { - uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - buf = NULL; - tor_asprintf(&buf, "%d=%u", i, num); - smartlist_add(streams_strings, buf); - } - } else { - other_read += exit_bytes_read[i]; - other_written += exit_bytes_written[i]; - other_streams += exit_streams[i]; + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; } } + + /* Add observations of other ports in a single element. */ other_written = round_uint64_to_next_multiple_of(other_written, EXIT_STATS_ROUND_UP_BYTES); other_written /= 1024; @@@ -2162,8 -2125,6 +2211,8 @@@ buf = NULL; tor_asprintf(&buf, "other=%u", other_streams); smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ written_string = smartlist_join_strings(written_strings, ",", 0, NULL); read_string = smartlist_join_strings(read_strings, ",", 0, NULL); streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); diff --combined src/or/rephist.h index 9a39070,610c1a0..5f6b9f9 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -33,7 -33,8 +33,8 @@@ void rep_hist_update_state(or_state_t * int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); - void rep_hist_note_router_reachable(const char *id, time_t when); + void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now);

1 0

[tor/master] update spec locations
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 9c72324ae85c3f2cc23fee7d383128fa239b36d0 Author: Roger Dingledine <arma(a)torproject.org> Date: Tue Mar 8 15:31:04 2011 -0500 update spec locations --- doc/HACKING | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/HACKING b/doc/HACKING index bdb86c0..f42628b 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -5,17 +5,17 @@ Getting started --------------- For full information on how Tor is supposed to work, look at the files in -doc/spec/ . +https://gitweb.torproject.org/torspec.git/tree For an explanation of how to change Tor's design to work differently, look at -doc/spec/proposals/001-process.txt . +https://gitweb.torproject.org/torspec.git/blob_plain/HEAD:/proposals/001-process.txt For the latest version of the code, get a copy of git, and git clone git://git.torproject.org/git/tor . -We talk about Tor on the or-talk mailing list. Design proposals and -discussion belong on the or-dev mailing list. We hang around on +We talk about Tor on the tor-talk mailing list. Design proposals and +discussion belong on the tor-dev mailing list. We hang around on irc.oftc.net, with general discussion happening on #tor and development happening on #tor-dev. @@ -65,8 +65,8 @@ If at all possible, try to create this file in the same commit where you are making the change. Please give it a distinctive name that no other branch will use for the lifetime of your change. -When Roger goes to make a release, he will concatenate all the entries -in changes to make a draft changelog, and clear the directory. He'll +When we go to make a release, we will concatenate all the entries +in changes to make a draft changelog, and clear the directory. We'll then edit the draft changelog into a nice readable format. What needs a changes file?:: @@ -405,4 +405,3 @@ function should mention that it does that something in the documentation. If you rely on a function doing something beyond what is in its documentation, then you should watch out, or it might do something else later. -

1 0

[tor/maint-0.2.2] steps roger takes when making a new release
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit dbd4a0175600833c7f84081deaa40fd6294ce1ce Author: Roger Dingledine <arma(a)torproject.org> Date: Tue Mar 8 16:06:32 2011 -0500 steps roger takes when making a new release --- doc/HACKING | 41 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 41 insertions(+), 0 deletions(-) diff --git a/doc/HACKING b/doc/HACKING index f42628b..32cea99 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -405,3 +405,44 @@ function should mention that it does that something in the documentation. If you rely on a function doing something beyond what is in its documentation, then you should watch out, or it might do something else later. +Putting out a new release +------------------------- + +Here are the steps Roger takes when putting out a new Tor release: + +1) Use it for a while, as a client, as a relay, as a hidden service, +and as a directory authority. See if it has any obvious bugs, and +resolve those. + +2) Gather the changes/* files into a changelog entry, rewriting many +of them and reordering to focus on what users and funders would find +interesting and understandable. + +3) Compose a short release blurb to highlight the user-facing +changes. Insert said release blurb into the ChangeLog stanza. If it's +a stable release, add it to the ReleaseNotes file too. If we're adding +to a release-0.2.x branch, manually commit the changelogs to the later +git branches too. + +4) Bump the version number in configure.in and rebuild. + +5) Make dist, put the tarball up somewhere, and tell #tor about it. Wait +a while to see if anybody has problems building it. Try to get Sebastian +or somebody to try building it on Windows. + +6) Get at least two of weasel/arma/karsten to put the new version number +in their approved versions list. + +7) Sign and push the tarball to the website in the dist/ directory. Sign +and push the git tag. + +8) Edit include/versions.wmi to note the new version. Rebuild and push +the website. + +9) Email Erinn and weasel (cc'ing tor-assistants) that a new tarball +is up. This step should probably change to mailing more packagers. + +10) Wait up to a day or two (for a development release), or until most +packages are up (for a stable release), and mail the release blurb and +changelog to tor-talk or tor-announce. +

1 0

[tor/maint-0.2.2] Fix spelling and an unused #define
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit a68e2043aba4d33e39a8cb47be5dc3082f27ad07 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Mon Feb 7 16:15:02 2011 +0100 Fix spelling and an unused #define both noticed by rransom --- src/or/rephist.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 1bbfe31..2869990 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -75,7 +75,7 @@ typedef struct or_history_t { time_t down_since; /** The address at which we most recently connected to this OR - * sucessfully. */ + * successfully. */ tor_addr_t last_reached_addr; /* === For MTBF tracking: */ @@ -329,7 +329,6 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, } else if (addr_changed) { /* If we're reachable, but the address changed, treat this as some * downtime. */ -#define MIN_DOWNTIME_FOR_ADDR_CHANGE 600 int penalty = get_options()->TestingTorNetwork ? 240 : 3600; networkstatus_t *ns; tor_assert(at_addr);

1 0

[tor/maint-0.2.2] Merge remote branch 'sebastian/bug1035' into maint-0.2.2
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 0d78a16c3642f7538266e007da79c39860aac332 Merge: 9c72324 5a4f7fa Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 8 15:52:43 2011 -0500 Merge remote branch 'sebastian/bug1035' into maint-0.2.2 changes/bug1035 | 13 +++++++++ src/or/connection_or.c | 3 -- src/or/dirserv.c | 18 +++++++++--- src/or/rephist.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/or/rephist.h | 3 +- 5 files changed, 85 insertions(+), 17 deletions(-) diff --combined src/or/connection_or.c index b93699c,6279c93..2d24444 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -370,11 -370,11 +370,11 @@@ connection_or_update_token_buckets_help * bandwidth parameters in the consensus, but allow local config * options to override. */ rate = options->PerConnBWRate ? (int)options->PerConnBWRate : - (int)networkstatus_get_param(NULL, "perconnbwrate", - (int)options->BandwidthRate); + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : - (int)networkstatus_get_param(NULL, "perconnbwburst", - (int)options->BandwidthBurst); + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); } conn->bandwidthrate = rate; @@@ -1098,9 -1098,6 +1098,6 @@@ connection_or_check_valid_tls_handshake as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@@ -1259,8 -1256,7 +1256,8 @@@ connection_or_write_var_cell_to_buf(con tor_assert(conn); var_cell_pack_header(cell, hdr); connection_write_to_buf(hdr, sizeof(hdr), TO_CONN(conn)); - connection_write_to_buf(cell->payload, cell->payload_len, TO_CONN(conn)); + connection_write_to_buf((char*)cell->payload, + cell->payload_len, TO_CONN(conn)); if (cell->command != CELL_PADDING) conn->timestamp_last_added_nonpadding = approx_time(); } @@@ -1389,7 -1385,7 +1386,7 @@@ connection_or_send_netinfo(or_connectio time_t now = time(NULL); routerinfo_t *me; int len; - char *out; + uint8_t *out; memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; diff --combined src/or/dirserv.c index 3c15c59,f426881..aeeab45 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@@ -1,6 -1,6 +1,6 @@@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@@ -386,19 -386,13 +386,19 @@@ dirserv_get_status_impl(const char *id_ strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, - * and there aren't any critical relay-side vulnerabilities. Once more - * of them die off, we should raise this minimum to 0.2.0.x. */ - if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@@ -1166,7 -1160,7 +1166,7 @@@ directory_fetches_from_authorities(or_o return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - refuseunknown = router_my_exit_policy_is_reject_star() && + refuseunknown = ! router_my_exit_policy_is_reject_star() && should_refuse_unknown_exits(options); if (options->DirPort == 0 && !refuseunknown) return 0; @@@ -2970,8 -2964,6 +2970,8 @@@ dirserv_get_routerdesc_fingerprints(sma SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, smartlist_add(fps_out, tor_memdup(r->cache_info.identity_digest, DIGEST_LEN))); + /* Treat "all" requests as if they were unencrypted */ + for_unencrypted_conn = 1; } else if (!strcmp(key, "authority")) { routerinfo_t *ri = router_get_my_routerinfo(); if (ri) @@@ -3116,19 -3108,27 +3116,27 @@@ dirserv_orconn_tls_done(const char *add tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ diff --combined src/or/rephist.c index 61ae2c3,207eb88..53214d6 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@@ -1,5 -1,5 +1,5 @@@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -14,6 -14,7 +14,7 @@@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" + #include "networkstatus.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@@ -73,6 -74,13 +74,13 @@@ typedef struct or_history_t /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@@ -119,6 -127,7 +127,7 @@@ get_or_history(const char* id rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@@ -289,13 -298,20 +298,20 @@@ rep_hist_note_connection_died(const cha /** We have just decided that this router with identity digest id is * reachable, meaning we will give it a "Running" flag for the next while. */ void - rep_hist_note_router_reachable(const char *id, time_t when) + rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@@ -315,6 -331,27 +331,27 @@@ down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@@ -324,6 -361,10 +361,10 @@@ log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@@ -344,12 -385,20 +385,20 @@@ rep_hist_note_router_unreachable(const long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; - + if (run_length < 0) { + unsigned long penalty = -run_length; + #define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@@ -422,7 -471,7 +471,7 @@@ rep_hist_downrate_old_runs(time_t now static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@@ -458,8 -507,8 +507,8 @@@ get_total_weighted_time(or_history_t *h static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@@ -1165,8 -1214,6 +1214,8 @@@ rep_hist_load_mtbf_data(time_t now * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@@ -1261,12 -1308,8 +1310,12 @@@ add_obs(bw_array_t *b, time_t when, uin /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@@ -1297,15 -1340,10 +1346,15 @@@ static bw_array_t *dir_read_array = NUL directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@@ -1403,7 -1441,7 +1452,7 @@@ rep_hist_bandwidth_assess(void * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@@ -1495,181 -1533,163 +1544,181 @@@ rep_hist_get_bandwidth_lines(void return buf; } +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + /** Update state with the newest bandwidth history. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values = NULL; - time_t *s_begins = NULL; - int *s_interval = NULL; - bw_array_t *b = NULL; +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = &state->BWHistoryWriteEnds; - s_interval = &state->BWHistoryWriteInterval; - s_values = &state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = &state->BWHistoryReadEnds; - s_interval = &state->BWHistoryReadInterval; - s_values = &state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = &state->BWHistoryDirWriteEnds; - s_interval = &state->BWHistoryDirWriteInterval; - s_values = &state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = &state->BWHistoryDirReadEnds; - s_interval = &state->BWHistoryDirReadInterval; - s_values = &state->BWHistoryDirReadValues; - break; - } - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins = 0, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval = 0; - smartlist_t *s_values = NULL; - bw_array_t *b = NULL; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = state->BWHistoryWriteEnds; - s_interval = state->BWHistoryWriteInterval; - s_values = state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = state->BWHistoryReadEnds; - s_interval = state->BWHistoryReadInterval; - s_values = state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = state->BWHistoryDirWriteEnds; - s_interval = state->BWHistoryDirWriteInterval; - s_values = state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = state->BWHistoryDirReadEnds; - s_interval = state->BWHistoryDirReadInterval; - s_values = state->BWHistoryDirReadValues; - break; - } - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += s_interval; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} +/** Set bandwidth history from our saved state. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@@ -1972,8 -1992,9 +2021,8 @@@ dump_pk_ops(int severity #define EXIT_STATS_ROUND_UP_STREAMS 4 /** Number of TCP ports */ #define EXIT_STATS_NUM_PORTS 65536 -/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to - * see in order to be included in exit stats. */ -#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at @@@ -2023,24 -2044,15 +2072,24 @@@ rep_hist_exit_stats_term(void tor_free(exit_streams); } +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) +{ + return (*(int*)x - *(int*)y); +} + /** Return a newly allocated string containing the exit port statistics * until now, or NULL if we're not collecting exit stats. */ char * rep_hist_format_exit_stats(time_t now) { - int i; - uint64_t total_bytes = 0, threshold_bytes, other_read = 0, - other_written = 0; - uint32_t other_streams = 0; + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; char *buf; smartlist_t *written_strings, *read_strings, *streams_strings; char *written_string, *read_string, *streams_string; @@@ -2050,101 -2062,52 +2099,101 @@@ if (!start_of_exit_stats_interval) return NULL; /* Not initialized. */ - /* Count total number of bytes, so that we can attribute observations - * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL - * of all bytes to a special port 'other'. */ + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - total_bytes += exit_bytes_read[i]; - total_bytes += exit_bytes_written[i]; + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } } - threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - /* Add observations of all ports above the threshold to smartlists and - * join them to single strings. Also count bytes and streams of ports - * below or equal to the threshold. */ + /* Add observations of top ports to smartlists. */ written_strings = smartlist_create(); read_strings = smartlist_create(); streams_strings = smartlist_create(); - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - if (exit_bytes_written[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(written_strings, buf); - } - if (exit_bytes_read[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(read_strings, buf); - } - if (exit_streams[i] > 0) { - uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - buf = NULL; - tor_asprintf(&buf, "%d=%u", i, num); - smartlist_add(streams_strings, buf); - } - } else { - other_read += exit_bytes_read[i]; - other_written += exit_bytes_written[i]; - other_streams += exit_streams[i]; + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; } } + + /* Add observations of other ports in a single element. */ other_written = round_uint64_to_next_multiple_of(other_written, EXIT_STATS_ROUND_UP_BYTES); other_written /= 1024; @@@ -2162,8 -2125,6 +2211,8 @@@ buf = NULL; tor_asprintf(&buf, "other=%u", other_streams); smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ written_string = smartlist_join_strings(written_strings, ",", 0, NULL); read_string = smartlist_join_strings(read_strings, ",", 0, NULL); streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); diff --combined src/or/rephist.h index 9a39070,610c1a0..5f6b9f9 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -33,7 -33,8 +33,8 @@@ void rep_hist_update_state(or_state_t * int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); - void rep_hist_note_router_reachable(const char *id, time_t when); + void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now);

1 0

[tor/maint-0.2.2] Routers count as down when they change ORPort, too
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 9b64227ffd38e9406c5c88ace137a0eae010771d Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Mon Feb 7 16:31:20 2011 +0100 Routers count as down when they change ORPort, too rransom noticed that a change of ORPort is just as bad as a change of IP address from a client's perspective, because both mean that the relay is not available to them while the new information hasn't propagated. Change the bug1035 fix accordingly. Also make sure we don't log a bridge's IP address (which might happen when we are the bridge authority). --- changes/bug1035 | 11 ++++++----- src/or/dirserv.c | 8 +++++--- src/or/rephist.c | 23 +++++++++++++++-------- src/or/rephist.h | 2 +- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/changes/bug1035 b/changes/bug1035 index 041e3b3..3d86330 100644 --- a/changes/bug1035 +++ b/changes/bug1035 @@ -1,9 +1,10 @@ o Minor features (authorities) - - Take altered router IPs into account when determining router stability. - Previously, if a router changed its IP, the authorities would not - treat it as having any downtime for the purposes of stability - calculation, whereas clients would experience downtime since the - IP could take a while to propagate to them. Resolves issue 1035. + - Take altered router IP addresses and ORPorts into account when + determining router stability. Previously, if a router changed + its IP or ORPort, the authorities would not treat it as having + any downtime for the purposes of stability calculation, whereas + clients would experience downtime since the change could take a + while to propagate to them. Resolves issue 1035. o Minor bugfixes (authorities) - Try to be more robust to hops back in time when calculating router stability. Previously, if a run of uptime or downtime diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 52e59cd..f426881 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3118,11 +3118,13 @@ dirserv_orconn_tls_done(const char *address, /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { tor_addr_t addr, *addrp=NULL; - log_info(LD_DIRSERV, "Found router %s to be reachable at %s. Yay.", - ri->nickname, address); + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); if (tor_addr_from_str(&addr, ri->address) != -1) addrp = &addr; - rep_hist_note_router_reachable(digest_rcvd, addrp, now); + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } diff --git a/src/or/rephist.c b/src/or/rephist.c index 2869990..7c570e2 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -78,6 +78,9 @@ typedef struct or_history_t { * successfully. */ tor_addr_t last_reached_addr; + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@ -296,17 +299,18 @@ rep_hist_note_connection_died(const char* id, time_t when) * reachable, meaning we will give it a "Running" flag for the next while. */ void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, - time_t when) + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; - int addr_changed; + int addr_changed, port_changed; tor_assert(hist); addr_changed = at_addr && tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@ -326,7 +330,7 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; - } else if (addr_changed) { + } else if (addr_changed || port_changed) { /* If we're reachable, but the address changed, treat this as some * downtime. */ int penalty = get_options()->TestingTorNetwork ? 240 : 3600; @@ -342,12 +346,13 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, penalty = (int)(fresh_interval + live_interval) / 2; } format_local_iso_time(tbuf, hist->start_of_run); - log_info(LD_HIST,"Router %s still seems Running, but its address appears " - "to have changed since the last time it was reachable. I'm " - "going to treat it as having been down for %d seconds", - hex_str(id, DIGEST_LEN), penalty); + if (!authdir_mode_bridge(get_options())) + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); rep_hist_note_router_unreachable(id, when-penalty); - rep_hist_note_router_reachable(id, NULL, when); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@ -359,6 +364,8 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, } if (at_addr) tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning diff --git a/src/or/rephist.h b/src/or/rephist.h index 1ddbac4..610c1a0 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -34,7 +34,7 @@ int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, - time_t when); + uint16_t at_port, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now);

1 0

[tor/maint-0.2.2] clarify an assert
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 5a4f7fa1e48923730376c0a42121e4c3022eef3b Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sat Feb 26 09:42:44 2011 +0100 clarify an assert also log about running changes, even on a bridge authority. --- src/or/rephist.c | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 7c570e2..207eb88 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -307,6 +307,7 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); addr_changed = at_addr && tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; @@ -335,7 +336,6 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, * downtime. */ int penalty = get_options()->TestingTorNetwork ? 240 : 3600; networkstatus_t *ns; - tor_assert(at_addr); if ((ns = networkstatus_get_latest_consensus())) { int fresh_interval = (int)(ns->fresh_until - ns->valid_after); @@ -346,11 +346,10 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, penalty = (int)(fresh_interval + live_interval) / 2; } format_local_iso_time(tbuf, hist->start_of_run); - if (!authdir_mode_bridge(get_options())) - log_info(LD_HIST,"Router %s still seems Running, but its address appears " - "to have changed since the last time it was reachable. I'm " - "going to treat it as having been down for %d seconds", - hex_str(id, DIGEST_LEN), penalty); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); rep_hist_note_router_unreachable(id, when-penalty); rep_hist_note_router_reachable(id, NULL, 0, when); } else {

1 0

[tor/maint-0.2.2] Fix compile without warnings on OS X 10.6
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 59a3d536d80d49da956d5e4e0c59a97164e1f9c4 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Mon Dec 27 11:37:16 2010 +0100 Fix compile without warnings on OS X 10.6 --- src/or/rephist.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index e59fcb5..1bbfe31 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -335,12 +335,12 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, tor_assert(at_addr); if ((ns = networkstatus_get_latest_consensus())) { - int fresh_interval = ns->fresh_until - ns->valid_after; - int live_interval = ns->valid_until - ns->valid_after; + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); /* on average, a descriptor addr change takes .5 intervals to make it * into a consensus, and half a liveness period to make it to * clients. */ - penalty = (fresh_interval + live_interval) / 2; + penalty = (int)(fresh_interval + live_interval) / 2; } format_local_iso_time(tbuf, hist->start_of_run); log_info(LD_HIST,"Router %s still seems Running, but its address appears "

1 0