- tor-commits - lists.torproject.org

[tor/release-0.2.2] make nickm's proposed convention from 2003 be gospel
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit 95edd51116c17327f314bc132a0a5f86499e410a Author: Roger Dingledine <arma(a)torproject.org> Date: Tue Mar 8 14:59:30 2011 -0500 make nickm's proposed convention from 2003 be gospel --- doc/HACKING | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/HACKING b/doc/HACKING index 486fe6d..bdb86c0 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -312,11 +312,11 @@ operation. If a library function is currently called such that failure always means ERR, then the library function should log WARN and let the caller log ERR. -[XXX Proposed convention: every message of severity INFO or higher should -either (A) be intelligible to end-users who don't know the Tor source; or (B) -somehow inform the end-users that they aren't expected to understand the -message (perhaps with a string like "internal error"). Option (A) is to be -preferred to option (B). -NM] +Every message of severity INFO or higher should either (A) be intelligible +to end-users who don't know the Tor source; or (B) somehow inform the +end-users that they aren't expected to understand the message (perhaps +with a string like "internal error"). Option (A) is to be preferred to +option (B). Doxygen ~~~~~~~~

1 0

[tor/release-0.2.2] Merge remote branch 'sebastian/bug1035' into maint-0.2.2
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit 0d78a16c3642f7538266e007da79c39860aac332 Merge: 9c72324 5a4f7fa Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 8 15:52:43 2011 -0500 Merge remote branch 'sebastian/bug1035' into maint-0.2.2 changes/bug1035 | 13 +++++++++ src/or/connection_or.c | 3 -- src/or/dirserv.c | 18 +++++++++--- src/or/rephist.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/or/rephist.h | 3 +- 5 files changed, 85 insertions(+), 17 deletions(-) diff --combined src/or/connection_or.c index b93699c,6279c93..2d24444 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -370,11 -370,11 +370,11 @@@ connection_or_update_token_buckets_help * bandwidth parameters in the consensus, but allow local config * options to override. */ rate = options->PerConnBWRate ? (int)options->PerConnBWRate : - (int)networkstatus_get_param(NULL, "perconnbwrate", - (int)options->BandwidthRate); + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : - (int)networkstatus_get_param(NULL, "perconnbwburst", - (int)options->BandwidthBurst); + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); } conn->bandwidthrate = rate; @@@ -1098,9 -1098,6 +1098,6 @@@ connection_or_check_valid_tls_handshake as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@@ -1259,8 -1256,7 +1256,8 @@@ connection_or_write_var_cell_to_buf(con tor_assert(conn); var_cell_pack_header(cell, hdr); connection_write_to_buf(hdr, sizeof(hdr), TO_CONN(conn)); - connection_write_to_buf(cell->payload, cell->payload_len, TO_CONN(conn)); + connection_write_to_buf((char*)cell->payload, + cell->payload_len, TO_CONN(conn)); if (cell->command != CELL_PADDING) conn->timestamp_last_added_nonpadding = approx_time(); } @@@ -1389,7 -1385,7 +1386,7 @@@ connection_or_send_netinfo(or_connectio time_t now = time(NULL); routerinfo_t *me; int len; - char *out; + uint8_t *out; memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; diff --combined src/or/dirserv.c index 3c15c59,f426881..aeeab45 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@@ -1,6 -1,6 +1,6 @@@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@@ -386,19 -386,13 +386,19 @@@ dirserv_get_status_impl(const char *id_ strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, - * and there aren't any critical relay-side vulnerabilities. Once more - * of them die off, we should raise this minimum to 0.2.0.x. */ - if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@@ -1166,7 -1160,7 +1166,7 @@@ directory_fetches_from_authorities(or_o return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - refuseunknown = router_my_exit_policy_is_reject_star() && + refuseunknown = ! router_my_exit_policy_is_reject_star() && should_refuse_unknown_exits(options); if (options->DirPort == 0 && !refuseunknown) return 0; @@@ -2970,8 -2964,6 +2970,8 @@@ dirserv_get_routerdesc_fingerprints(sma SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, smartlist_add(fps_out, tor_memdup(r->cache_info.identity_digest, DIGEST_LEN))); + /* Treat "all" requests as if they were unencrypted */ + for_unencrypted_conn = 1; } else if (!strcmp(key, "authority")) { routerinfo_t *ri = router_get_my_routerinfo(); if (ri) @@@ -3116,19 -3108,27 +3116,27 @@@ dirserv_orconn_tls_done(const char *add tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ diff --combined src/or/rephist.c index 61ae2c3,207eb88..53214d6 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@@ -1,5 -1,5 +1,5 @@@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -14,6 -14,7 +14,7 @@@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" + #include "networkstatus.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@@ -73,6 -74,13 +74,13 @@@ typedef struct or_history_t /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@@ -119,6 -127,7 +127,7 @@@ get_or_history(const char* id rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@@ -289,13 -298,20 +298,20 @@@ rep_hist_note_connection_died(const cha /** We have just decided that this router with identity digest id is * reachable, meaning we will give it a "Running" flag for the next while. */ void - rep_hist_note_router_reachable(const char *id, time_t when) + rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@@ -315,6 -331,27 +331,27 @@@ down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@@ -324,6 -361,10 +361,10 @@@ log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@@ -344,12 -385,20 +385,20 @@@ rep_hist_note_router_unreachable(const long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; - + if (run_length < 0) { + unsigned long penalty = -run_length; + #define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@@ -422,7 -471,7 +471,7 @@@ rep_hist_downrate_old_runs(time_t now static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@@ -458,8 -507,8 +507,8 @@@ get_total_weighted_time(or_history_t *h static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@@ -1165,8 -1214,6 +1214,8 @@@ rep_hist_load_mtbf_data(time_t now * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@@ -1261,12 -1308,8 +1310,12 @@@ add_obs(bw_array_t *b, time_t when, uin /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@@ -1297,15 -1340,10 +1346,15 @@@ static bw_array_t *dir_read_array = NUL directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@@ -1403,7 -1441,7 +1452,7 @@@ rep_hist_bandwidth_assess(void * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@@ -1495,181 -1533,163 +1544,181 @@@ rep_hist_get_bandwidth_lines(void return buf; } +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + /** Update state with the newest bandwidth history. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values = NULL; - time_t *s_begins = NULL; - int *s_interval = NULL; - bw_array_t *b = NULL; +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = &state->BWHistoryWriteEnds; - s_interval = &state->BWHistoryWriteInterval; - s_values = &state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = &state->BWHistoryReadEnds; - s_interval = &state->BWHistoryReadInterval; - s_values = &state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = &state->BWHistoryDirWriteEnds; - s_interval = &state->BWHistoryDirWriteInterval; - s_values = &state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = &state->BWHistoryDirReadEnds; - s_interval = &state->BWHistoryDirReadInterval; - s_values = &state->BWHistoryDirReadValues; - break; - } - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins = 0, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval = 0; - smartlist_t *s_values = NULL; - bw_array_t *b = NULL; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = state->BWHistoryWriteEnds; - s_interval = state->BWHistoryWriteInterval; - s_values = state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = state->BWHistoryReadEnds; - s_interval = state->BWHistoryReadInterval; - s_values = state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = state->BWHistoryDirWriteEnds; - s_interval = state->BWHistoryDirWriteInterval; - s_values = state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = state->BWHistoryDirReadEnds; - s_interval = state->BWHistoryDirReadInterval; - s_values = state->BWHistoryDirReadValues; - break; - } - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += s_interval; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} +/** Set bandwidth history from our saved state. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@@ -1972,8 -1992,9 +2021,8 @@@ dump_pk_ops(int severity #define EXIT_STATS_ROUND_UP_STREAMS 4 /** Number of TCP ports */ #define EXIT_STATS_NUM_PORTS 65536 -/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to - * see in order to be included in exit stats. */ -#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at @@@ -2023,24 -2044,15 +2072,24 @@@ rep_hist_exit_stats_term(void tor_free(exit_streams); } +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) +{ + return (*(int*)x - *(int*)y); +} + /** Return a newly allocated string containing the exit port statistics * until now, or NULL if we're not collecting exit stats. */ char * rep_hist_format_exit_stats(time_t now) { - int i; - uint64_t total_bytes = 0, threshold_bytes, other_read = 0, - other_written = 0; - uint32_t other_streams = 0; + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; char *buf; smartlist_t *written_strings, *read_strings, *streams_strings; char *written_string, *read_string, *streams_string; @@@ -2050,101 -2062,52 +2099,101 @@@ if (!start_of_exit_stats_interval) return NULL; /* Not initialized. */ - /* Count total number of bytes, so that we can attribute observations - * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL - * of all bytes to a special port 'other'. */ + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - total_bytes += exit_bytes_read[i]; - total_bytes += exit_bytes_written[i]; + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } } - threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - /* Add observations of all ports above the threshold to smartlists and - * join them to single strings. Also count bytes and streams of ports - * below or equal to the threshold. */ + /* Add observations of top ports to smartlists. */ written_strings = smartlist_create(); read_strings = smartlist_create(); streams_strings = smartlist_create(); - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - if (exit_bytes_written[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(written_strings, buf); - } - if (exit_bytes_read[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(read_strings, buf); - } - if (exit_streams[i] > 0) { - uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - buf = NULL; - tor_asprintf(&buf, "%d=%u", i, num); - smartlist_add(streams_strings, buf); - } - } else { - other_read += exit_bytes_read[i]; - other_written += exit_bytes_written[i]; - other_streams += exit_streams[i]; + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; } } + + /* Add observations of other ports in a single element. */ other_written = round_uint64_to_next_multiple_of(other_written, EXIT_STATS_ROUND_UP_BYTES); other_written /= 1024; @@@ -2162,8 -2125,6 +2211,8 @@@ buf = NULL; tor_asprintf(&buf, "other=%u", other_streams); smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ written_string = smartlist_join_strings(written_strings, ",", 0, NULL); read_string = smartlist_join_strings(read_strings, ",", 0, NULL); streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); diff --combined src/or/rephist.h index 9a39070,610c1a0..5f6b9f9 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -33,7 -33,8 +33,8 @@@ void rep_hist_update_state(or_state_t * int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); - void rep_hist_note_router_reachable(const char *id, time_t when); + void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now);

1 0

[tor/release-0.2.2] Handle negative run lengths in wfu/mtbf calculations
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit 5a9903b9e09697ac131c841310dd82eebcca02e0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Nov 22 12:39:22 2010 -0500 Handle negative run lengths in wfu/mtbf calculations --- changes/bug1035 | 5 +++++ src/or/rephist.c | 22 +++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/changes/bug1035 b/changes/bug1035 index 4b62b46..041e3b3 100644 --- a/changes/bug1035 +++ b/changes/bug1035 @@ -4,4 +4,9 @@ treat it as having any downtime for the purposes of stability calculation, whereas clients would experience downtime since the IP could take a while to propagate to them. Resolves issue 1035. + o Minor bugfixes (authorities) + - Try to be more robust to hops back in time when calculating + router stability. Previously, if a run of uptime or downtime + appeared to be negative, the calculation could give incorrect + results. Bugfix on 0.2.0.6-alpha. diff --git a/src/or/rephist.c b/src/or/rephist.c index a0c9c9f..e59fcb5 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -380,12 +380,20 @@ rep_hist_note_router_unreachable(const char *id, time_t when) long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; + if (run_length < 0) { + unsigned long penalty = -run_length; +#define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@ -458,7 +466,7 @@ rep_hist_downrate_old_runs(time_t now) static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@ -494,8 +502,8 @@ get_total_weighted_time(or_history_t *hist, time_t when) static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run);

1 0

[tor/release-0.2.2] Treat routers whose IPs have changed as having been down for MTBF/routerinfo
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit bea0a31c1c859a563f065d8b868570560abc5135 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Nov 22 12:30:33 2010 -0500 Treat routers whose IPs have changed as having been down for MTBF/routerinfo calculation purposes. --- changes/bug1035 | 7 +++++++ src/or/dirserv.c | 16 +++++++++++----- src/or/rephist.c | 38 +++++++++++++++++++++++++++++++++++++- src/or/rephist.h | 3 ++- 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/changes/bug1035 b/changes/bug1035 new file mode 100644 index 0000000..4b62b46 --- /dev/null +++ b/changes/bug1035 @@ -0,0 +1,7 @@ + o Minor features (authorities) + - Take altered router IPs into account when determining router stability. + Previously, if a router changed its IP, the authorities would not + treat it as having any downtime for the purposes of stability + calculation, whereas clients would experience downtime since the + IP could take a while to propagate to them. Resolves issue 1035. + diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 42d7d56..52e59cd 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3108,19 +3108,25 @@ dirserv_orconn_tls_done(const char *address, tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s. Yay.", + ri->nickname, address); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + rep_hist_note_router_reachable(digest_rcvd, addrp, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ diff --git a/src/or/rephist.c b/src/or/rephist.c index 22b3ec5..a0c9c9f 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -14,6 +14,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "networkstatus.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@ -73,6 +74,10 @@ typedef struct or_history_t { /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * sucessfully. */ + tor_addr_t last_reached_addr; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@ -119,6 +124,7 @@ get_or_history(const char* id) rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@ -289,14 +295,19 @@ rep_hist_note_connection_died(const char* id, time_t when) /** We have just decided that this router with identity digest id is * reachable, meaning we will give it a "Running" flag for the next while. */ void -rep_hist_note_router_reachable(const char *id, time_t when) +rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed; tor_assert(hist); + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + if (!started_tracking_stability) started_tracking_stability = time(NULL); if (!hist->start_of_run) { @@ -315,6 +326,29 @@ rep_hist_note_router_reachable(const char *id, time_t when) down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ +#define MIN_DOWNTIME_FOR_ADDR_CHANGE 600 + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + tor_assert(at_addr); + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = ns->fresh_until - ns->valid_after; + int live_interval = ns->valid_until - ns->valid_after; + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@ -324,6 +358,8 @@ rep_hist_note_router_reachable(const char *id, time_t when) log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); } /** We have just decided that this router is unreachable, meaning diff --git a/src/or/rephist.h b/src/or/rephist.h index 8f5a34d..1ddbac4 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -33,7 +33,8 @@ void rep_hist_update_state(or_state_t *state); int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); -void rep_hist_note_router_reachable(const char *id, time_t when); +void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now);

1 0

[tor/release-0.2.2] Fix compile without warnings on OS X 10.6
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit 59a3d536d80d49da956d5e4e0c59a97164e1f9c4 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Mon Dec 27 11:37:16 2010 +0100 Fix compile without warnings on OS X 10.6 --- src/or/rephist.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index e59fcb5..1bbfe31 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -335,12 +335,12 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, tor_assert(at_addr); if ((ns = networkstatus_get_latest_consensus())) { - int fresh_interval = ns->fresh_until - ns->valid_after; - int live_interval = ns->valid_until - ns->valid_after; + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); /* on average, a descriptor addr change takes .5 intervals to make it * into a consensus, and half a liveness period to make it to * clients. */ - penalty = (fresh_interval + live_interval) / 2; + penalty = (int)(fresh_interval + live_interval) / 2; } format_local_iso_time(tbuf, hist->start_of_run); log_info(LD_HIST,"Router %s still seems Running, but its address appears "

1 0

[tor/release-0.2.2] Remove an incorrect comment in connection_or_check_valid_tls_handshake
by arma＠torproject.org 08 Mar '11

08 Mar '11

commit 80357abb11023ff285c251ee75b864e0a27d5809 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Nov 22 11:36:22 2010 -0500 Remove an incorrect comment in connection_or_check_valid_tls_handshake --- src/or/connection_or.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 415a48d..6279c93 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1098,9 +1098,6 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); }

1 0

[tor/master] clarify an assert
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 5a4f7fa1e48923730376c0a42121e4c3022eef3b Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Sat Feb 26 09:42:44 2011 +0100 clarify an assert also log about running changes, even on a bridge authority. --- src/or/rephist.c | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 7c570e2..207eb88 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -307,6 +307,7 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); addr_changed = at_addr && tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; @@ -335,7 +336,6 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, * downtime. */ int penalty = get_options()->TestingTorNetwork ? 240 : 3600; networkstatus_t *ns; - tor_assert(at_addr); if ((ns = networkstatus_get_latest_consensus())) { int fresh_interval = (int)(ns->fresh_until - ns->valid_after); @@ -346,11 +346,10 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, penalty = (int)(fresh_interval + live_interval) / 2; } format_local_iso_time(tbuf, hist->start_of_run); - if (!authdir_mode_bridge(get_options())) - log_info(LD_HIST,"Router %s still seems Running, but its address appears " - "to have changed since the last time it was reachable. I'm " - "going to treat it as having been down for %d seconds", - hex_str(id, DIGEST_LEN), penalty); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); rep_hist_note_router_unreachable(id, when-penalty); rep_hist_note_router_reachable(id, NULL, 0, when); } else {

1 0

[tor/master] Fix spelling and an unused #define
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit a68e2043aba4d33e39a8cb47be5dc3082f27ad07 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Mon Feb 7 16:15:02 2011 +0100 Fix spelling and an unused #define both noticed by rransom --- src/or/rephist.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 1bbfe31..2869990 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -75,7 +75,7 @@ typedef struct or_history_t { time_t down_since; /** The address at which we most recently connected to this OR - * sucessfully. */ + * successfully. */ tor_addr_t last_reached_addr; /* === For MTBF tracking: */ @@ -329,7 +329,6 @@ rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, } else if (addr_changed) { /* If we're reachable, but the address changed, treat this as some * downtime. */ -#define MIN_DOWNTIME_FOR_ADDR_CHANGE 600 int penalty = get_options()->TestingTorNetwork ? 240 : 3600; networkstatus_t *ns; tor_assert(at_addr);

1 0

[tor/master] update spec locations
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 9c72324ae85c3f2cc23fee7d383128fa239b36d0 Author: Roger Dingledine <arma(a)torproject.org> Date: Tue Mar 8 15:31:04 2011 -0500 update spec locations --- doc/HACKING | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/HACKING b/doc/HACKING index bdb86c0..f42628b 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -5,17 +5,17 @@ Getting started --------------- For full information on how Tor is supposed to work, look at the files in -doc/spec/ . +https://gitweb.torproject.org/torspec.git/tree For an explanation of how to change Tor's design to work differently, look at -doc/spec/proposals/001-process.txt . +https://gitweb.torproject.org/torspec.git/blob_plain/HEAD:/proposals/001-process.txt For the latest version of the code, get a copy of git, and git clone git://git.torproject.org/git/tor . -We talk about Tor on the or-talk mailing list. Design proposals and -discussion belong on the or-dev mailing list. We hang around on +We talk about Tor on the tor-talk mailing list. Design proposals and +discussion belong on the tor-dev mailing list. We hang around on irc.oftc.net, with general discussion happening on #tor and development happening on #tor-dev. @@ -65,8 +65,8 @@ If at all possible, try to create this file in the same commit where you are making the change. Please give it a distinctive name that no other branch will use for the lifetime of your change. -When Roger goes to make a release, he will concatenate all the entries -in changes to make a draft changelog, and clear the directory. He'll +When we go to make a release, we will concatenate all the entries +in changes to make a draft changelog, and clear the directory. We'll then edit the draft changelog into a nice readable format. What needs a changes file?:: @@ -405,4 +405,3 @@ function should mention that it does that something in the documentation. If you rely on a function doing something beyond what is in its documentation, then you should watch out, or it might do something else later. -

1 0

[tor/master] Merge remote branch 'sebastian/bug1035' into maint-0.2.2
by nickm＠torproject.org 08 Mar '11

08 Mar '11

commit 0d78a16c3642f7538266e007da79c39860aac332 Merge: 9c72324 5a4f7fa Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 8 15:52:43 2011 -0500 Merge remote branch 'sebastian/bug1035' into maint-0.2.2 changes/bug1035 | 13 +++++++++ src/or/connection_or.c | 3 -- src/or/dirserv.c | 18 +++++++++--- src/or/rephist.c | 65 ++++++++++++++++++++++++++++++++++++++++++------ src/or/rephist.h | 3 +- 5 files changed, 85 insertions(+), 17 deletions(-) diff --combined src/or/connection_or.c index b93699c,6279c93..2d24444 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -370,11 -370,11 +370,11 @@@ connection_or_update_token_buckets_help * bandwidth parameters in the consensus, but allow local config * options to override. */ rate = options->PerConnBWRate ? (int)options->PerConnBWRate : - (int)networkstatus_get_param(NULL, "perconnbwrate", - (int)options->BandwidthRate); + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : - (int)networkstatus_get_param(NULL, "perconnbwburst", - (int)options->BandwidthBurst); + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); } conn->bandwidthrate = rate; @@@ -1098,9 -1098,6 +1098,6 @@@ connection_or_check_valid_tls_handshake as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@@ -1259,8 -1256,7 +1256,8 @@@ connection_or_write_var_cell_to_buf(con tor_assert(conn); var_cell_pack_header(cell, hdr); connection_write_to_buf(hdr, sizeof(hdr), TO_CONN(conn)); - connection_write_to_buf(cell->payload, cell->payload_len, TO_CONN(conn)); + connection_write_to_buf((char*)cell->payload, + cell->payload_len, TO_CONN(conn)); if (cell->command != CELL_PADDING) conn->timestamp_last_added_nonpadding = approx_time(); } @@@ -1389,7 -1385,7 +1386,7 @@@ connection_or_send_netinfo(or_connectio time_t now = time(NULL); routerinfo_t *me; int len; - char *out; + uint8_t *out; memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; diff --combined src/or/dirserv.c index 3c15c59,f426881..aeeab45 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@@ -1,6 -1,6 +1,6 @@@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@@ -386,19 -386,13 +386,19 @@@ dirserv_get_status_impl(const char *id_ strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, - * and there aren't any critical relay-side vulnerabilities. Once more - * of them die off, we should raise this minimum to 0.2.0.x. */ - if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@@ -1166,7 -1160,7 +1166,7 @@@ directory_fetches_from_authorities(or_o return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - refuseunknown = router_my_exit_policy_is_reject_star() && + refuseunknown = ! router_my_exit_policy_is_reject_star() && should_refuse_unknown_exits(options); if (options->DirPort == 0 && !refuseunknown) return 0; @@@ -2970,8 -2964,6 +2970,8 @@@ dirserv_get_routerdesc_fingerprints(sma SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, smartlist_add(fps_out, tor_memdup(r->cache_info.identity_digest, DIGEST_LEN))); + /* Treat "all" requests as if they were unencrypted */ + for_unencrypted_conn = 1; } else if (!strcmp(key, "authority")) { routerinfo_t *ri = router_get_my_routerinfo(); if (ri) @@@ -3116,19 -3108,27 +3116,27 @@@ dirserv_orconn_tls_done(const char *add tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ diff --combined src/or/rephist.c index 61ae2c3,207eb88..53214d6 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@@ -1,5 -1,5 +1,5 @@@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -14,6 -14,7 +14,7 @@@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" + #include "networkstatus.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@@ -73,6 -74,13 +74,13 @@@ typedef struct or_history_t /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@@ -119,6 -127,7 +127,7 @@@ get_or_history(const char* id rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@@ -289,13 -298,20 +298,20 @@@ rep_hist_note_connection_died(const cha /** We have just decided that this router with identity digest id is * reachable, meaning we will give it a "Running" flag for the next while. */ void - rep_hist_note_router_reachable(const char *id, time_t when) + rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@@ -315,6 -331,27 +331,27 @@@ down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@@ -324,6 -361,10 +361,10 @@@ log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@@ -344,12 -385,20 +385,20 @@@ rep_hist_note_router_unreachable(const long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; - + if (run_length < 0) { + unsigned long penalty = -run_length; + #define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@@ -422,7 -471,7 +471,7 @@@ rep_hist_downrate_old_runs(time_t now static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@@ -458,8 -507,8 +507,8 @@@ get_total_weighted_time(or_history_t *h static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@@ -1165,8 -1214,6 +1214,8 @@@ rep_hist_load_mtbf_data(time_t now * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@@ -1261,12 -1308,8 +1310,12 @@@ add_obs(bw_array_t *b, time_t when, uin /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@@ -1297,15 -1340,10 +1346,15 @@@ static bw_array_t *dir_read_array = NUL directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@@ -1403,7 -1441,7 +1452,7 @@@ rep_hist_bandwidth_assess(void * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@@ -1495,181 -1533,163 +1544,181 @@@ rep_hist_get_bandwidth_lines(void return buf; } +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + /** Update state with the newest bandwidth history. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values = NULL; - time_t *s_begins = NULL; - int *s_interval = NULL; - bw_array_t *b = NULL; +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = &state->BWHistoryWriteEnds; - s_interval = &state->BWHistoryWriteInterval; - s_values = &state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = &state->BWHistoryReadEnds; - s_interval = &state->BWHistoryReadInterval; - s_values = &state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = &state->BWHistoryDirWriteEnds; - s_interval = &state->BWHistoryDirWriteInterval; - s_values = &state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = &state->BWHistoryDirReadEnds; - s_interval = &state->BWHistoryDirReadInterval; - s_values = &state->BWHistoryDirReadValues; - break; - } - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins = 0, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval = 0; - smartlist_t *s_values = NULL; - bw_array_t *b = NULL; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = state->BWHistoryWriteEnds; - s_interval = state->BWHistoryWriteInterval; - s_values = state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = state->BWHistoryReadEnds; - s_interval = state->BWHistoryReadInterval; - s_values = state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = state->BWHistoryDirWriteEnds; - s_interval = state->BWHistoryDirWriteInterval; - s_values = state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = state->BWHistoryDirReadEnds; - s_interval = state->BWHistoryDirReadInterval; - s_values = state->BWHistoryDirReadValues; - break; - } - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += s_interval; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} +/** Set bandwidth history from our saved state. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@@ -1972,8 -1992,9 +2021,8 @@@ dump_pk_ops(int severity #define EXIT_STATS_ROUND_UP_STREAMS 4 /** Number of TCP ports */ #define EXIT_STATS_NUM_PORTS 65536 -/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to - * see in order to be included in exit stats. */ -#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 /* The following data structures are arrays and no fancy smartlists or maps, * so that all write operations can be done in constant time. This comes at @@@ -2023,24 -2044,15 +2072,24 @@@ rep_hist_exit_stats_term(void tor_free(exit_streams); } +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) +{ + return (*(int*)x - *(int*)y); +} + /** Return a newly allocated string containing the exit port statistics * until now, or NULL if we're not collecting exit stats. */ char * rep_hist_format_exit_stats(time_t now) { - int i; - uint64_t total_bytes = 0, threshold_bytes, other_read = 0, - other_written = 0; - uint32_t other_streams = 0; + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; char *buf; smartlist_t *written_strings, *read_strings, *streams_strings; char *written_string, *read_string, *streams_string; @@@ -2050,101 -2062,52 +2099,101 @@@ if (!start_of_exit_stats_interval) return NULL; /* Not initialized. */ - /* Count total number of bytes, so that we can attribute observations - * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL - * of all bytes to a special port 'other'. */ + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - total_bytes += exit_bytes_read[i]; - total_bytes += exit_bytes_written[i]; + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } } - threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; - /* Add observations of all ports above the threshold to smartlists and - * join them to single strings. Also count bytes and streams of ports - * below or equal to the threshold. */ + /* Add observations of top ports to smartlists. */ written_strings = smartlist_create(); read_strings = smartlist_create(); streams_strings = smartlist_create(); - for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { - if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { - if (exit_bytes_written[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(written_strings, buf); - } - if (exit_bytes_read[i] > 0) { - uint64_t num = round_uint64_to_next_multiple_of( - exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); - num /= 1024; - buf = NULL; - tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); - smartlist_add(read_strings, buf); - } - if (exit_streams[i] > 0) { - uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], - EXIT_STATS_ROUND_UP_STREAMS); - buf = NULL; - tor_asprintf(&buf, "%d=%u", i, num); - smartlist_add(streams_strings, buf); - } - } else { - other_read += exit_bytes_read[i]; - other_written += exit_bytes_written[i]; - other_streams += exit_streams[i]; + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; } } + + /* Add observations of other ports in a single element. */ other_written = round_uint64_to_next_multiple_of(other_written, EXIT_STATS_ROUND_UP_BYTES); other_written /= 1024; @@@ -2162,8 -2125,6 +2211,8 @@@ buf = NULL; tor_asprintf(&buf, "other=%u", other_streams); smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ written_string = smartlist_join_strings(written_strings, ",", 0, NULL); read_string = smartlist_join_strings(read_strings, ",", 0, NULL); streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); diff --combined src/or/rephist.h index 9a39070,610c1a0..5f6b9f9 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@@ -1,7 -1,7 +1,7 @@@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ + * Copyright (c) 2007-2011, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@@ -33,7 -33,8 +33,8 @@@ void rep_hist_update_state(or_state_t * int rep_hist_load_state(or_state_t *state, char **err); void rep_history_clean(time_t before); - void rep_hist_note_router_reachable(const char *id, time_t when); + void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); void rep_hist_note_router_unreachable(const char *id, time_t when); int rep_hist_record_mtbf_data(time_t now, int missing_means_down); int rep_hist_load_mtbf_data(time_t now);

1 0