February 2011 - tor-commits - lists.torproject.org

[tor/master] Merge remote branch 'origin/maint-0.2.2'
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit a2a8adeee5a3176c3173a85c903c72c5c03ac729 Merge: 0a6beb9 aa178aa Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 25 11:24:35 2011 -0500 Merge remote branch 'origin/maint-0.2.2' changes/1863_bwhist | 16 +++ src/or/config.c | 4 + src/or/or.h | 8 ++- src/or/rephist.c | 295 +++++++++++++++++++++++++++----------------------- 4 files changed, 186 insertions(+), 137 deletions(-) diff --combined src/or/config.c index aa4d974,4d2c872..769831b --- a/src/or/config.c +++ b/src/or/config.c @@@ -44,8 -44,6 +44,8 @@@ typedef enum config_type_t CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ CONFIG_TYPE_UINT, /**< A non-negative integer less than MAX_INT */ CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ + CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional + * units */ CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ CONFIG_TYPE_DOUBLE, /**< A floating-point value */ CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ @@@ -201,7 -199,6 +201,7 @@@ static config_var_t _option_vars[] = V(ClientOnly, BOOL, "0"), V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), + V(ConnDirectionStatistics, BOOL, "0"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), V(ContactInfo, STRING, NULL), @@@ -225,10 -222,9 +225,10 @@@ OBSOLETE("DirRecordUsageGranularity"), OBSOLETE("DirRecordUsageRetainIPs"), OBSOLETE("DirRecordUsageSaveInterval"), - V(DirReqStatistics, BOOL, "0"), + V(DirReqStatistics, BOOL, "1"), VAR("DirServer", LINELIST, DirServers, NULL), V(DisableAllSwap, BOOL, "0"), + V(DisableIOCP, BOOL, "1"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), @@@ -243,7 -239,7 +243,7 @@@ V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPortStatistics, BOOL, "0"), - V(ExtraInfoStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "1"), #if defined (WINCE) V(FallbackNetworkstatusFile, FILENAME, "fallback-consensus"), @@@ -267,7 -263,6 +267,7 @@@ #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(HeartbeatPeriod, INTERVAL, "6 hours"), V(AccelName, STRING, NULL), V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), @@@ -297,7 -292,6 +297,7 @@@ OBSOLETE("LinkPadding"), OBSOLETE("LogLevel"), OBSOLETE("LogFile"), + V(LogTimeGranularity, MSEC_INTERVAL, "1 second"), V(LongLivedPorts, CSV, "21,22,706,1863,5050,5190,5222,5223,6667,6697,8300"), VAR("MapAddress", LINELIST, AddressMap, NULL), @@@ -314,7 -308,7 +314,7 @@@ V(WarnUnsafeSocks, BOOL, "1"), OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), - V(NumCPUs, UINT, "1"), + V(NumCPUs, UINT, "0"), V(NumEntryGuards, UINT, "3"), V(ORListenAddress, LINELIST, NULL), V(ORPort, UINT, "0"), @@@ -324,8 -318,6 +324,8 @@@ V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), + V(PortForwarding, BOOL, "0"), + V(PortForwardingHelper, FILENAME, "tor-fw-helper"), V(PreferTunneledDirConns, BOOL, "1"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), @@@ -394,7 -386,6 +394,7 @@@ VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), + V(_UseFilteringSSLBufferevents, BOOL, "0"), VAR("__ReloadTorrcOnSIGHUP", BOOL, ReloadTorrcOnSIGHUP, "1"), VAR("__AllDirActionsPrivate", BOOL, AllDirActionsPrivate, "0"), VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"), @@@ -455,15 -446,19 +455,19 @@@ static config_var_t _state_vars[] = V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), V(BWHistoryDirReadInterval, UINT, "900"), V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), V(BWHistoryDirWriteEnds, ISOTIME, NULL), V(BWHistoryDirWriteInterval, UINT, "900"), V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), @@@ -567,9 -562,8 +571,9 @@@ static int is_listening_on_low_port(uin const config_line_t *listen_options); static uint64_t config_parse_memunit(const char *s, int *ok); +static int config_parse_msec_interval(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); -static void init_libevent(void); +static void init_libevent(const or_options_t *options); static int opt_streq(const char *s1, const char *s2); /** Magic value for or_options_t. */ @@@ -703,11 -697,6 +707,11 @@@ or_options_free(or_options_t *options return; routerset_free(options->_ExcludeExitNodesUnion); + if (options->NodeFamilySets) { + SMARTLIST_FOREACH(options->NodeFamilySets, routerset_t *, + rs, routerset_free(rs)); + smartlist_free(options->NodeFamilySets); + } config_free(&options_format, options); } @@@ -975,7 -964,7 +979,7 @@@ options_act_reversible(or_options_t *ol /* Set up libevent. (We need to do this before we can register the * listeners as listeners.) */ if (running_tor && !libevent_initialized) { - init_libevent(); + init_libevent(options); libevent_initialized = 1; } @@@ -1251,17 -1240,6 +1255,17 @@@ options_act(or_options_t *old_options if (accounting_is_enabled(options)) configure_accounting(time(NULL)); +#ifdef USE_BUFFEREVENTS + /* If we're using the bufferevents implementation and our rate limits + * changed, we need to tell the rate-limiting system about it. */ + if (!old_options || + old_options->BandwidthRate != options->BandwidthRate || + old_options->BandwidthBurst != options->BandwidthBurst || + old_options->RelayBandwidthRate != options->RelayBandwidthRate || + old_options->RelayBandwidthBurst != options->RelayBandwidthBurst) + connection_bucket_init(); +#endif + /* parse RefuseUnknownExits tristate */ if (!strcmp(options->RefuseUnknownExits, "0")) options->RefuseUnknownExits_ = 0; @@@ -1373,54 -1351,44 +1377,54 @@@ tor_free(actual_fname); } - if (options->DirReqStatistics && !geoip_is_loaded()) { - /* Check if GeoIP database could be loaded. */ - log_warn(LD_CONFIG, "Configured to measure directory request " - "statistics, but no GeoIP database found!"); - return -1; - } - - if (options->EntryStatistics) { - if (should_record_bridge_info(options)) { - /* Don't allow measuring statistics on entry guards when configured - * as bridge. */ - log_warn(LD_CONFIG, "Bridges cannot be configured to measure " - "additional GeoIP statistics as entry guards."); - return -1; - } else if (!geoip_is_loaded()) { - /* Check if GeoIP database could be loaded. */ - log_warn(LD_CONFIG, "Configured to measure entry node statistics, " - "but no GeoIP database found!"); - return -1; - } - } - if (options->CellStatistics || options->DirReqStatistics || - options->EntryStatistics || options->ExitPortStatistics) { + options->EntryStatistics || options->ExitPortStatistics || + options->ConnDirectionStatistics) { time_t now = time(NULL); + int print_notice = 0; if ((!old_options || !old_options->CellStatistics) && - options->CellStatistics) + options->CellStatistics) { rep_hist_buffer_stats_init(now); + print_notice = 1; + } if ((!old_options || !old_options->DirReqStatistics) && - options->DirReqStatistics) - geoip_dirreq_stats_init(now); + options->DirReqStatistics) { + if (geoip_is_loaded()) { + geoip_dirreq_stats_init(now); + print_notice = 1; + } else { + options->DirReqStatistics = 0; + /* Don't warn Tor clients, they don't use statistics */ + if (options->ORPort) + log_notice(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found. " + "Please specify a GeoIP database using the " + "GeoIPFile option."); + } + } if ((!old_options || !old_options->EntryStatistics) && - options->EntryStatistics) - geoip_entry_stats_init(now); + options->EntryStatistics && !should_record_bridge_info(options)) { + if (geoip_is_loaded()) { + geoip_entry_stats_init(now); + print_notice = 1; + } else { + options->EntryStatistics = 0; + log_notice(LD_CONFIG, "Configured to measure entry node " + "statistics, but no GeoIP database found. " + "Please specify a GeoIP database using the " + "GeoIPFile option."); + } + } if ((!old_options || !old_options->ExitPortStatistics) && - options->ExitPortStatistics) + options->ExitPortStatistics) { rep_hist_exit_stats_init(now); - if (!old_options) + print_notice = 1; + } + if ((!old_options || !old_options->ConnDirectionStatistics) && + options->ConnDirectionStatistics) { + rep_hist_conn_stats_init(now); + } + if (print_notice) log_notice(LD_CONFIG, "Configured to measure statistics. Look for " "the *-stats files that will first be written to the " "data directory in 24 hours from now."); @@@ -1438,9 -1406,6 +1442,9 @@@ if (old_options && old_options->ExitPortStatistics && !options->ExitPortStatistics) rep_hist_exit_stats_term(); + if (old_options && old_options->ConnDirectionStatistics && + !options->ConnDirectionStatistics) + rep_hist_conn_stats_term(); /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && @@@ -1746,18 -1711,6 +1750,18 @@@ config_assign_value(config_format_t *fm break; } + case CONFIG_TYPE_MSEC_INTERVAL: { + i = config_parse_msec_interval(c->value, &ok); + if (!ok) { + tor_asprintf(msg, + "Msec interval '%s %s' is malformed or out of bounds.", + c->key, c->value); + return -1; + } + *(int *)lvalue = i; + break; + } + case CONFIG_TYPE_MEMUNIT: { uint64_t u64 = config_parse_memunit(c->value, &ok); if (!ok) { @@@ -2045,7 -1998,6 +2049,7 @@@ get_assigned_option(config_format_t *fm escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_INTERVAL: + case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_UINT: /* This means every or_options_t uint or bool element * needs to be an int. Not, say, a uint16_t or char. */ @@@ -2273,7 -2225,6 +2277,7 @@@ option_clear(config_format_t *fmt, or_o *(time_t*)lvalue = 0; break; case CONFIG_TYPE_INTERVAL: + case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_UINT: case CONFIG_TYPE_BOOL: *(int*)lvalue = 0; @@@ -2380,7 -2331,7 +2384,7 @@@ resolve_my_address(int warn_severity, o int explicit_ip=1; int explicit_hostname=1; int from_interface=0; - char tmpbuf[INET_NTOA_BUF_LEN]; + char *addr_string = NULL; const char *address = options->Address; int notice_severity = warn_severity <= LOG_NOTICE ? LOG_NOTICE : warn_severity; @@@ -2422,43 -2373,48 +2426,43 @@@ return -1; } from_interface = 1; - in.s_addr = htonl(interface_ip); - tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf)); + addr = interface_ip; log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for " - "local interface. Using that.", tmpbuf); + "local interface. Using that.", fmt_addr32(addr)); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); } else { /* resolved hostname into addr */ - in.s_addr = htonl(addr); - if (!explicit_hostname && - is_internal_IP(ntohl(in.s_addr), 0)) { + is_internal_IP(addr, 0)) { uint32_t interface_ip; - tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf)); log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' " - "resolves to a private IP address (%s). Trying something " - "else.", hostname, tmpbuf); + "resolves to a private IP address (%s). Trying something " + "else.", hostname, fmt_addr32(addr)); if (get_interface_address(warn_severity, &interface_ip)) { log_fn(warn_severity, LD_CONFIG, "Could not get local interface IP address. Too bad."); } else if (is_internal_IP(interface_ip, 0)) { - struct in_addr in2; - in2.s_addr = htonl(interface_ip); - tor_inet_ntoa(&in2,tmpbuf,sizeof(tmpbuf)); log_fn(notice_severity, LD_CONFIG, "Interface IP address '%s' is a private address too. " - "Ignoring.", tmpbuf); + "Ignoring.", fmt_addr32(interface_ip)); } else { from_interface = 1; - in.s_addr = htonl(interface_ip); - tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf)); + addr = interface_ip; log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for local interface." - " Using that.", tmpbuf); + " Using that.", fmt_addr32(addr)); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); } } } + } else { + addr = ntohl(in.s_addr); /* set addr so that addr_string is not + * illformed */ } - tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf)); - if (is_internal_IP(ntohl(in.s_addr), 0)) { + addr_string = tor_dup_ip(addr); + if (is_internal_IP(addr, 0)) { /* make sure we're ok with publishing an internal IP */ if (!options->DirServers && !options->AlternateDirAuthority) { /* if they are using the default dirservers, disallow internal IPs @@@ -2466,8 -2422,7 +2470,8 @@@ log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private IP address '%s'. " "Tor servers that use the default DirServers must have public " - "IP addresses.", hostname, tmpbuf); + "IP addresses.", hostname, addr_string); + tor_free(addr_string); return -1; } if (!explicit_ip) { @@@ -2475,20 -2430,19 +2479,20 @@@ * they're using an internal address. */ log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private " "IP address '%s'. Please set the Address config option to be " - "the IP address you want to use.", hostname, tmpbuf); + "the IP address you want to use.", hostname, addr_string); + tor_free(addr_string); return -1; } } - log_debug(LD_CONFIG, "Resolved Address to '%s'.", tmpbuf); - *addr_out = ntohl(in.s_addr); + log_debug(LD_CONFIG, "Resolved Address to '%s'.", fmt_addr32(addr)); + *addr_out = addr; if (last_resolved_addr && last_resolved_addr != *addr_out) { /* Leave this as a notice, regardless of the requested severity, * at least until dynamic IP address support becomes bulletproof. */ log_notice(LD_NET, "Your IP address seems to have changed to %s. Updating.", - tmpbuf); + addr_string); ip_address_changed(0); } if (last_resolved_addr != *addr_out) { @@@ -2507,12 -2461,11 +2511,12 @@@ } control_event_server_status(LOG_NOTICE, "EXTERNAL_ADDRESS ADDRESS=%s METHOD=%s %s%s", - tmpbuf, method, h?"HOSTNAME=":"", h); + addr_string, method, h?"HOSTNAME=":"", h); } last_resolved_addr = *addr_out; if (hostname_out) *hostname_out = tor_strdup(hostname); + tor_free(addr_string); return 0; } @@@ -2919,10 -2872,6 +2923,10 @@@ compute_publishserverdescriptor(or_opti * will generate too many circuits and potentially overload the network. */ #define MIN_CIRCUIT_STREAM_TIMEOUT 10 +/** Lowest allowable value for HeartbeatPeriod; if this is too low, we might + * expose more information than we're comfortable with. */ +#define MIN_HEARTBEAT_PERIOD (30*60) + /** Return 0 if every setting in options is reasonable, and a * permissible transition from old_options. Else return -1. * Should have no side effects, except for normalizing the contents of @@@ -3123,24 -3072,17 +3127,24 @@@ options_validate(or_options_t *old_opti routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } + if (options->NodeFamilies) { + options->NodeFamilySets = smartlist_create(); + for (cl = options->NodeFamilies; cl; cl = cl->next) { + routerset_t *rs = routerset_new(); + if (routerset_parse(rs, cl->value, cl->key) == 0) { + smartlist_add(options->NodeFamilySets, rs); + } else { + routerset_free(rs); + } + } + } + if (options->ExcludeNodes && options->StrictNodes) { COMPLAIN("You have asked to exclude certain relays from all positions " "in your circuits. Expect hidden services and other Tor " "features to be broken in unpredictable ways."); } - if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { - /* XXXX fix this; see entry_guards_prepend_from_config(). */ - REJECT("IPs or countries are not yet supported in EntryNodes."); - } - if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); @@@ -3385,13 -3327,6 +3389,13 @@@ options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; } + if (options->HeartbeatPeriod && + options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) { + log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; " + "raising to %d seconds.", MIN_HEARTBEAT_PERIOD); + options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@@ -3610,12 -3545,8 +3614,12 @@@ if (check_nickname_list(options->MyFamily, "MyFamily", msg)) return -1; for (cl = options->NodeFamilies; cl; cl = cl->next) { - if (check_nickname_list(cl->value, "NodeFamily", msg)) + routerset_t *rs = routerset_new(); + if (routerset_parse(rs, cl->value, cl->key)) { + routerset_free(rs); return -1; + } + routerset_free(rs); } if (validate_addr_policies(options, msg) < 0) @@@ -4109,8 -4040,6 +4113,8 @@@ load_torrc_from_disk(int argc, char **a "Unable to open configuration file \"%s\".", fname); goto err; } + } else { + log(LOG_NOTICE, LD_CONFIG, "Read configuration file \"%s\".", fname); } return cf; @@@ -4399,35 -4328,6 +4403,35 @@@ options_init_logs(or_options_t *options options->RunAsDaemon; #endif + if (options->LogTimeGranularity <= 0) { + log_warn(LD_CONFIG, "Log time granularity '%d' has to be positive.", + options->LogTimeGranularity); + return -1; + } else if (1000 % options->LogTimeGranularity != 0 && + options->LogTimeGranularity % 1000 != 0) { + int granularity = options->LogTimeGranularity; + if (granularity < 40) { + do granularity++; + while (1000 % granularity != 0); + } else if (granularity < 1000) { + granularity = 1000 / granularity; + while (1000 % granularity != 0) + granularity--; + granularity = 1000 / granularity; + } else { + granularity = 1000 * ((granularity / 1000) + 1); + } + log_warn(LD_CONFIG, "Log time granularity '%d' has to be either a " + "divisor or a multiple of 1 second. Changing to " + "'%d'.", + options->LogTimeGranularity, granularity); + if (!validate_only) + set_log_time_granularity(granularity); + } else { + if (!validate_only) + set_log_time_granularity(options->LogTimeGranularity); + } + ok = 1; elts = smartlist_create(); @@@ -4919,26 -4819,6 +4923,26 @@@ static struct unit_table_t time_units[ { NULL, 0 }, }; +/** Table to map the names of time units to the number of milliseconds + * they contain. */ +static struct unit_table_t time_msec_units[] = { + { "", 1 }, + { "msec", 1 }, + { "millisecond", 1 }, + { "milliseconds", 1 }, + { "second", 1000 }, + { "seconds", 1000 }, + { "minute", 60*1000 }, + { "minutes", 60*1000 }, + { "hour", 60*60*1000 }, + { "hours", 60*60*1000 }, + { "day", 24*60*60*1000 }, + { "days", 24*60*60*1000 }, + { "week", 7*24*60*60*1000 }, + { "weeks", 7*24*60*60*1000 }, + { NULL, 0 }, +}; + /** Parse a string val containing a number, zero or more * spaces, and an optional unit string. If the unit appears in the * table u, then multiply the number by the unit multiplier. @@@ -5002,25 -4882,6 +5006,25 @@@ config_parse_memunit(const char *s, in return u; } +/** Parse a string in the format "number unit", where unit is a unit of + * time in milliseconds. On success, set *ok to true and return + * the number of milliseconds in the provided interval. Otherwise, set + * *ok to 0 and return -1. */ +static int +config_parse_msec_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_msec_units, ok); + if (!ok) + return -1; + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} + /** Parse a string in the format "number unit", where unit is a unit of time. * On success, set *ok to true and return the number of seconds in * the provided interval. Otherwise, set *ok to 0 and return -1. @@@ -5040,29 -4901,13 +5044,29 @@@ config_parse_interval(const char *s, in return (int)r; } +/** Return the number of cpus configured in options. If we are + * told to auto-detect the number of cpus, return the auto-detected number. */ +int +get_num_cpus(const or_options_t *options) +{ + if (options->NumCPUs == 0) { + int n = compute_num_cpus(); + return (n >= 1) ? n : 1; + } else { + return options->NumCPUs; + } +} + /** * Initialize the libevent library. */ static void -init_libevent(void) +init_libevent(const or_options_t *options) { const char *badness=NULL; + tor_libevent_cfg cfg; + + tor_assert(options); configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't @@@ -5072,11 -4917,7 +5076,11 @@@ tor_check_libevent_header_compatibility(); - tor_libevent_initialize(); + memset(&cfg, 0, sizeof(cfg)); + cfg.disable_iocp = options->DisableIOCP; + cfg.num_cpus = get_num_cpus(options); + + tor_libevent_initialize(&cfg); suppress_libevent_log_msg(NULL); @@@ -5417,7 -5258,6 +5421,7 @@@ getinfo_helper_config(control_connectio case CONFIG_TYPE_FILENAME: type = "Filename"; break; case CONFIG_TYPE_UINT: type = "Integer"; break; case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break; + case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break; case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break; case CONFIG_TYPE_DOUBLE: type = "Float"; break; case CONFIG_TYPE_BOOL: type = "Boolean"; break; diff --combined src/or/or.h index 1617d94,74ff743..e033a3e --- a/src/or/or.h +++ b/src/or/or.h @@@ -83,13 -83,6 +83,13 @@@ #define snprintf _snprintf #endif +#ifdef USE_BUFFEREVENTS +#include <event2/bufferevent.h> +#include <event2/buffer.h> +#include <event2/util.h> +#endif + +#include "crypto.h" #include "tortls.h" #include "../common/torlog.h" #include "container.h" @@@ -391,9 -384,7 +391,9 @@@ typedef enum /** A connection to a hidden service directory server: download a v2 rendezvous * descriptor. */ #define DIR_PURPOSE_FETCH_RENDDESC_V2 18 -#define _DIR_PURPOSE_MAX 18 +/** A connection to a directory server: download a microdescriptor. */ +#define DIR_PURPOSE_FETCH_MICRODESC 19 +#define _DIR_PURPOSE_MAX 19 /** True iff p is a purpose corresponding to uploading data to a * directory server. */ @@@ -814,9 -805,6 +814,9 @@@ typedef enum * Tor 0.1.2.x is obsolete, we can remove this. */ #define DEFAULT_CLIENT_NICKNAME "client" +/** Name chosen by routers that don't configure nicknames */ +#define UNNAMED_ROUTER_NICKNAME "Unnamed" + /** Number of bytes in a SOCKS4 header. */ #define SOCKS4_NETWORK_LEN 8 @@@ -862,8 -850,8 +862,8 @@@ typedef struct cell_t typedef struct var_cell_t { uint8_t command; circid_t circ_id; - uint16_t payload_len; - uint8_t payload[1]; + uint16_t payload_len; /**< The actual length of payload. */ + uint8_t payload[FLEXIBLE_ARRAY_MEMBER]; } var_cell_t; /** A cell as packed for writing to the network. */ @@@ -980,7 -968,6 +980,7 @@@ typedef struct connection_t /** Our socket; -1 if this connection is closed, or has no socket. */ evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ + struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ buf_t *inbuf; /**< Buffer holding data read over this connection. */ @@@ -991,11 -978,6 +991,11 @@@ * read? */ time_t timestamp_lastwritten; /**< When was the last time libevent said we * could write? */ + +#ifdef USE_BUFFEREVENTS + struct bufferevent *bufev; /**< A Libevent buffered IO structure. */ +#endif + time_t timestamp_created; /**< When was this connection_t created? */ /* XXXX_IP6 make this IPv6-capable */ @@@ -1095,16 -1077,10 +1095,16 @@@ typedef struct or_connection_t /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ +#ifndef USE_BUFFEREVENTS int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ +#else + /** DOCDOC */ + /* XXXX we could share this among all connections. */ + struct ev_token_bucket_cfg *bucket_cfg; +#endif int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@@ -1212,13 -1188,8 +1212,13 @@@ typedef struct edge_connection_t typedef struct dir_connection_t { connection_t _base; - char *requested_resource; /**< Which 'resource' did we ask the directory - * for? */ + /** Which 'resource' did we ask the directory for? This is typically the part + * of the URL string that defines, relative to the directory conn purpose, + * what thing we want. For example, in router descriptor downloads by + * descriptor digest, it contains "d/", then one ore more +-separated + * fingerprints. + **/ + char *requested_resource; unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */ /* Used only for server sides of some dir connections, to implement @@@ -1309,51 -1280,6 +1309,51 @@@ static INLINE control_connection_t *TO_ return DOWNCAST(control_connection_t, c); } +/* Conditional macros to help write code that works whether bufferevents are + disabled or not. + + We can't just write: + if (conn->bufev) { + do bufferevent stuff; + } else { + do other stuff; + } + because the bufferevent stuff won't even compile unless we have a fairly + new version of Libevent. Instead, we say: + IF_HAS_BUFFEREVENT(conn, { do_bufferevent_stuff } ); + or: + IF_HAS_BUFFEREVENT(conn, { + do bufferevent stuff; + }) ELSE_IF_NO_BUFFEREVENT { + do non-bufferevent stuff; + } + If we're compiling with bufferevent support, then the macros expand more or + less to: + if (conn->bufev) { + do_bufferevent_stuff; + } else { + do non-bufferevent stuff; + } + and if we aren't using bufferevents, they expand more or less to: + { do non-bufferevent stuff; } +*/ +#ifdef USE_BUFFEREVENTS +#define HAS_BUFFEREVENT(c) (((c)->bufev) != NULL) +#define IF_HAS_BUFFEREVENT(c, stmt) \ + if ((c)->bufev) do { \ + stmt ; \ + } while (0) +#define ELSE_IF_NO_BUFFEREVENT ; else +#define IF_HAS_NO_BUFFEREVENT(c) \ + if (NULL == (c)->bufev) +#else +#define HAS_BUFFEREVENT(c) (0) +#define IF_HAS_BUFFEREVENT(c, stmt) (void)0 +#define ELSE_IF_NO_BUFFEREVENT ; +#define IF_HAS_NO_BUFFEREVENT(c) \ + if (1) +#endif + /** What action type does an address policy indicate: accept or reject? */ typedef enum { ADDR_POLICY_ACCEPT=1, @@@ -1517,49 -1443,59 +1517,49 @@@ typedef struct char *contact_info; /**< Declared contact info for this router. */ unsigned int is_hibernating:1; /**< Whether the router claims to be * hibernating */ - unsigned int has_old_dnsworkers:1; /**< Whether the router is using - * dnsworker code. */ - unsigned int caches_extra_info:1; /**< Whether the router caches and serves - * extrainfo documents. */ - unsigned int allow_single_hop_exits:1; /**< Whether the router allows - * single hop exits. */ - - /* local info */ - unsigned int is_running:1; /**< As far as we know, is this OR currently - * running? */ - unsigned int is_valid:1; /**< Has a trusted dirserver validated this OR? - * (For Authdir: Have we validated this OR?) - */ - unsigned int is_named:1; /**< Do we believe the nickname that this OR gives - * us? */ - unsigned int is_fast:1; /** Do we think this is a fast OR? */ - unsigned int is_stable:1; /** Do we think this is a stable OR? */ - unsigned int is_possible_guard:1; /**< Do we think this is an OK guard? */ - unsigned int is_exit:1; /**< Do we think this is an OK exit? */ - unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked, - * or otherwise nasty? */ - unsigned int is_bad_directory:1; /**< Do we think this directory is junky, - * underpowered, or otherwise useless? */ + unsigned int caches_extra_info:1; /**< Whether the router says it caches and + * serves extrainfo documents. */ + unsigned int allow_single_hop_exits:1; /**< Whether the router says + * it allows single hop exits. */ + unsigned int wants_to_be_hs_dir:1; /**< True iff this router claims to be * a hidden service directory. */ - unsigned int is_hs_dir:1; /**< True iff this router is a hidden service - * directory according to the authorities. */ unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this * router rejects everything. */ /** True if, after we have added this router, we should re-launch * tests for it. */ unsigned int needs_retest_if_added:1; -/** Tor can use this router for general positions in circuits. */ +/** Tor can use this router for general positions in circuits; we got it + * from a directory server as usual, or we're an authority and a server + * uploaded it. */ #define ROUTER_PURPOSE_GENERAL 0 -/** Tor should avoid using this router for circuit-building. */ +/** Tor should avoid using this router for circuit-building: we got it + * from a crontroller. If the controller wants to use it, it'll have to + * ask for it by identity. */ #define ROUTER_PURPOSE_CONTROLLER 1 -/** Tor should use this router only for bridge positions in circuits. */ +/** Tor should use this router only for bridge positions in circuits: we got + * it via a directory request from the bridge itself, or a bridge + * authority. x*/ #define ROUTER_PURPOSE_BRIDGE 2 /** Tor should not use this router; it was marked in cached-descriptors with * a purpose we didn't recognize. */ #define ROUTER_PURPOSE_UNKNOWN 255 - uint8_t purpose; /** What positions in a circuit is this router good for? */ + /* In what way did we find out about this router? One of ROUTER_PURPOSE_*. + * Routers of different purposes are kept segregated and used for different + * things; see notes on ROUTER_PURPOSE_* macros above. + */ + uint8_t purpose; /* The below items are used only by authdirservers for * reachability testing. */ + /** When was the last time we could reach this OR? */ time_t last_reachable; /** When did we start testing reachability for this OR? */ time_t testing_since; - /** According to the geoip db what country is this router in? */ - country_t country; + } routerinfo_t; /** Information needed to keep and cache a signed extra-info document. */ @@@ -1585,9 -1521,8 +1585,9 @@@ typedef struct routerstatus_t * has. */ char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity * key. */ - char descriptor_digest[DIGEST_LEN]; /**< Digest of the router's most recent - * descriptor. */ + /** Digest of the router's most recent descriptor or microdescriptor. + * If it's a descriptor, we only use the first DIGEST_LEN bytes. */ + char descriptor_digest[DIGEST256_LEN]; uint32_t addr; /**< IPv4 address for this router. */ uint16_t or_port; /**< OR port for this router. */ uint16_t dir_port; /**< Directory port for this router. */ @@@ -1595,11 -1530,7 +1595,11 @@@ unsigned int is_exit:1; /**< True iff this router is a good exit. */ unsigned int is_stable:1; /**< True iff this router stays up a long time. */ unsigned int is_fast:1; /**< True iff this router has good bandwidth. */ - unsigned int is_running:1; /**< True iff this router is up. */ + /** True iff this router is called 'running' in the consensus. We give it + * this funny name so that we don't accidentally use this bit as a view of + * whether we think the router is *currently* running. If that's what you + * want to know, look at is_running in node_t. */ + unsigned int is_flagged_running:1; unsigned int is_named:1; /**< True iff "nickname" belongs to this router. */ unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another * router. */ @@@ -1651,31 -1582,15 +1651,31 @@@ * from this authority.) Applies in v2 networkstatus document only. */ unsigned int need_to_mirror:1; - unsigned int name_lookup_warned:1; /**< Have we warned the user for referring - * to this (unnamed) router by nickname? - */ time_t last_dir_503_at; /**< When did this router last tell us that it * was too busy to serve directory info? */ download_status_t dl_status; } routerstatus_t; +/** A single entry in a parsed policy summary, describing a range of ports. */ +typedef struct short_policy_entry_t { + uint16_t min_port, max_port; +} short_policy_entry_t; + +/** A short_poliy_t is the parsed version of a policy summary. */ +typedef struct short_policy_t { + /** True if the members of 'entries' are port ranges to accept; false if + * they are port ranges to reject */ + unsigned int is_accept : 1; + /** The actual number of values in 'entries'. */ + unsigned int n_entries : 31; + /** An array of 0 or more short_policy_entry_t values, each describing a + * range of ports that this policy accepts or rejects (depending on the + * value of is_accept). + */ + short_policy_entry_t entries[FLEXIBLE_ARRAY_MEMBER]; +} short_policy_t; + /** A microdescriptor is the smallest amount of information needed to build a * circuit through a router. They are generated by the directory authorities, * using information from the uploaded routerinfo documents. They are not @@@ -1717,83 -1632,15 +1717,83 @@@ typedef struct microdesc_t crypto_pk_env_t *onion_pkey; /** As routerinfo_t.family */ smartlist_t *family; - /** Encoded exit policy summary */ - char *exitsummary; /**< exit policy summary - - * XXX this probably should not stay a string. */ + /** Exit policy summary */ + short_policy_t *exit_policy; } microdesc_t; +/** A node_t represents a Tor router. + * + * Specifically, a node_t is a Tor router as we are using it: a router that + * we are considering for circuits, connections, and so on. A node_t is a + * thin wrapper around the routerstatus, routerinfo, and microdesc for a + * single wrapper, and provides a consistent interface for all of them. + * + * Also, a node_t has mutable state. While a routerinfo, a routerstatus, + * and a microdesc have[*] only the information read from a router + * descriptor, a consensus entry, and a microdescriptor (respectively)... + * a node_t has flags based on *our own current opinion* of the node. + * + * [*] Actually, there is some leftover information in each that is mutable. + * We should try to excise that. + */ +typedef struct node_t { + /* Indexing information */ + + /** Used to look up the node_t by its identity digest. */ + HT_ENTRY(node_t) ht_ent; + /** Position of the node within the list of nodes */ + int nodelist_idx; + + /** The identity digest of this node_t. No more than one node_t per + * identity may exist at a time. */ + char identity[DIGEST_LEN]; + + microdesc_t *md; + routerinfo_t *ri; + routerstatus_t *rs; + + /* local info: copied from routerstatus, then possibly frobbed based + * on experience. Authorities set this stuff directly. */ + + unsigned int is_running:1; /**< As far as we know, is this OR currently + * running? */ + unsigned int is_valid:1; /**< Has a trusted dirserver validated this OR? + * (For Authdir: Have we validated this OR?) + */ + unsigned int is_fast:1; /** Do we think this is a fast OR? */ + unsigned int is_stable:1; /** Do we think this is a stable OR? */ + unsigned int is_possible_guard:1; /**< Do we think this is an OK guard? */ + unsigned int is_exit:1; /**< Do we think this is an OK exit? */ + unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked, + * or otherwise nasty? */ + unsigned int is_bad_directory:1; /**< Do we think this directory is junky, + * underpowered, or otherwise useless? */ + unsigned int is_hs_dir:1; /**< True iff this router is a hidden service + * directory according to the authorities. */ + + /* Local info: warning state. */ + + unsigned int name_lookup_warned:1; /**< Have we warned the user for referring + * to this (unnamed) router by nickname? + */ + + /** Local info: we treat this node as if it rejects everything */ + unsigned int rejects_all:1; + + /* Local info: derived. */ + + /** According to the geoip db what country is this router in? */ + country_t country; +} node_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 +/** How many times will we try to download a microdescriptor before giving + * up? */ +#define MAX_MICRODESC_DOWNLOAD_FAILURES 8 + /** Contents of a v2 (non-consensus, non-vote) network status object. */ typedef struct networkstatus_v2_t { /** When did we receive the network-status document? */ @@@ -2269,12 -2116,10 +2269,12 @@@ typedef struct circuit_t * length ONIONSKIN_CHALLENGE_LEN. */ char *n_conn_onionskin; - time_t timestamp_created; /**< When was this circuit created? */ + /** When was this circuit created? We keep this timestamp with a higher + * resolution than most so that the circuit-build-time tracking code can + * get millisecond resolution. */ + struct timeval timestamp_created; time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the * circuit is clean. */ - struct timeval highres_created; /**< When exactly was the circuit created? */ uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@@ -2499,7 -2344,6 +2499,7 @@@ typedef struct config_line_t *Logs; /**< New-style list of configuration lines * for logs */ + int LogTimeGranularity; /**< Log resolution in milliseconds. */ int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which * each log message occurs? */ @@@ -2722,9 -2566,6 +2722,9 @@@ * authorizations for hidden services */ char *ContactInfo; /**< Contact info to be published in the directory. */ + int HeartbeatPeriod; /**< Log heartbeat messages after this many seconds + * have passed. */ + char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */ tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ uint16_t HTTPProxyPort; /**< Parsed port for http proxy, if any. */ @@@ -2762,8 -2603,7 +2762,8 @@@ char *MyFamily; /**< Declared family for this OR. */ config_line_t *NodeFamilies; /**< List of config lines for - * node families */ + * node families */ + smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */ config_line_t *AuthDirBadDir; /**< Address policy for descriptors to * mark as bad dir mirrors. */ config_line_t *AuthDirBadExit; /**< Address policy for descriptors to @@@ -2868,10 -2708,6 +2868,10 @@@ * possible. */ int PreferTunneledDirConns; /**< If true, avoid dirservers that don't * support BEGIN_DIR, when possible. */ + int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically + * forward the DirPort and ORPort on the NAT device */ + char *PortForwardingHelper; /** < Filename or full path of the port + forwarding helper executable */ int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames * with weird characters. */ /** If true, we try resolving hostnames with weird characters. */ @@@ -2909,9 -2745,6 +2909,9 @@@ /** If true, the user wants us to collect statistics on port usage. */ int ExitPortStatistics; + /** If true, the user wants us to collect connection statistics. */ + int ConnDirectionStatistics; + /** If true, the user wants us to collect cell statistics. */ int CellStatistics; @@@ -3008,12 -2841,6 +3008,12 @@@ */ double CircuitPriorityHalflife; + /** If true, do not enable IOCP on windows with bufferevents, even if + * we think we could. */ + int DisableIOCP; + /** For testing only: will go away in 0.2.3.x. */ + int _UseFilteringSSLBufferevents; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@@ -3045,19 -2872,25 +3045,25 @@@ typedef struct * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; time_t BWHistoryDirReadEnds; int BWHistoryDirReadInterval; smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; time_t BWHistoryDirWriteEnds; int BWHistoryDirWriteInterval; smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; /** Build time histogram */ config_line_t * BuildtimeHistogram; @@@ -3122,6 -2955,8 +3128,6 @@@ struct socks_request_t * every connection. */ }; -/* all the function prototypes go here */ - /********************************* circuitbuild.c **********************/ /** How many hops does a general-purpose circuit have by default? */ @@@ -3513,7 -3348,7 +3519,7 @@@ typedef enum ADDR_POLICY_PROBABLY_ACCEPTED=1, /** Part of the address was unknown, but as far as we can tell, it was * rejected. */ - ADDR_POLICY_PROBABLY_REJECTED=2 + ADDR_POLICY_PROBABLY_REJECTED=2, } addr_policy_result_t; /********************************* rephist.c ***************************/ @@@ -3644,8 -3479,6 +3650,8 @@@ typedef struct trusted_dir_server_t * fetches to _any_ single directory server.] */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) +#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4) + #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16) /** Possible ways to weight routers when choosing one randomly. See @@@ -3663,8 -3496,7 +3669,8 @@@ typedef enum CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_WEIGHT_AS_EXIT = 1<<5 + CRN_WEIGHT_AS_EXIT = 1<<5, + CRN_NEED_DESC = 1<<6 } router_crn_flags_t; /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ diff --combined src/or/rephist.c index 3146d70,3f4a704..f34c5b6 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@@ -7,14 -7,13 +7,14 @@@ * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've * been using, which ports we tend to want, and so on; further, - * exit port statistics and cell statistics. + * exit port statistics, cell statistics, and connection statistics. **/ #include "or.h" #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "nodelist.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@@ -580,7 -579,7 +580,7 @@@ rep_hist_dump_stats(time_t now, int sev size_t len; int ret; unsigned long upt, downt; - routerinfo_t *r; + const node_t *node; rep_history_clean(now - get_options()->RephistTrackTime); @@@ -594,8 -593,8 +594,8 @@@ digestmap_iter_get(orhist_it, &digest1, &or_history_p); or_history = (or_history_t*) or_history_p; - if ((r = router_get_by_digest(digest1))) - name1 = r->nickname; + if ((node = node_get_by_id(digest1)) && node_get_nickname(node)) + name1 = node_get_nickname(node); else name1 = "(unknown)"; base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN); @@@ -625,8 -624,8 +625,8 @@@ lhist_it = digestmap_iter_next(or_history->link_history_map, lhist_it)) { digestmap_iter_get(lhist_it, &digest2, &link_history_p); - if ((r = router_get_by_digest(digest2))) - name2 = r->nickname; + if ((node = node_get_by_id(digest2)) && node_get_nickname(node)) + name2 = node_get_nickname(node); else name2 = "(unknown)"; @@@ -757,7 -756,7 +757,7 @@@ rep_hist_record_mtbf_data(time_t now, i base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN); if (missing_means_down && hist->start_of_run && - !router_get_by_digest(digest)) { + !router_get_by_id_digest(digest)) { /* We think this relay is running, but it's not listed in our * routerlist. Somehow it fell out without telling us it went * down. Complain and also correct it. */ @@@ -872,32 -871,28 +872,32 @@@ rep_hist_get_router_stability_doc(time_ } DIGESTMAP_FOREACH(history_map, id, or_history_t *, hist) { - routerinfo_t *ri; + const node_t *node; char dbuf[BASE64_DIGEST_LEN+1]; char header_buf[512]; char *info; digest_to_base64(dbuf, id); - ri = router_get_by_digest(id); - if (ri) { - char *ip = tor_dup_ip(ri->addr); + node = node_get_by_id(id); + if (node) { + char ip[INET_NTOA_BUF_LEN+1]; char tbuf[ISO_TIME_LEN+1]; - format_iso_time(tbuf, ri->cache_info.published_on); + time_t published = node_get_published_on(node); + node_get_address_string(node,ip,sizeof(ip)); + if (published > 0) + format_iso_time(tbuf, published); + else + strlcpy(tbuf, "???", sizeof(tbuf)); tor_snprintf(header_buf, sizeof(header_buf), "router %s %s %s\n" "published %s\n" "relevant-flags %s%s%s\n" "declared-uptime %ld\n", - dbuf, ri->nickname, ip, + dbuf, node_get_nickname(node), ip, tbuf, - ri->is_running ? "Running " : "", - ri->is_valid ? "Valid " : "", - ri->is_hibernating ? "Hibernating " : "", - ri->uptime); - tor_free(ip); + node->is_running ? "Running " : "", + node->is_valid ? "Valid " : "", + node->ri && node->ri->is_hibernating ? "Hibernating " : "", + node_get_declared_uptime(node)); } else { tor_snprintf(header_buf, sizeof(header_buf), "router %s {no descriptor}\n", dbuf); @@@ -1170,6 -1165,8 +1170,8 @@@ rep_hist_load_mtbf_data(time_t now * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ + /* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@@ -1264,8 -1261,12 +1266,12 @@@ add_obs(bw_array_t *b, time_t when, uin /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@@ -1296,10 -1297,15 +1302,15 @@@ static bw_array_t *dir_read_array = NUL directory protocol. */ static bw_array_t *dir_write_array = NULL; - /** Set up [dir-]read_array and [dir-]write_array. */ + /** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@@ -1397,7 -1403,7 +1408,7 @@@ rep_hist_bandwidth_assess(void * It returns the number of bytes written. */ static size_t - rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) + rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@@ -1489,163 -1495,175 +1500,175 @@@ rep_hist_get_bandwidth_lines(void return buf; } - /** Update state with the newest bandwidth history. */ - void - rep_hist_update_state(or_state_t *state) + /** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ + static void + rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) { - int len, r; - char *buf, *cp; - smartlist_t **s_values = NULL; - time_t *s_begins = NULL; - int *s_interval = NULL; - bw_array_t *b = NULL; - - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); + char *cp; + int i,j; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = &state->BWHistoryWriteEnds; - s_interval = &state->BWHistoryWriteInterval; - s_values = &state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = &state->BWHistoryReadEnds; - s_interval = &state->BWHistoryReadInterval; - s_values = &state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = &state->BWHistoryDirWriteEnds; - s_interval = &state->BWHistoryDirWriteInterval; - s_values = &state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = &state->BWHistoryDirReadEnds; - s_interval = &state->BWHistoryDirReadInterval; - s_values = &state->BWHistoryDirReadValues; - break; - } - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); + *s_begins = 0; + *s_interval = 900; *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); } - tor_free(buf); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + + /** Update state with the newest bandwidth history. */ + void + rep_hist_update_state(or_state_t *state) + { + #define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) + + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); + if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } + #undef UPDATE } - /** Set bandwidth history from our saved state. */ - int - rep_hist_load_state(or_state_t *state, char **err) + /** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ + static int + rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins = 0, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval = 0; - smartlist_t *s_values = NULL; - bw_array_t *b = NULL; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = state->BWHistoryWriteEnds; - s_interval = state->BWHistoryWriteInterval; - s_values = state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = state->BWHistoryReadEnds; - s_interval = state->BWHistoryReadInterval; - s_values = state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = state->BWHistoryDirWriteEnds; - s_interval = state->BWHistoryDirWriteInterval; - s_values = state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = state->BWHistoryDirReadEnds; - s_interval = state->BWHistoryDirReadInterval; - s_values = state->BWHistoryDirReadValues; - break; - } - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); - if (!ok) { - all_ok=0; + if (have_maxima) { + const char *maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } + if (!ok || !ok_m) { + retval = -1; log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); } + if (start < now) { add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += s_interval; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + return retval; + } + + /** Set bandwidth history from our saved state. */ + int + rep_hist_load_state(or_state_t *state, char **err) + { + int all_ok = 1; + + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + + #define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + + #undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@@ -1653,13 -1671,10 +1676,13 @@@ /*********************************************************************/ +typedef struct predicted_port_t { + uint16_t port; + time_t time; +} predicted_port_t; + /** A list of port numbers that have been used recently. */ static smartlist_t *predicted_ports_list=NULL; -/** The corresponding most recently used time for each port. */ -static smartlist_t *predicted_ports_times=NULL; /** We just got an application request for a connection with * port port. Remember it for the future, so we can keep @@@ -1668,11 -1683,14 +1691,11 @@@ static void add_predicted_port(time_t now, uint16_t port) { - /* XXXX we could just use uintptr_t here, I think. */ - uint16_t *tmp_port = tor_malloc(sizeof(uint16_t)); - time_t *tmp_time = tor_malloc(sizeof(time_t)); - *tmp_port = port; - *tmp_time = now; - rephist_total_alloc += sizeof(uint16_t) + sizeof(time_t); - smartlist_add(predicted_ports_list, tmp_port); - smartlist_add(predicted_ports_times, tmp_time); + predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t)); + pp->port = port; + pp->time = now; + rephist_total_alloc += sizeof(*pp); + smartlist_add(predicted_ports_list, pp); } /** Initialize whatever memory and structs are needed for predicting @@@ -1683,6 -1701,7 +1706,6 @@@ static voi predicted_ports_init(void) { predicted_ports_list = smartlist_create(); - predicted_ports_times = smartlist_create(); add_predicted_port(time(NULL), 80); /* add one to kickstart us */ } @@@ -1692,11 -1711,12 +1715,11 @@@ static void predicted_ports_free(void) { - rephist_total_alloc -= smartlist_len(predicted_ports_list)*sizeof(uint16_t); - SMARTLIST_FOREACH(predicted_ports_list, char *, cp, tor_free(cp)); + rephist_total_alloc -= + smartlist_len(predicted_ports_list)*sizeof(predicted_port_t); + SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *, + pp, tor_free(pp)); smartlist_free(predicted_ports_list); - rephist_total_alloc -= smartlist_len(predicted_ports_times)*sizeof(time_t); - SMARTLIST_FOREACH(predicted_ports_times, char *, cp, tor_free(cp)); - smartlist_free(predicted_ports_times); } /** Remember that port has been asked for as of time now. @@@ -1706,17 -1726,24 +1729,17 @@@ void rep_hist_note_used_port(time_t now, uint16_t port) { - int i; - uint16_t *tmp_port; - time_t *tmp_time; - tor_assert(predicted_ports_list); - tor_assert(predicted_ports_times); if (!port) /* record nothing */ return; - for (i = 0; i < smartlist_len(predicted_ports_list); ++i) { - tmp_port = smartlist_get(predicted_ports_list, i); - tmp_time = smartlist_get(predicted_ports_times, i); - if (*tmp_port == port) { - *tmp_time = now; + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (pp->port == port) { + pp->time = now; return; } - } + } SMARTLIST_FOREACH_END(pp); /* it's not there yet; we need to add it */ add_predicted_port(now, port); } @@@ -1725,28 -1752,36 +1748,28 @@@ * we'll want to make connections to the same port in the future. */ #define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) -/** Return a pointer to the list of port numbers that +/** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. - * - * The caller promises not to mess with it. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { - int i; - uint16_t *tmp_port; - time_t *tmp_time; - + smartlist_t *out = smartlist_create(); tor_assert(predicted_ports_list); - tor_assert(predicted_ports_times); /* clean out obsolete entries */ - for (i = 0; i < smartlist_len(predicted_ports_list); ++i) { - tmp_time = smartlist_get(predicted_ports_times, i); - if (*tmp_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { - tmp_port = smartlist_get(predicted_ports_list, i); - log_debug(LD_CIRC, "Expiring predicted port %d", *tmp_port); - smartlist_del(predicted_ports_list, i); - smartlist_del(predicted_ports_times, i); - rephist_total_alloc -= sizeof(uint16_t)+sizeof(time_t); - tor_free(tmp_port); - tor_free(tmp_time); - i--; + SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { + if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); + + rephist_total_alloc -= sizeof(predicted_port_t); + tor_free(pp); + SMARTLIST_DEL_CURRENT(predicted_ports_list, pp); + } else { + smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t))); } - } - return predicted_ports_list; + } SMARTLIST_FOREACH_END(pp); + return out; } /** The user asked us to do a resolve. Rather than keeping track of @@@ -1982,9 -2017,7 +2005,9 @@@ rep_hist_exit_stats_term(void tor_free(exit_streams); } -/** Helper for qsort: compare two ints. */ +/** Helper for qsort: compare two ints. Does not handle overflow properly, + * but works fine for sorting an array of port numbers, which is what we use + * it for. */ static int _compare_int(const void *x, const void *y) { @@@ -2232,6 -2265,7 +2255,6 @@@ typedef struct circ_buffer_stats_t uint32_t processed_cells; double mean_num_cells_in_queue; double mean_time_cells_in_queue; - uint32_t local_circ_id; } circ_buffer_stats_t; /** Holds stats. */ @@@ -2254,9 -2288,9 +2277,9 @@@ rep_hist_buffer_stats_add_circ(circuit_ return; if (!circuits_for_buffer_stats) circuits_for_buffer_stats = smartlist_create(); - start_of_interval = circ->timestamp_created > - start_of_buffer_stats_interval ? - circ->timestamp_created : + start_of_interval = (circ->timestamp_created.tv_sec > + start_of_buffer_stats_interval) ? + circ->timestamp_created.tv_sec : start_of_buffer_stats_interval; interval_length = (int) (end_of_interval - start_of_interval); stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); @@@ -2422,227 -2456,6 +2445,227 @@@ rep_hist_buffer_stats_write(time_t now return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } +/*** Connection statistics ***/ + +/** Start of the current connection stats interval or 0 if we're not + * collecting connection statistics. */ +static time_t start_of_conn_stats_interval; + +/** Initialize connection stats. */ +void +rep_hist_conn_stats_init(time_t now) +{ + start_of_conn_stats_interval = now; +} + +/* Count connections that we read and wrote less than these many bytes + * from/to as below threshold. */ +#define BIDI_THRESHOLD 20480 + +/* Count connections that we read or wrote at least this factor as many + * bytes from/to than we wrote or read to/from as mostly reading or + * writing. */ +#define BIDI_FACTOR 10 + +/* Interval length in seconds for considering read and written bytes for + * connection stats. */ +#define BIDI_INTERVAL 10 + +/* Start of next BIDI_INTERVAL second interval. */ +static time_t bidi_next_interval = 0; + +/* Number of connections that we read and wrote less than BIDI_THRESHOLD + * bytes from/to in BIDI_INTERVAL seconds. */ +static uint32_t below_threshold = 0; + +/* Number of connections that we read at least BIDI_FACTOR times more + * bytes from than we wrote to in BIDI_INTERVAL seconds. */ +static uint32_t mostly_read = 0; + +/* Number of connections that we wrote at least BIDI_FACTOR times more + * bytes to than we read from in BIDI_INTERVAL seconds. */ +static uint32_t mostly_written = 0; + +/* Number of connections that we read and wrote at least BIDI_THRESHOLD + * bytes from/to, but not BIDI_FACTOR times more in either direction in + * BIDI_INTERVAL seconds. */ +static uint32_t both_read_and_written = 0; + +/* Entry in a map from connection ID to the number of read and written + * bytes on this connection in a BIDI_INTERVAL second interval. */ +typedef struct bidi_map_entry_t { + HT_ENTRY(bidi_map_entry_t) node; + uint64_t conn_id; /**< Connection ID */ + size_t read; /**< Number of read bytes */ + size_t written; /**< Number of written bytes */ +} bidi_map_entry_t; + +/** Map of OR connections together with the number of read and written + * bytes in the current BIDI_INTERVAL second interval. */ +static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map = + HT_INITIALIZER(); + +static int +bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b) +{ + return a->conn_id == b->conn_id; +} + +static unsigned +bidi_map_ent_hash(const bidi_map_entry_t *entry) +{ + return (unsigned) entry->conn_id; +} + +HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq); +HT_GENERATE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq, 0.6, malloc, realloc, free); + +static void +bidi_map_free(void) +{ + bidi_map_entry_t **ptr, **next, *ent; + for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) { + ent = *ptr; + next = HT_NEXT_RMV(bidimap, &bidi_map, ptr); + tor_free(ent); + } + HT_CLEAR(bidimap, &bidi_map); +} + +/** Reset counters for conn statistics. */ +void +rep_hist_reset_conn_stats(time_t now) +{ + start_of_conn_stats_interval = now; + below_threshold = 0; + mostly_read = 0; + mostly_written = 0; + both_read_and_written = 0; + bidi_map_free(); +} + +/** Stop collecting connection stats in a way that we can re-start doing + * so in rep_hist_conn_stats_init(). */ +void +rep_hist_conn_stats_term(void) +{ + rep_hist_reset_conn_stats(0); +} + +/** We read num_read bytes and wrote num_written from/to OR + * connection conn_id in second when. If this is the first + * observation in a new interval, sum up the last observations. Add bytes + * for this connection. */ +void +rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read, + size_t num_written, time_t when) +{ + if (!start_of_conn_stats_interval) + return; + /* Initialize */ + if (bidi_next_interval == 0) + bidi_next_interval = when + BIDI_INTERVAL; + /* Sum up last period's statistics */ + if (when >= bidi_next_interval) { + bidi_map_entry_t **ptr, **next, *ent; + for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) { + ent = *ptr; + if (ent->read + ent->written < BIDI_THRESHOLD) + below_threshold++; + else if (ent->read >= ent->written * BIDI_FACTOR) + mostly_read++; + else if (ent->written >= ent->read * BIDI_FACTOR) + mostly_written++; + else + both_read_and_written++; + next = HT_NEXT_RMV(bidimap, &bidi_map, ptr); + tor_free(ent); + } + while (when >= bidi_next_interval) + bidi_next_interval += BIDI_INTERVAL; + log_info(LD_GENERAL, "%d below threshold, %d mostly read, " + "%d mostly written, %d both read and written.", + below_threshold, mostly_read, mostly_written, + both_read_and_written); + } + /* Add this connection's bytes. */ + if (num_read > 0 || num_written > 0) { + bidi_map_entry_t *entry, lookup; + lookup.conn_id = conn_id; + entry = HT_FIND(bidimap, &bidi_map, &lookup); + if (entry) { + entry->written += num_written; + entry->read += num_read; + } else { + entry = tor_malloc_zero(sizeof(bidi_map_entry_t)); + entry->conn_id = conn_id; + entry->written = num_written; + entry->read = num_read; + HT_INSERT(bidimap, &bidi_map, entry); + } + } +} + +/** Return a newly allocated string containing the connection statistics + * until now, or NULL if we're not collecting conn stats. */ +char * +rep_hist_format_conn_stats(time_t now) +{ + char *result, written[ISO_TIME_LEN+1]; + + if (!start_of_conn_stats_interval) + return NULL; /* Not initialized. */ + + format_iso_time(written, now); + tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n", + written, + (unsigned) (now - start_of_conn_stats_interval), + below_threshold, + mostly_read, + mostly_written, + both_read_and_written); + return result; +} + +/** If 24 hours have passed since the beginning of the current conn stats + * period, write conn stats to $DATADIR/stats/conn-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write conn stats or 0 if we never want to write. */ +time_t +rep_hist_conn_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *str = NULL; + + if (!start_of_conn_stats_interval) + return 0; /* Not initialized. */ + if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* Generate history string. */ + str = rep_hist_format_conn_stats(now); + + /* Reset counters. */ + rep_hist_reset_conn_stats(now); + + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; + } + filename = get_datadir_fname2("stats", "conn-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write conn stats to disk!"); + + done: + tor_free(str); + tor_free(filename); + tor_free(statsdir); + return start_of_conn_stats_interval + WRITE_STATS_INTERVAL; +} + /** Free all storage held by the OR/link history caches, by the * bandwidth history arrays, by the port history, or by statistics . */ void @@@ -2657,6 -2470,5 +2680,6 @@@ rep_hist_free_all(void tor_free(exit_streams); built_last_stability_doc_at = 0; predicted_ports_free(); + bidi_map_free(); }

1 0

[tor/master] Turn the loop bodies of rep_hist_save/load_state into functions
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 8dd4ecd14e6e6e886c76c8464595d4f903447c42 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 12:57:44 2011 -0500 Turn the loop bodies of rep_hist_save/load_state into functions The trick of looping from i=0..4 , switching on i to set up some variables, then running some common code is much better expressed by just calling a function 4 times with 4 sets of arguments. This should make the code a little easier to follow and maintain here. --- src/or/rephist.c | 243 +++++++++++++++++++++++++---------------------------- 1 files changed, 115 insertions(+), 128 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index d85a8b7..c220426 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1392,7 +1392,7 @@ rep_hist_bandwidth_assess(void) * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@ -1484,161 +1484,148 @@ rep_hist_get_bandwidth_lines(void) return buf; } +/** Write a single bw_warray_t from its Values, Ends, and Interval entries + * from a state. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + time_t *s_begins, + int *s_interval) +{ + char buf[20*NUM_TOTALS + 1], *cp; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + cp = buf; + cp += rep_hist_fill_bandwidth_history(cp, sizeof(buf), b); + tor_snprintf(cp, sizeof(buf)-(cp-buf), + cp == buf ? U64_FORMAT : ","U64_FORMAT, + U64_PRINTF_ARG(b->total_in_period)); + *s_values = smartlist_create(); + if (server_mode(get_options())) + smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); +} + /** Update state with the newest bandwidth history. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values = NULL; - time_t *s_begins = NULL; - int *s_interval = NULL; - bw_array_t *b = NULL; +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = &state->BWHistoryWriteEnds; - s_interval = &state->BWHistoryWriteInterval; - s_values = &state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = &state->BWHistoryReadEnds; - s_interval = &state->BWHistoryReadInterval; - s_values = &state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = &state->BWHistoryDirWriteEnds; - s_interval = &state->BWHistoryDirWriteInterval; - s_values = &state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = &state->BWHistoryDirReadEnds; - s_interval = &state->BWHistoryDirReadInterval; - s_values = &state->BWHistoryDirReadValues; - break; - } - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_warray_t from its Values, Ends, and Interval entries from + * a state. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const time_t s_begins, + const int s_interval) { - time_t s_begins = 0, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval = 0; - smartlist_t *s_values = NULL; - bw_array_t *b = NULL; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<4;++r) { - switch (r) { - case 0: - b = write_array; - s_begins = state->BWHistoryWriteEnds; - s_interval = state->BWHistoryWriteInterval; - s_values = state->BWHistoryWriteValues; - break; - case 1: - b = read_array; - s_begins = state->BWHistoryReadEnds; - s_interval = state->BWHistoryReadInterval; - s_values = state->BWHistoryReadValues; - break; - case 2: - b = dir_write_array; - s_begins = state->BWHistoryDirWriteEnds; - s_interval = state->BWHistoryDirWriteInterval; - s_values = state->BWHistoryDirWriteValues; - break; - case 3: - b = dir_read_array; - s_begins = state->BWHistoryDirReadEnds; - s_interval = state->BWHistoryDirReadInterval; - s_values = state->BWHistoryDirReadValues; - break; - } - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v; + int i,ok; + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); if (!ok) { - all_ok=0; + retval = -1; log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); } if (start < now) { add_obs(b, start, v); start += NUM_SECS_BW_SUM_INTERVAL; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + /* Do we really want to zero this for the purpose of max capacity? */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; + } + b->total_obs = 0; + for (i=0; i<NUM_TOTALS; ++i) { + b->maxima[i] = 0; } + b->max_total = 0; + + return retval; +} + +/** Set bandwidth history from our saved state. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ tor_free(read_array); tor_free(write_array); + read_array = bw_array_new(); write_array = bw_array_new(); return -1;

1 0

[tor/master] On bwhist state load failure, clear dir_read/write hist too
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 28844c8403f16f184c185798c78dcd6a959450a3 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 13:15:51 2011 -0500 On bwhist state load failure, clear dir_read/write hist too --- changes/1863_bwhist | 5 +++++ src/or/rephist.c | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/changes/1863_bwhist b/changes/1863_bwhist index c1d4d86..b9e8ef4 100644 --- a/changes/1863_bwhist +++ b/changes/1863_bwhist @@ -3,3 +3,8 @@ triggered if a future version of Tor ever changed the timing granularity at which bandwidth history is measured. Bugfix on Tor 0.1.1.11-alpha. + - Correctly clear out dir_read/dir_write history when there is an + error parsing any bw history value from the state file. Bugfix on + Tor 0.2.2.15-alpha. + + diff --git a/src/or/rephist.c b/src/or/rephist.c index 5a3a15e..f0dd451 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1293,10 +1293,15 @@ static bw_array_t *dir_read_array = NULL; directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they alrady + * exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@ -1627,11 +1632,7 @@ rep_hist_load_state(or_state_t *state, char **err) if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0;

1 0

[tor/master] Fix bug when parsing bwhist with unexpected Interval
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 7e1502c0d1aa0875fba0b26ddcf021bdfa5e11a0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 13:06:50 2011 -0500 Fix bug when parsing bwhist with unexpected Interval Previously, our state parsing code would fail to parse a bwhist correctly if the Interval was anything other than the default hardcoded 15 minutes. This change makes the parsing less incorrect, though the resulting history array might get strange values in it if the intervals don't match the one we're using. (That is, if stuff was generated in 15 minute intervals, and we read it into an array that expects 30 minute intervals, we're fine, since values can be combined pairwise. But if we generate data at 30 minute intervals and read it into 15 minute intervals, alternating buckets will be empty.) Bugfix on 0.1.1.11-alpha. --- changes/1863_bwhist | 5 +++++ src/or/rephist.c | 6 +++++- 2 files changed, 10 insertions(+), 1 deletions(-) diff --git a/changes/1863_bwhist b/changes/1863_bwhist new file mode 100644 index 0000000..c1d4d86 --- /dev/null +++ b/changes/1863_bwhist @@ -0,0 +1,5 @@ + o Minor bugfixes + - Fix a bug in banwidth history state parsing that could have been + triggered if a future version of Tor ever changed the timing + granularity at which bandwidth history is measured. Bugfix on + Tor 0.1.1.11-alpha. diff --git a/src/or/rephist.c b/src/or/rephist.c index c220426..5a3a15e 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1165,6 +1165,8 @@ rep_hist_load_mtbf_data(time_t now) * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@ -1577,7 +1579,9 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, } if (start < now) { add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + /* This will result in some fairly choppy history if s_interval + * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += s_interval; } } SMARTLIST_FOREACH_END(cp); }

1 0

[tor/master] Add Maxima lists to bandwidth state.
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 105b94b75b51376128037cab05b930c4e4c15497 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 14:11:22 2011 -0500 Add Maxima lists to bandwidth state. Right now, Tor routers don't save the maxima values from the bw_history_t between sessions. That's no good, since we use those values to determine bandwidth. This code adds a new BWHist.*Maximum set of values to the state file. If they're not present, we estimate them by taking the observed total bandwidth and dividing it by the period length, which provides a lower bound. This should fix bug 1863. I'm calling it a feature. --- changes/1863_bwhist | 8 +++++- src/or/config.c | 4 +++ src/or/or.h | 8 +++++- src/or/rephist.c | 65 +++++++++++++++++++++++++++++++++++++------------- 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/changes/1863_bwhist b/changes/1863_bwhist index b9e8ef4..b942509 100644 --- a/changes/1863_bwhist +++ b/changes/1863_bwhist @@ -1,3 +1,10 @@ + o Minor features + - Servers now save observed maximum bandwidth throughput rates + to their state file (along with total usage, which was already + saved) so that they can determine their correct estimated + bandwidth on restart. Resolves bug 1863, where Tor servers + would reset their estimated bandwidth to 0 after restarting. + o Minor bugfixes - Fix a bug in banwidth history state parsing that could have been triggered if a future version of Tor ever changed the timing @@ -7,4 +14,3 @@ error parsing any bw history value from the state file. Bugfix on Tor 0.2.2.15-alpha. - diff --git a/src/or/config.c b/src/or/config.c index 17d776e..78978dc 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -443,15 +443,19 @@ static config_var_t _state_vars[] = { V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), V(BWHistoryDirReadInterval, UINT, "900"), V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), V(BWHistoryDirWriteEnds, ISOTIME, NULL), V(BWHistoryDirWriteInterval, UINT, "900"), V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), diff --git a/src/or/or.h b/src/or/or.h index cb36126..e0f8bab 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2856,19 +2856,25 @@ typedef struct { * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; time_t BWHistoryDirReadEnds; int BWHistoryDirReadInterval; smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; time_t BWHistoryDirWriteEnds; int BWHistoryDirWriteInterval; smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; /** Build time histogram */ config_line_t * BuildtimeHistogram; diff --git a/src/or/rephist.c b/src/or/rephist.c index f0dd451..a3a2bab 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1261,8 +1261,12 @@ add_obs(bw_array_t *b, time_t when, uint64_t n) /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@ -1497,15 +1501,21 @@ static void rep_hist_update_bwhist_state_section(or_state_t *state, const bw_array_t *b, smartlist_t **s_values, + smartlist_t **s_maxima, time_t *s_begins, int *s_interval) { - char buf[20*NUM_TOTALS + 1], *cp; + char *cp; + int i,j; if (*s_values) { SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); smartlist_free(*s_values); } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } if (! server_mode(get_options())) { /* Clients don't need to store bandwidth history persistently; * force these values to the defaults. */ @@ -1519,18 +1529,30 @@ rep_hist_update_bwhist_state_section(or_state_t *state, *s_begins = 0; *s_interval = 900; *s_values = smartlist_create(); + *s_maxima = smartlist_create(); return; } *s_begins = b->next_period; *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, sizeof(buf), b); - tor_snprintf(cp, sizeof(buf)-(cp-buf), - cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); + *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i > NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); } /** Update state with the newest bandwidth history. */ @@ -1541,6 +1563,7 @@ rep_hist_update_state(or_state_t *state) rep_hist_update_bwhist_state_section(state,\ (arrname),\ &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ &state->BWHistory ## st ## Ends, \ &state->BWHistory ## st ## Interval) @@ -1560,6 +1583,7 @@ rep_hist_update_state(or_state_t *state) static int rep_hist_load_bwhist_state_section(bw_array_t *b, const smartlist_t *s_values, + const smartlist_t *s_maxima, const time_t s_begins, const int s_interval) { @@ -1567,8 +1591,9 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, int retval = 0; time_t start; - uint64_t v; - int i,ok; + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { start = s_begins - s_interval*(smartlist_len(s_values)); @@ -1578,12 +1603,22 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); - if (!ok) { + if (have_maxima) { + const char *maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } + if (!ok || !ok_m) { retval = -1; log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); } + if (start < now) { add_obs(b, start, v); + b->max_total = mv; /* This will result in some fairly choppy history if s_interval * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ start += s_interval; @@ -1592,15 +1627,10 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, } /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { b->obs[i] = 0; } b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; return retval; } @@ -1619,6 +1649,7 @@ rep_hist_load_state(or_state_t *state, char **err) if (rep_hist_load_bwhist_state_section( \ (arrname), \ state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ state->BWHistory ## st ## Ends, \ state->BWHistory ## st ## Interval)<0) \ all_ok = 0

1 0

[tor/master] fix some spelling in rephist comments
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 80253ba4d77a95ab14ee80dbc4eea88bb264124c Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 15:13:07 2011 -0500 fix some spelling in rephist comments --- src/or/rephist.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index a3a2bab..265fff8 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1297,8 +1297,8 @@ static bw_array_t *dir_read_array = NULL; directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array, freeing them if they alrady - * exist. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { @@ -1495,8 +1495,8 @@ rep_hist_get_bandwidth_lines(void) return buf; } -/** Write a single bw_warray_t from its Values, Ends, and Interval entries - * from a state. */ +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ static void rep_hist_update_bwhist_state_section(or_state_t *state, const bw_array_t *b, @@ -1578,8 +1578,8 @@ rep_hist_update_state(or_state_t *state) #undef UPDATE } -/** Load a single bw_warray_t from its Values, Ends, and Interval entries from - * a state. */ +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ static int rep_hist_load_bwhist_state_section(bw_array_t *b, const smartlist_t *s_values,

1 0

[tor/master] Merge branch 'bug1863_bwhist' into maint-0.2.2
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit aa178aae03048caf24ce5df4b2d643c7e7afdc7b Merge: 16b8b9b 0642b92 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Feb 25 11:22:12 2011 -0500 Merge branch 'bug1863_bwhist' into maint-0.2.2 changes/1863_bwhist | 16 +++ src/or/config.c | 4 + src/or/or.h | 8 ++- src/or/rephist.c | 295 +++++++++++++++++++++++++++----------------------- 4 files changed, 186 insertions(+), 137 deletions(-) diff --combined src/or/config.c index 178ed1e,78978dc..4d2c872 --- a/src/or/config.c +++ b/src/or/config.c @@@ -195,7 -195,6 +195,7 @@@ static config_var_t _option_vars[] = V(CircuitStreamTimeout, INTERVAL, "0"), V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), + V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientOnly, BOOL, "0"), V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), @@@ -288,7 -287,6 +288,7 @@@ OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), + V(LogMessageDomains, BOOL, "0"), OBSOLETE("LinkPadding"), OBSOLETE("LogLevel"), OBSOLETE("LogFile"), @@@ -407,7 -405,6 +407,7 @@@ static config_var_t testing_tor_network V(AuthDirMaxServersPerAddr, UINT, "0"), V(AuthDirMaxServersPerAuthAddr,UINT, "0"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), + V(ClientRejectInternalAddresses, BOOL, "0"), V(ExitPolicyRejectPrivate, BOOL, "0"), V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), @@@ -446,15 -443,19 +446,19 @@@ static config_var_t _state_vars[] = V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), V(BWHistoryDirReadInterval, UINT, "900"), V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), V(BWHistoryDirWriteEnds, ISOTIME, NULL), V(BWHistoryDirWriteInterval, UINT, "900"), V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), @@@ -1253,6 -1254,7 +1257,6 @@@ options_act(or_options_t *old_options /* Check for transitions that need action. */ if (old_options) { - if ((options->UseEntryGuards && !old_options->UseEntryGuards) || (options->ExcludeNodes && !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)) || @@@ -1300,12 -1302,11 +1304,12 @@@ if (options_transition_affects_workers(old_options, options)) { log_info(LD_GENERAL, "Worker-related options changed. Rotating workers."); + + if (init_keys() < 0) { + log_warn(LD_BUG,"Error initializing keys; exiting"); + return -1; + } if (server_mode(options) && !server_mode(old_options)) { - if (init_keys() < 0) { - log_warn(LD_BUG,"Error initializing keys; exiting"); - return -1; - } ip_address_changed(0); if (can_complete_circuit || !any_predicted_circuits(time(NULL))) inform_testing_reachability(); @@@ -2842,9 -2843,7 +2846,9 @@@ compute_publishserverdescriptor(or_opti else if (!strcasecmp(string, "bridge")) *auth |= BRIDGE_AUTHORITY; else if (!strcasecmp(string, "hidserv")) - *auth |= HIDSERV_AUTHORITY; + log_warn(LD_CONFIG, + "PublishServerDescriptor hidserv is invalid. See " + "PublishHidServDescriptors."); else if (!strcasecmp(string, "") || !strcmp(string, "0")) /* no authority */; else @@@ -3348,11 -3347,6 +3352,11 @@@ options_validate(or_options_t *old_opti "PerConnBWBurst", msg) < 0) return -1; + if (options->RelayBandwidthRate && !options->RelayBandwidthBurst) + options->RelayBandwidthBurst = options->RelayBandwidthRate; + if (options->RelayBandwidthBurst && !options->RelayBandwidthRate) + options->RelayBandwidthRate = options->RelayBandwidthBurst; + if (server_mode(options)) { if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { tor_asprintf(msg, @@@ -3381,6 -3375,9 +3385,6 @@@ } } - if (options->RelayBandwidthRate && !options->RelayBandwidthBurst) - options->RelayBandwidthBurst = options->RelayBandwidthRate; - if (options->RelayBandwidthRate > options->RelayBandwidthBurst) REJECT("RelayBandwidthBurst must be at least equal " "to RelayBandwidthRate."); @@@ -3817,9 -3814,7 +3821,9 @@@ options_transition_affects_workers(or_o new_options->ServerDNSSearchDomains || old_options->SafeLogging != new_options->SafeLogging || old_options->ClientOnly != new_options->ClientOnly || - !config_lines_eq(old_options->Logs, new_options->Logs)) + public_server_mode(old_options) != public_server_mode(new_options) || + !config_lines_eq(old_options->Logs, new_options->Logs) || + old_options->LogMessageDomains != new_options->LogMessageDomains) return 1; /* Check whether log options match. */ @@@ -4395,9 -4390,6 +4399,9 @@@ options_init_logs(or_options_t *options } smartlist_free(elts); + if (ok && !validate_only) + logs_set_domain_logging(options->LogMessageDomains); + return ok?0:-1; } @@@ -5157,9 -5149,6 +5161,9 @@@ or_state_load(void return r; } +/** If writing the state to disk fails, try again after this many seconds. */ +#define STATE_WRITE_RETRY_INTERVAL 3600 + /** Write the persistent state to disk. Return 0 for success, <0 on failure. */ int or_state_save(time_t now) @@@ -5194,14 -5183,10 +5198,14 @@@ tor_free(state); fname = get_datadir_fname("state"); if (write_str_to_file(fname, contents, 0)<0) { - log_warn(LD_FS, "Unable to write state to file \"%s\"", fname); + log_warn(LD_FS, "Unable to write state to file \"%s\"; " + "will try again later", fname); global_state->LastWritten = -1; tor_free(fname); tor_free(contents); + /* Try again after STATE_WRITE_RETRY_INTERVAL (or sooner, if the state + * changes sooner). */ + global_state->next_write = now + STATE_WRITE_RETRY_INTERVAL; return -1; } diff --combined src/or/or.h index 2a55668,e0f8bab..74ff743 --- a/src/or/or.h +++ b/src/or/or.h @@@ -583,9 -583,6 +583,9 @@@ typedef enum /** This is a connection on the NATD port, and the destination IP:Port was * either ill-formed or out-of-range. */ #define END_STREAM_REASON_INVALID_NATD_DEST 261 +/** The target address is in a private network (like 127.0.0.1 or 10.0.0.1); + * you don't want to do that over a randomly chosen exit */ +#define END_STREAM_REASON_PRIVATE_ADDR 262 /** Bitwise-and this value with endreason to mask out all flags. */ #define END_STREAM_REASON_MASK 511 @@@ -768,8 -765,6 +768,8 @@@ typedef enum /** Initial value for both sides of a circuit transmission window when the * circuit is initialized. Measured in cells. */ #define CIRCWINDOW_START 1000 +#define CIRCWINDOW_START_MIN 100 +#define CIRCWINDOW_START_MAX 1000 /** Amount to increment a circuit window when we get a circuit SENDME. */ #define CIRCWINDOW_INCREMENT 100 /** Initial value on both sides of a stream transmission window when the @@@ -1173,10 -1168,6 +1173,10 @@@ typedef struct edge_connection_t * zero, abandon the associated mapaddress. */ unsigned int chosen_exit_retries:3; + /** True iff this is an AP connection that came from a transparent or + * NATd connection */ + unsigned int is_transparent_ap:1; + /** If this is a DNSPort connection, this field holds the pending DNS * request that we're going to try to answer. */ struct evdns_server_request *dns_server_request; @@@ -2345,9 -2336,6 +2345,9 @@@ typedef struct config_line_t *Logs; /**< New-style list of configuration lines * for logs */ + int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which + * each log message occurs? */ + char *DebugLogFile; /**< Where to send verbose log messages. */ char *DataDirectory; /**< OR only: where to store long-term data. */ char *Nickname; /**< OR only: nickname of this onion router. */ @@@ -2759,10 -2747,6 +2759,10 @@@ * Helps avoid some cross-site attacks. */ int ClientDNSRejectInternalAddresses; + /** If true, do not accept any requests to connect to internal addresses + * over randomly chosen exits. */ + int ClientRejectInternalAddresses; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@@ -2872,19 -2856,25 +2872,25 @@@ typedef struct * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; time_t BWHistoryDirReadEnds; int BWHistoryDirReadInterval; smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; time_t BWHistoryDirWriteEnds; int BWHistoryDirWriteInterval; smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; /** Build time histogram */ config_line_t * BuildtimeHistogram; @@@ -2958,11 -2948,6 +2964,11 @@@ struct socks_request_t /* Circuit Build Timeout "public" structures. */ +/** Precision multiplier for the Bw weights */ +#define BW_WEIGHT_SCALE 10000 +#define BW_MIN_WEIGHT_SCALE 1 +#define BW_MAX_WEIGHT_SCALE INT32_MAX + /** Total size of the circuit timeout history to accumulate. * 1000 is approx 2.5 days worth of continual-use circuits. */ #define CBT_NCIRCUITS_TO_OBSERVE 1000 @@@ -2972,8 -2957,6 +2978,8 @@@ /** Number of modes to use in the weighted-avg computation of Xm */ #define CBT_DEFAULT_NUM_XM_MODES 3 +#define CBT_MIN_NUM_XM_MODES 1 +#define CBT_MAX_NUM_XM_MODES 20 /** A build_time_t is milliseconds */ typedef uint32_t build_time_t; @@@ -2995,16 -2978,12 +3001,16 @@@ * build in terms of CDF quantile. */ #define CBT_DEFAULT_CLOSE_QUANTILE 95 +#define CBT_MIN_CLOSE_QUANTILE CBT_MIN_QUANTILE_CUTOFF +#define CBT_MAX_CLOSE_QUANTILE CBT_MAX_QUANTILE_CUTOFF /** * How many circuits count as recent when considering if the * connection has gone gimpy or changed. */ #define CBT_DEFAULT_RECENT_CIRCUITS 20 +#define CBT_MIN_RECENT_CIRCUITS 3 +#define CBT_MAX_RECENT_CIRCUITS 1000 /** * Maximum count of timeouts that finish the first hop in the past @@@ -3015,37 -2994,25 +3021,37 @@@ * gives us. */ #define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10) +#define CBT_MIN_MAX_RECENT_TIMEOUT_COUNT 3 +#define CBT_MAX_MAX_RECENT_TIMEOUT_COUNT 10000 /** Minimum circuits before estimating a timeout */ #define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100 +#define CBT_MIN_MIN_CIRCUITS_TO_OBSERVE 1 +#define CBT_MAX_MIN_CIRCUITS_TO_OBSERVE 10000 /** Cutoff percentile on the CDF for our timeout estimation. */ #define CBT_DEFAULT_QUANTILE_CUTOFF 80 +#define CBT_MIN_QUANTILE_CUTOFF 10 +#define CBT_MAX_QUANTILE_CUTOFF 99 double circuit_build_times_quantile_cutoff(void); /** How often in seconds should we build a test circuit */ #define CBT_DEFAULT_TEST_FREQUENCY 60 +#define CBT_MIN_TEST_FREQUENCY 1 +#define CBT_MAX_TEST_FREQUENCY INT32_MAX /** Lowest allowable value for CircuitBuildTimeout in milliseconds */ #define CBT_DEFAULT_TIMEOUT_MIN_VALUE (1500) +#define CBT_MIN_TIMEOUT_MIN_VALUE 500 +#define CBT_MAX_TIMEOUT_MIN_VALUE INT32_MAX /** Initial circuit build timeout in milliseconds */ #define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000) +#define CBT_MIN_TIMEOUT_INITIAL_VALUE CBT_MIN_TIMEOUT_MIN_VALUE +#define CBT_MAX_TIMEOUT_INITIAL_VALUE INT32_MAX int32_t circuit_build_times_initial_timeout(void); -#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < 1 +#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < CBT_MIN_MAX_RECENT_TIMEOUT_COUNT #error "RECENT_CIRCUITS is set too low." #endif

1 0

[tor/master] Fix two potential bugs in the bug1863 code.
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 0642b927928df4ec556a76fed4b0b4523d515a7d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Feb 24 16:44:54 2011 +0100 Fix two potential bugs in the bug1863 code. --- src/or/rephist.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index 265fff8..3f4a704 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1513,7 +1513,7 @@ rep_hist_update_bwhist_state_section(or_state_t *state, smartlist_free(*s_values); } if (*s_maxima) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); smartlist_free(*s_maxima); } if (! server_mode(get_options())) { @@ -1541,7 +1541,7 @@ rep_hist_update_bwhist_state_section(or_state_t *state, i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; for (j=0; j < b->num_maxes_set; ++j,++i) { uint64_t maxval; - if (i > NUM_TOTALS) + if (i >= NUM_TOTALS) i = 0; tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); smartlist_add(*s_values, cp);

1 0

[tor/maint-0.2.2] Add Maxima lists to bandwidth state.
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 105b94b75b51376128037cab05b930c4e4c15497 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 14:11:22 2011 -0500 Add Maxima lists to bandwidth state. Right now, Tor routers don't save the maxima values from the bw_history_t between sessions. That's no good, since we use those values to determine bandwidth. This code adds a new BWHist.*Maximum set of values to the state file. If they're not present, we estimate them by taking the observed total bandwidth and dividing it by the period length, which provides a lower bound. This should fix bug 1863. I'm calling it a feature. --- changes/1863_bwhist | 8 +++++- src/or/config.c | 4 +++ src/or/or.h | 8 +++++- src/or/rephist.c | 65 +++++++++++++++++++++++++++++++++++++------------- 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/changes/1863_bwhist b/changes/1863_bwhist index b9e8ef4..b942509 100644 --- a/changes/1863_bwhist +++ b/changes/1863_bwhist @@ -1,3 +1,10 @@ + o Minor features + - Servers now save observed maximum bandwidth throughput rates + to their state file (along with total usage, which was already + saved) so that they can determine their correct estimated + bandwidth on restart. Resolves bug 1863, where Tor servers + would reset their estimated bandwidth to 0 after restarting. + o Minor bugfixes - Fix a bug in banwidth history state parsing that could have been triggered if a future version of Tor ever changed the timing @@ -7,4 +14,3 @@ error parsing any bw history value from the state file. Bugfix on Tor 0.2.2.15-alpha. - diff --git a/src/or/config.c b/src/or/config.c index 17d776e..78978dc 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -443,15 +443,19 @@ static config_var_t _state_vars[] = { V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), V(BWHistoryDirReadInterval, UINT, "900"), V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), V(BWHistoryDirWriteEnds, ISOTIME, NULL), V(BWHistoryDirWriteInterval, UINT, "900"), V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), diff --git a/src/or/or.h b/src/or/or.h index cb36126..e0f8bab 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2856,19 +2856,25 @@ typedef struct { * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; time_t BWHistoryDirReadEnds; int BWHistoryDirReadInterval; smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; time_t BWHistoryDirWriteEnds; int BWHistoryDirWriteInterval; smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; /** Build time histogram */ config_line_t * BuildtimeHistogram; diff --git a/src/or/rephist.c b/src/or/rephist.c index f0dd451..a3a2bab 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1261,8 +1261,12 @@ add_obs(bw_array_t *b, time_t when, uint64_t n) /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + while (when>b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@ -1497,15 +1501,21 @@ static void rep_hist_update_bwhist_state_section(or_state_t *state, const bw_array_t *b, smartlist_t **s_values, + smartlist_t **s_maxima, time_t *s_begins, int *s_interval) { - char buf[20*NUM_TOTALS + 1], *cp; + char *cp; + int i,j; if (*s_values) { SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); smartlist_free(*s_values); } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } if (! server_mode(get_options())) { /* Clients don't need to store bandwidth history persistently; * force these values to the defaults. */ @@ -1519,18 +1529,30 @@ rep_hist_update_bwhist_state_section(or_state_t *state, *s_begins = 0; *s_interval = 900; *s_values = smartlist_create(); + *s_maxima = smartlist_create(); return; } *s_begins = b->next_period; *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, sizeof(buf), b); - tor_snprintf(cp, sizeof(buf)-(cp-buf), - cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); + *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + uint64_t maxval; + if (i > NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->max_total & ~0x3ff)); + smartlist_add(*s_maxima, cp); } /** Update state with the newest bandwidth history. */ @@ -1541,6 +1563,7 @@ rep_hist_update_state(or_state_t *state) rep_hist_update_bwhist_state_section(state,\ (arrname),\ &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ &state->BWHistory ## st ## Ends, \ &state->BWHistory ## st ## Interval) @@ -1560,6 +1583,7 @@ rep_hist_update_state(or_state_t *state) static int rep_hist_load_bwhist_state_section(bw_array_t *b, const smartlist_t *s_values, + const smartlist_t *s_maxima, const time_t s_begins, const int s_interval) { @@ -1567,8 +1591,9 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, int retval = 0; time_t start; - uint64_t v; - int i,ok; + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { start = s_begins - s_interval*(smartlist_len(s_values)); @@ -1578,12 +1603,22 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); - if (!ok) { + if (have_maxima) { + const char *maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = v / s_interval; + } + if (!ok || !ok_m) { retval = -1; log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); } + if (start < now) { add_obs(b, start, v); + b->max_total = mv; /* This will result in some fairly choppy history if s_interval * is notthe same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ start += s_interval; @@ -1592,15 +1627,10 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, } /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { b->obs[i] = 0; } b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; return retval; } @@ -1619,6 +1649,7 @@ rep_hist_load_state(or_state_t *state, char **err) if (rep_hist_load_bwhist_state_section( \ (arrname), \ state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ state->BWHistory ## st ## Ends, \ state->BWHistory ## st ## Interval)<0) \ all_ok = 0

1 0

[tor/maint-0.2.2] fix some spelling in rephist comments
by nickm＠torproject.org 25 Feb '11

25 Feb '11

commit 80253ba4d77a95ab14ee80dbc4eea88bb264124c Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 10 15:13:07 2011 -0500 fix some spelling in rephist comments --- src/or/rephist.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/or/rephist.c b/src/or/rephist.c index a3a2bab..265fff8 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1297,8 +1297,8 @@ static bw_array_t *dir_read_array = NULL; directory protocol. */ static bw_array_t *dir_write_array = NULL; -/** Set up [dir-]read_array and [dir-]write_array, freeing them if they alrady - * exist. */ +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { @@ -1495,8 +1495,8 @@ rep_hist_get_bandwidth_lines(void) return buf; } -/** Write a single bw_warray_t from its Values, Ends, and Interval entries - * from a state. */ +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. */ static void rep_hist_update_bwhist_state_section(or_state_t *state, const bw_array_t *b, @@ -1578,8 +1578,8 @@ rep_hist_update_state(or_state_t *state) #undef UPDATE } -/** Load a single bw_warray_t from its Values, Ends, and Interval entries from - * a state. */ +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. */ static int rep_hist_load_bwhist_state_section(bw_array_t *b, const smartlist_t *s_values,

1 0