September 2018 - tor-commits - lists.torproject.org

[tor/master] Move the n_v3_ns_requests field out of geoip_country_t
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit 79208ee852b84c94b32db1b13a2ba802cae360f7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 09:04:08 2018 -0400 Move the n_v3_ns_requests field out of geoip_country_t This is preparation for splitting geoip.c into stats and non-stats portions. --- src/feature/stats/geoip.c | 49 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip.c index 019aa7a28..7bae44848 100644 --- a/src/feature/stats/geoip.c +++ b/src/feature/stats/geoip.c @@ -60,7 +60,6 @@ typedef struct geoip_ipv6_entry_t { /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { char countrycode[3]; - uint32_t n_v3_ns_requests; } geoip_country_t; /** A list of geoip_country_t */ @@ -77,6 +76,12 @@ static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL; static char geoip_digest[DIGEST_LEN]; static char geoip6_digest[DIGEST_LEN]; +/** Number of entries in n_v3_ns_requests */ +size_t n_v3_ns_requests_len = 0; +/** Array, indexed by country index, of number of v3 networkstatus requests + * received from that country */ +static uint32_t *n_v3_ns_requests; + /* Total size in bytes of the geoip client history cache. Used by the OOM * handler. */ static size_t geoip_client_history_cache_size; @@ -109,6 +114,32 @@ geoip_decrement_client_history_cache_size(size_t bytes) geoip_client_history_cache_size -= bytes; } +/** Add 1 to the count of v3 ns requests received from country. */ +static void +increment_v3_ns_request(country_t country) +{ + if (country < 0) + return; + + if ((size_t)country >= n_v3_ns_requests_len) { + /* We need to reallocate the array. */ + size_t new_len; + if (n_v3_ns_requests_len == 0) + new_len = 256; + else + new_len = n_v3_ns_requests_len * 2; + if (new_len <= (size_t)country) + new_len = ((size_t)country)+1; + n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len, + sizeof(uint32_t)); + memset(n_v3_ns_requests + n_v3_ns_requests_len, 0, + sizeof(uint32_t)*(new_len - n_v3_ns_requests_len)); + n_v3_ns_requests_len = new_len; + } + + n_v3_ns_requests[country] += 1; +} + /** Return the index of the country's entry in the GeoIP * country list if it is a valid 2-letter country code, otherwise * return -1. */ @@ -660,10 +691,7 @@ geoip_note_client_seen(geoip_client_action_t action, int country_idx = geoip_get_country_by_addr(addr); if (country_idx < 0) country_idx = 0; /** unresolved requests are stored at index 0. */ - if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { - geoip_country_t *country = smartlist_get(geoip_countries, country_idx); - ++country->n_v3_ns_requests; - } + increment_v3_ns_request(country_idx); } } @@ -1282,7 +1310,10 @@ geoip_get_request_history(void) SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) { uint32_t tot = 0; c_hist_t *ent; - tot = c->n_v3_ns_requests; + if ((size_t)c_sl_idx < n_v3_ns_requests_len) + tot = n_v3_ns_requests[c_sl_idx]; + else + tot = 0; if (!tot) continue; ent = tor_malloc_zero(sizeof(c_hist_t)); @@ -1319,9 +1350,8 @@ geoip_dirreq_stats_init(time_t now) void geoip_reset_dirreq_stats(time_t now) { - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - c->n_v3_ns_requests = 0; - }); + memset(n_v3_ns_requests, 0, + n_v3_ns_requests_len * sizeof(uint32_t)); { clientmap_entry_t **ent, **next, *this; for (ent = HT_START(clientmap, &client_history); ent != NULL; @@ -1883,6 +1913,7 @@ geoip_free_all(void) clear_geoip_db(); tor_free(bridge_stats_extrainfo); + tor_free(n_v3_ns_requests); memset(geoip_digest, 0, sizeof(geoip_digest)); memset(geoip6_digest, 0, sizeof(geoip6_digest));

1 0

[tor/master] Split geoip from geoip-related stats.
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit f403af22076b0092807822ddc9890acdbd62f578 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 09:35:47 2018 -0400 Split geoip from geoip-related stats. This commit just moves the code to two separate files. The geoip code still has a few needless dependencies on core/* and features/*. --- src/app/config/config.c | 1 + src/app/main/main.c | 3 + src/core/include.am | 4 + src/core/mainloop/mainloop.c | 2 +- src/core/or/channel.c | 23 +- src/core/or/dos.c | 10 +- src/core/or/relay.c | 2 +- src/core/or/status.c | 2 +- src/feature/control/control.c | 3 +- src/feature/control/getinfo_geoip.c | 45 ++ src/feature/control/getinfo_geoip.h | 14 + src/feature/dircache/dircache.c | 2 +- src/feature/dircommon/directory.c | 2 +- src/feature/relay/router.c | 1 + src/feature/stats/geoip.c | 1433 +---------------------------------- src/feature/stats/geoip.h | 129 +--- src/feature/stats/geoip_stats.c | 1422 ++++++++++++++++++++++++++++++++++ src/feature/stats/geoip_stats.h | 139 ++++ src/test/test_dir_handle_get.c | 1 + src/test/test_dos.c | 3 +- src/test/test_geoip.c | 1 + 21 files changed, 1673 insertions(+), 1569 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 2818cdaad..0909ced50 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -103,6 +103,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index c756e55b9..2266a5ff7 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -27,6 +27,7 @@ #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" +#include "core/or/dos.h" #include "core/or/policies.h" #include "core/or/protover.h" #include "core/or/relay.h" @@ -61,6 +62,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" @@ -755,6 +757,7 @@ tor_free_all(int postfork) evdns_shutdown(1); } geoip_free_all(); + geoip_stats_free_all(); dirvote_free_all(); routerlist_free_all(); networkstatus_free_all(); diff --git a/src/core/include.am b/src/core/include.am index 68bbf9474..1a26a65c2 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -60,6 +60,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/transports.c \ src/feature/control/control.c \ src/feature/control/fmt_serverstatus.c \ + src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ src/feature/dircache/conscache.c \ src/feature/dircache/consdiffmgr.c \ @@ -117,6 +118,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/rend/rendmid.c \ src/feature/rend/rendservice.c \ src/feature/stats/geoip.c \ + src/feature/stats/geoip_stats.c \ src/feature/stats/rephist.c \ src/feature/stats/predict_ports.c @@ -254,6 +256,7 @@ noinst_HEADERS += \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/fmt_serverstatus.h \ + src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ src/feature/dirauth/bwauth.h \ src/feature/dirauth/dircollate.h \ @@ -348,6 +351,7 @@ noinst_HEADERS += \ src/feature/rend/rendmid.h \ src/feature/rend/rendservice.h \ src/feature/stats/geoip.h \ + src/feature/stats/geoip_stats.h \ src/feature/stats/rephist.h \ src/feature/stats/predict_ports.h diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index ebfdf1c94..4eaeef46c 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -92,7 +92,7 @@ #include "feature/relay/selftest.h" #include "feature/rend/rendcache.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/container/buffers.h" diff --git a/src/core/or/channel.c b/src/core/or/channel.c index e9011520a..fe3785d52 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -58,28 +58,29 @@ #define CHANNEL_PRIVATE_ #include "core/or/or.h" +#include "app/config/config.h" +#include "core/mainloop/mainloop.h" #include "core/or/channel.h" -#include "core/or/channeltls.h" #include "core/or/channelpadding.h" +#include "core/or/channeltls.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" +#include "core/or/circuitmux.h" #include "core/or/circuitstats.h" -#include "app/config/config.h" #include "core/or/connection_or.h" /* For var_cell_free() */ -#include "core/or/circuitmux.h" -#include "feature/client/entrynodes.h" -#include "feature/stats/geoip.h" -#include "core/mainloop/mainloop.h" -#include "feature/nodelist/nodelist.h" +#include "core/or/dos.h" #include "core/or/relay.h" -#include "feature/stats/rephist.h" -#include "feature/relay/router.h" -#include "feature/nodelist/routerlist.h" #include "core/or/scheduler.h" -#include "lib/time/compat_time.h" +#include "feature/client/entrynodes.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/relay/router.h" #include "feature/rend/rendservice.h" +#include "feature/stats/geoip_stats.h" +#include "feature/stats/rephist.h" #include "lib/evloop/timers.h" +#include "lib/time/compat_time.h" #include "core/or/cell_queue_st.h" diff --git a/src/core/or/dos.c b/src/core/or/dos.c index c59e2f636..4303b8fd5 100644 --- a/src/core/or/dos.c +++ b/src/core/or/dos.c @@ -9,17 +9,17 @@ #define DOS_PRIVATE #include "core/or/or.h" -#include "core/or/channel.h" #include "app/config/config.h" #include "core/mainloop/connection.h" -#include "core/or/connection_or.h" -#include "lib/crypt_ops/crypto_rand.h" -#include "feature/stats/geoip.h" #include "core/mainloop/mainloop.h" +#include "core/or/channel.h" +#include "core/or/connection_or.h" +#include "core/or/relay.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" -#include "core/or/relay.h" #include "feature/relay/routermode.h" +#include "feature/stats/geoip_stats.h" +#include "lib/crypt_ops/crypto_rand.h" #include "core/or/dos.h" diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 88bcdf070..9e45a3524 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -65,7 +65,7 @@ #include "lib/crypt_ops/crypto_util.h" #include "feature/dircommon/directory.h" #include "feature/relay/dns.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/hs/hs_cache.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/or/status.c b/src/core/or/status.c index d8a9da0a5..cdb6df123 100644 --- a/src/core/or/status.c +++ b/src/core/or/status.c @@ -30,13 +30,13 @@ #include "feature/hs/hs_stats.h" #include "feature/hs/hs_service.h" #include "core/or/dos.h" +#include "feature/stats/geoip_stats.h" #include "app/config/or_state_st.h" #include "feature/nodelist/routerinfo_st.h" #include "lib/tls/tortls.h" static void log_accounting(const time_t now, const or_options_t *options); -#include "feature/stats/geoip.h" /** Return the total number of circuits. */ STATIC int diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b8ddb3c13..d02ecbac8 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -60,6 +60,7 @@ #include "feature/client/entrynodes.h" #include "feature/control/control.h" #include "feature/control/fmt_serverstatus.h" +#include "feature/control/getinfo_geoip.h" #include "feature/dircache/dirserv.h" #include "feature/dirclient/dirclient.h" #include "feature/dirclient/dlstatus.h" @@ -83,7 +84,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "lib/container/buffers.h" #include "lib/crypt_ops/crypto_rand.h" diff --git a/src/feature/control/getinfo_geoip.c b/src/feature/control/getinfo_geoip.c new file mode 100644 index 000000000..35eb3f6ae --- /dev/null +++ b/src/feature/control/getinfo_geoip.c @@ -0,0 +1,45 @@ + +#include "core/or/or.h" +#include "core/mainloop/connection.h" +#include "feature/control/control.h" +#include "feature/control/getinfo_geoip.h" +#include "feature/stats/geoip.h" + +/** Helper used to implement GETINFO ip-to-country/... controller command. */ +int +getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + (void)control_conn; + if (!strcmpstart(question, "ip-to-country/")) { + int c; + sa_family_t family; + tor_addr_t addr; + question += strlen("ip-to-country/"); + + if (!strcmp(question, "ipv4-available") || + !strcmp(question, "ipv6-available")) { + family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6; + const int available = geoip_is_loaded(family); + tor_asprintf(answer, "%d", !! available); + return 0; + } + + family = tor_addr_parse(&addr, question); + if (family != AF_INET && family != AF_INET6) { + *errmsg = "Invalid address family"; + return -1; + } + if (!geoip_is_loaded(family)) { + *errmsg = "GeoIP data not loaded"; + return -1; + } + if (family == AF_INET) + c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr)); + else /* AF_INET6 */ + c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr)); + *answer = tor_strdup(geoip_get_country_name(c)); + } + return 0; +} diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h new file mode 100644 index 000000000..ff77cefec --- /dev/null +++ b/src/feature/control/getinfo_geoip.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_GETINFO_GEOIP_H +#define TOR_GETINFO_GEOIP_H + +int getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); + +#endif diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 76bde0b78..872a88018 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -25,7 +25,7 @@ #include "feature/nodelist/routerlist.h" #include "feature/relay/routermode.h" #include "feature/rend/rendcache.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c index 8116d72c6..dff68fcf0 100644 --- a/src/feature/dircommon/directory.c +++ b/src/feature/dircommon/directory.c @@ -12,7 +12,7 @@ #include "feature/dirclient/dirclient.h" #include "feature/dircommon/directory.h" #include "feature/dircommon/fp_pair.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "lib/compress/compress.h" #include "feature/dircommon/dir_connection_st.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 8629ad956..494634845 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -37,6 +37,7 @@ #include "feature/relay/routermode.h" #include "feature/relay/selftest.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_format.h" diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip.c index 7bae44848..885f26bc3 100644 --- a/src/feature/stats/geoip.c +++ b/src/feature/stats/geoip.c @@ -31,12 +31,9 @@ #include "core/or/or.h" #include "ht.h" #include "lib/container/buffers.h" -#include "app/config/config.h" -#include "feature/control/control.h" -#include "feature/client/dnsserv.h" -#include "core/or/dos.h" +#include "app/config/config.h" // XXXX #include "feature/stats/geoip.h" -#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerlist.h" // XXXX #include "lib/container/order.h" #include "lib/time/tvdiff.h" @@ -57,11 +54,6 @@ typedef struct geoip_ipv6_entry_t { intptr_t country; /**< An index into geoip_countries */ } geoip_ipv6_entry_t; -/** A per-country record for GeoIP request history. */ -typedef struct geoip_country_t { - char countrycode[3]; -} geoip_country_t; - /** A list of geoip_country_t */ static smartlist_t *geoip_countries = NULL; /** A map from lowercased country codes to their position in geoip_countries. @@ -76,68 +68,14 @@ static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL; static char geoip_digest[DIGEST_LEN]; static char geoip6_digest[DIGEST_LEN]; -/** Number of entries in n_v3_ns_requests */ -size_t n_v3_ns_requests_len = 0; -/** Array, indexed by country index, of number of v3 networkstatus requests - * received from that country */ -static uint32_t *n_v3_ns_requests; - -/* Total size in bytes of the geoip client history cache. Used by the OOM - * handler. */ -static size_t geoip_client_history_cache_size; - -/* Increment the geoip client history cache size counter with the given bytes. - * This prevents an overflow and set it to its maximum in that case. */ -static inline void -geoip_increment_client_history_cache_size(size_t bytes) +/** Return a list of geoip_country_t for all known countries. */ +const smartlist_t * +geoip_get_countries(void) { - /* This is shockingly high, lets log it so it can be reported. */ - IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) { - geoip_client_history_cache_size = SIZE_MAX; - return; - } - geoip_client_history_cache_size += bytes; -} - -/* Decrement the geoip client history cache size counter with the given bytes. - * This prevents an underflow and set it to 0 in that case. */ -static inline void -geoip_decrement_client_history_cache_size(size_t bytes) -{ - /* Going below 0 means that we either allocated an entry without - * incrementing the counter or we have different sizes when allocating and - * freeing. It shouldn't happened so log it. */ - IF_BUG_ONCE(geoip_client_history_cache_size < bytes) { - geoip_client_history_cache_size = 0; - return; - } - geoip_client_history_cache_size -= bytes; -} - -/** Add 1 to the count of v3 ns requests received from country. */ -static void -increment_v3_ns_request(country_t country) -{ - if (country < 0) - return; - - if ((size_t)country >= n_v3_ns_requests_len) { - /* We need to reallocate the array. */ - size_t new_len; - if (n_v3_ns_requests_len == 0) - new_len = 256; - else - new_len = n_v3_ns_requests_len * 2; - if (new_len <= (size_t)country) - new_len = ((size_t)country)+1; - n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len, - sizeof(uint32_t)); - memset(n_v3_ns_requests + n_v3_ns_requests_len, 0, - sizeof(uint32_t)*(new_len - n_v3_ns_requests_len)); - n_v3_ns_requests_len = new_len; + if (geoip_countries == NULL) { + init_geoip_countries(); } - - n_v3_ns_requests[country] += 1; + return geoip_countries; } /** Return the index of the country's entry in the GeoIP @@ -330,14 +268,6 @@ geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member) return 0; } -/** Return 1 if we should collect geoip stats on bridge users, and - * include them in our extrainfo descriptor. Else return 0. */ -int -should_record_bridge_info(const or_options_t *options) -{ - return options->BridgeRelay && options->BridgeRecordUsageByCountry; -} - /** Set up a new list of geoip countries with no countries (yet) set in it, * except for the unknown country. */ @@ -448,7 +378,7 @@ geoip_load_file(sa_family_t family, const char *filename) * be less than geoip_get_n_countries(). To decode it, call * geoip_get_country_name(). */ -STATIC int +int geoip_get_country_by_ipv4(uint32_t ipaddr) { geoip_ipv4_entry_t *ent; @@ -464,7 +394,7 @@ geoip_get_country_by_ipv4(uint32_t ipaddr) * 0 for the 'unknown country'. The return value will always be less than * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). */ -STATIC int +int geoip_get_country_by_ipv6(const struct in6_addr *addr) { geoip_ipv6_entry_t *ent; @@ -540,1328 +470,6 @@ geoip_db_digest(sa_family_t family) return hex_str(geoip6_digest, DIGEST_LEN); } -/** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, - * so it can hold up to (1u<<30)-1, or 0x3fffffffu. - */ -#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu - -/** Map from client IP address to last time seen. */ -static HT_HEAD(clientmap, clientmap_entry_t) client_history = - HT_INITIALIZER(); - -/** Hashtable helper: compute a hash of a clientmap_entry_t. */ -static inline unsigned -clientmap_entry_hash(const clientmap_entry_t *a) -{ - unsigned h = (unsigned) tor_addr_hash(&a->addr); - - if (a->transport_name) - h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name)); - - return h; -} -/** Hashtable helper: compare two clientmap_entry_t values for equality. */ -static inline int -clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) -{ - if (strcmp_opt(a->transport_name, b->transport_name)) - return 0; - - return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) && - a->action == b->action; -} - -HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq) -HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) - -#define clientmap_entry_free(ent) \ - FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent) - -/** Return the size of a client map entry. */ -static inline size_t -clientmap_entry_size(const clientmap_entry_t *ent) -{ - tor_assert(ent); - return (sizeof(clientmap_entry_t) + - (ent->transport_name ? strlen(ent->transport_name) : 0)); -} - -/** Free all storage held by ent. */ -static void -clientmap_entry_free_(clientmap_entry_t *ent) -{ - if (!ent) - return; - - /* This entry is about to be freed so pass it to the DoS subsystem to see if - * any actions can be taken about it. */ - dos_geoip_entry_about_to_free(ent); - geoip_decrement_client_history_cache_size(clientmap_entry_size(ent)); - - tor_free(ent->transport_name); - tor_free(ent); -} - -/* Return a newly allocated clientmap entry with the given action and address - * that are mandatory. The transport_name can be optional. This can't fail. */ -static clientmap_entry_t * -clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr, - const char *transport_name) -{ - clientmap_entry_t *entry; - - tor_assert(action == GEOIP_CLIENT_CONNECT || - action == GEOIP_CLIENT_NETWORKSTATUS); - tor_assert(addr); - - entry = tor_malloc_zero(sizeof(clientmap_entry_t)); - entry->action = action; - tor_addr_copy(&entry->addr, addr); - if (transport_name) { - entry->transport_name = tor_strdup(transport_name); - } - - /* Allocated and initialized, note down its size for the OOM handler. */ - geoip_increment_client_history_cache_size(clientmap_entry_size(entry)); - - return entry; -} - -/** Clear history of connecting clients used by entry and bridge stats. */ -static void -client_history_clear(void) -{ - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; - ent = next) { - if ((*ent)->action == GEOIP_CLIENT_CONNECT) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - clientmap_entry_free(this); - } else { - next = HT_NEXT(clientmap, &client_history, ent); - } - } -} - -/** Note that we've seen a client connect from the IP addr - * at time now. Ignored by all but bridges and directories if - * configured accordingly. */ -void -geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, - const char *transport_name, - time_t now) -{ - const or_options_t *options = get_options(); - clientmap_entry_t *ent; - - if (action == GEOIP_CLIENT_CONNECT) { - /* Only remember statistics if the DoS mitigation subsystem is enabled. If - * not, only if as entry guard or as bridge. */ - if (!dos_enabled()) { - if (!options->EntryStatistics && !should_record_bridge_info(options)) { - return; - } - } - } else { - /* Only gather directory-request statistics if configured, and - * forcibly disable them on bridge authorities. */ - if (!options->DirReqStatistics || options->BridgeAuthoritativeDir) - return; - } - - log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.", - safe_str_client(fmt_addr((addr))), - transport_name ? transport_name : "<no transport>"); - - ent = geoip_lookup_client(addr, transport_name, action); - if (! ent) { - ent = clientmap_entry_new(action, addr, transport_name); - HT_INSERT(clientmap, &client_history, ent); - } - if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0) - ent->last_seen_in_minutes = (unsigned)(now/60); - else - ent->last_seen_in_minutes = 0; - - if (action == GEOIP_CLIENT_NETWORKSTATUS) { - int country_idx = geoip_get_country_by_addr(addr); - if (country_idx < 0) - country_idx = 0; /** unresolved requests are stored at index 0. */ - increment_v3_ns_request(country_idx); - } -} - -/** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's - * older than a certain time. */ -static int -remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff) -{ - time_t cutoff = *(time_t*)_cutoff / 60; - if (ent->last_seen_in_minutes < cutoff) { - clientmap_entry_free(ent); - return 1; - } else { - return 0; - } -} - -/** Forget about all clients that haven't connected since cutoff. */ -void -geoip_remove_old_clients(time_t cutoff) -{ - clientmap_HT_FOREACH_FN(&client_history, - remove_old_client_helper_, - &cutoff); -} - -/* Return a client entry object matching the given address, transport name and - * geoip action from the clientmap. NULL if not found. The transport_name can - * be NULL. */ -clientmap_entry_t * -geoip_lookup_client(const tor_addr_t *addr, const char *transport_name, - geoip_client_action_t action) -{ - clientmap_entry_t lookup; - - tor_assert(addr); - - /* We always look for a client connection with no transport. */ - tor_addr_copy(&lookup.addr, addr); - lookup.action = action; - lookup.transport_name = (char *) transport_name; - - return HT_FIND(clientmap, &client_history, &lookup); -} - -/* Cleanup client entries older than the cutoff. Used for the OOM. Return the - * number of bytes freed. If 0 is returned, nothing was freed. */ -static size_t -oom_clean_client_entries(time_t cutoff) -{ - size_t bytes = 0; - clientmap_entry_t **ent, **ent_next; - - for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) { - clientmap_entry_t *entry = *ent; - if (entry->last_seen_in_minutes < (cutoff / 60)) { - ent_next = HT_NEXT_RMV(clientmap, &client_history, ent); - bytes += clientmap_entry_size(entry); - clientmap_entry_free(entry); - } else { - ent_next = HT_NEXT(clientmap, &client_history, ent); - } - } - return bytes; -} - -/* Below this minimum lifetime, the OOM won't cleanup any entries. */ -#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60) -/* The OOM moves the cutoff by that much every run. */ -#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50) - -/* Cleanup the geoip client history cache called from the OOM handler. Return - * the amount of bytes removed. This can return a value below or above - * min_remove_bytes but will stop as oon as the min_remove_bytes has been - * reached. */ -size_t -geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes) -{ - time_t k; - size_t bytes_removed = 0; - - /* Our OOM handler called with 0 bytes to remove is a code flow error. */ - tor_assert(min_remove_bytes != 0); - - /* Set k to the initial cutoff of an entry. We then going to move it by step - * to try to remove as much as we can. */ - k = WRITE_STATS_INTERVAL; - - do { - time_t cutoff; - - /* If k has reached the minimum lifetime, we have to stop else we might - * remove every single entries which would be pretty bad for the DoS - * mitigation subsystem if by just filling the geoip cache, it was enough - * to trigger the OOM and clean every single entries. */ - if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) { - break; - } - - cutoff = now - k; - bytes_removed += oom_clean_client_entries(cutoff); - k -= GEOIP_CLIENT_CACHE_OOM_STEP; - } while (bytes_removed < min_remove_bytes); - - return bytes_removed; -} - -/* Return the total size in bytes of the client history cache. */ -size_t -geoip_client_cache_total_allocation(void) -{ - return geoip_client_history_cache_size; -} - -/** How many responses are we giving to clients requesting v3 network - * statuses? */ -static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; - -/** Note that we've rejected a client's request for a v3 network status - * for reason reason at time now. */ -void -geoip_note_ns_response(geoip_ns_response_t response) -{ - static int arrays_initialized = 0; - if (!get_options()->DirReqStatistics) - return; - if (!arrays_initialized) { - memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); - arrays_initialized = 1; - } - tor_assert(response < GEOIP_NS_RESPONSE_NUM); - ns_v3_responses[response]++; -} - -/** Do not mention any country from which fewer than this number of IPs have - * connected. This conceivably avoids reporting information that could - * deanonymize users, though analysis is lacking. */ -#define MIN_IPS_TO_NOTE_COUNTRY 1 -/** Do not report any geoip data at all if we have fewer than this number of - * IPs to report about. */ -#define MIN_IPS_TO_NOTE_ANYTHING 1 -/** When reporting geoip data about countries, round up to the nearest - * multiple of this value. */ -#define IP_GRANULARITY 8 - -/** Helper type: used to sort per-country totals by value. */ -typedef struct c_hist_t { - char country[3]; /**< Two-letter country code. */ - unsigned total; /**< Total IP addresses seen in this country. */ -} c_hist_t; - -/** Sorting helper: return -1, 1, or 0 based on comparison of two - * geoip_ipv4_entry_t. Sort in descending order of total, and then by country - * code. */ -static int -c_hist_compare_(const void **_a, const void **_b) -{ - const c_hist_t *a = *_a, *b = *_b; - if (a->total > b->total) - return -1; - else if (a->total < b->total) - return 1; - else - return strcmp(a->country, b->country); -} - -/** When there are incomplete directory requests at the end of a 24-hour - * period, consider those requests running for longer than this timeout as - * failed, the others as still running. */ -#define DIRREQ_TIMEOUT (10*60) - -/** Entry in a map from either chan->global_identifier for direct requests - * or a unique circuit identifier for tunneled requests to request time, - * response size, and completion time of a network status request. Used to - * measure download times of requests to derive average client - * bandwidths. */ -typedef struct dirreq_map_entry_t { - HT_ENTRY(dirreq_map_entry_t) node; - /** Unique identifier for this network status request; this is either the - * chan->global_identifier of the dir channel (direct request) or a new - * locally unique identifier of a circuit (tunneled request). This ID is - * only unique among other direct or tunneled requests, respectively. */ - uint64_t dirreq_id; - unsigned int state:3; /**< State of this directory request. */ - unsigned int type:1; /**< Is this a direct or a tunneled request? */ - unsigned int completed:1; /**< Is this request complete? */ - /** When did we receive the request and started sending the response? */ - struct timeval request_time; - size_t response_size; /**< What is the size of the response in bytes? */ - struct timeval completion_time; /**< When did the request succeed? */ -} dirreq_map_entry_t; - -/** Map of all directory requests asking for v2 or v3 network statuses in - * the current geoip-stats interval. Values are - * of type *dirreq_map_entry_t. */ -static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = - HT_INITIALIZER(); - -static int -dirreq_map_ent_eq(const dirreq_map_entry_t *a, - const dirreq_map_entry_t *b) -{ - return a->dirreq_id == b->dirreq_id && a->type == b->type; -} - -/* DOCDOC dirreq_map_ent_hash */ -static unsigned -dirreq_map_ent_hash(const dirreq_map_entry_t *entry) -{ - unsigned u = (unsigned) entry->dirreq_id; - u += entry->type << 20; - return u; -} - -HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq) -HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) - -/** Helper: Put entry into map of directory requests using - * type and dirreq_id as key parts. If there is - * already an entry for that key, print out a BUG warning and return. */ -static void -dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type, - uint64_t dirreq_id) -{ - dirreq_map_entry_t *old_ent; - tor_assert(entry->type == type); - tor_assert(entry->dirreq_id == dirreq_id); - - /* XXXX we could switch this to HT_INSERT some time, since it seems that - * this bug doesn't happen. But since this function doesn't seem to be - * critical-path, it's sane to leave it alone. */ - old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); - if (old_ent && old_ent != entry) { - log_warn(LD_BUG, "Error when putting directory request into local " - "map. There was already an entry for the same identifier."); - return; - } -} - -/** Helper: Look up and return an entry in the map of directory requests - * using type and dirreq_id as key parts. If there - * is no such entry, return NULL. */ -static dirreq_map_entry_t * -dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id) -{ - dirreq_map_entry_t lookup; - lookup.type = type; - lookup.dirreq_id = dirreq_id; - return HT_FIND(dirreqmap, &dirreq_map, &lookup); -} - -/** Note that an either direct or tunneled (see type) directory - * request for a v3 network status with unique ID dirreq_id of size - * response_size has started. */ -void -geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, - dirreq_type_t type) -{ - dirreq_map_entry_t *ent; - if (!get_options()->DirReqStatistics) - return; - ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); - ent->dirreq_id = dirreq_id; - tor_gettimeofday(&ent->request_time); - ent->response_size = response_size; - ent->type = type; - dirreq_map_put_(ent, type, dirreq_id); -} - -/** Change the state of the either direct or tunneled (see type) - * directory request with dirreq_id to new_state and - * possibly mark it as completed. If no entry can be found for the given - * key parts (e.g., if this is a directory request that we are not - * measuring, or one that was started in the previous measurement period), - * or if the state cannot be advanced to new_state, do nothing. */ -void -geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, - dirreq_state_t new_state) -{ - dirreq_map_entry_t *ent; - if (!get_options()->DirReqStatistics) - return; - ent = dirreq_map_get_(type, dirreq_id); - if (!ent) - return; - if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) - return; - if (new_state - 1 != ent->state) - return; - ent->state = new_state; - if ((type == DIRREQ_DIRECT && - new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || - (type == DIRREQ_TUNNELED && - new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) { - tor_gettimeofday(&ent->completion_time); - ent->completed = 1; - } -} - -/** Return the bridge-ip-transports string that should be inserted in - * our extra-info descriptor. Return NULL if the bridge-ip-transports - * line should be empty. */ -char * -geoip_get_transport_history(void) -{ - unsigned granularity = IP_GRANULARITY; - /** String hash table (name of transport) -> (number of users). */ - strmap_t *transport_counts = strmap_new(); - - /** Smartlist that contains copies of the names of the transports - that have been used. */ - smartlist_t *transports_used = smartlist_new(); - - /* Special string to signify that no transport was used for this - connection. Pluggable transport names can't have symbols in their - names, so this string will never collide with a real transport. */ - static const char* no_transport_str = "<OR>"; - - clientmap_entry_t **ent; - smartlist_t *string_chunks = smartlist_new(); - char *the_string = NULL; - - /* If we haven't seen any clients yet, return NULL. */ - if (HT_EMPTY(&client_history)) - goto done; - - /** We do the following steps to form the transport history string: - * a) Foreach client that uses a pluggable transport, we increase the - * times that transport was used by one. If the client did not use - * a transport, we increase the number of times someone connected - * without obfuscation. - * b) Foreach transport we observed, we write its transport history - * string and push it to string_chunks. So, for example, if we've - * seen 665 obfs2 clients, we write "obfs2=665". - * c) We concatenate string_chunks to form the final string. - */ - - log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.", - HT_SIZE(&client_history)); - - /* Loop through all clients. */ - HT_FOREACH(ent, clientmap, &client_history) { - uintptr_t val; - void *ptr; - const char *transport_name = (*ent)->transport_name; - if (!transport_name) - transport_name = no_transport_str; - - /* Increase the count for this transport name. */ - ptr = strmap_get(transport_counts, transport_name); - val = (uintptr_t)ptr; - val++; - ptr = (void*)val; - strmap_set(transport_counts, transport_name, ptr); - - /* If it's the first time we see this transport, note it. */ - if (val == 1) - smartlist_add_strdup(transports_used, transport_name); - - log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. " - "I've now seen %d clients.", - safe_str_client(fmt_addr(&(*ent)->addr)), - transport_name ? transport_name : "<no transport>", - (int)val); - } - - /* Sort the transport names (helps with unit testing). */ - smartlist_sort_strings(transports_used); - - /* Loop through all seen transports. */ - SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) { - void *transport_count_ptr = strmap_get(transport_counts, transport_name); - uintptr_t transport_count = (uintptr_t) transport_count_ptr; - - log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.", - ((uint64_t)transport_count), transport_name); - - smartlist_add_asprintf(string_chunks, "%s=%"PRIu64, - transport_name, - (round_uint64_to_next_multiple_of( - (uint64_t)transport_count, - granularity))); - } SMARTLIST_FOREACH_END(transport_name); - - the_string = smartlist_join_strings(string_chunks, ",", 0, NULL); - - log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string); - - done: - strmap_free(transport_counts, NULL); - SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s)); - smartlist_free(transports_used); - SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s)); - smartlist_free(string_chunks); - - return the_string; -} - -/** Return a newly allocated comma-separated string containing statistics - * on network status downloads. The string contains the number of completed - * requests, timeouts, and still running requests as well as the download - * times by deciles and quartiles. Return NULL if we have not observed - * requests for long enough. */ -static char * -geoip_get_dirreq_history(dirreq_type_t type) -{ - char *result = NULL; - buf_t *buf = NULL; - smartlist_t *dirreq_completed = NULL; - uint32_t complete = 0, timeouts = 0, running = 0; - dirreq_map_entry_t **ptr, **next; - struct timeval now; - - tor_gettimeofday(&now); - dirreq_completed = smartlist_new(); - for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { - dirreq_map_entry_t *ent = *ptr; - if (ent->type != type) { - next = HT_NEXT(dirreqmap, &dirreq_map, ptr); - continue; - } else { - if (ent->completed) { - smartlist_add(dirreq_completed, ent); - complete++; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); - } else { - if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) - timeouts++; - else - running++; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); - tor_free(ent); - } - } - } -#define DIR_REQ_GRANULARITY 4 - complete = round_uint32_to_next_multiple_of(complete, - DIR_REQ_GRANULARITY); - timeouts = round_uint32_to_next_multiple_of(timeouts, - DIR_REQ_GRANULARITY); - running = round_uint32_to_next_multiple_of(running, - DIR_REQ_GRANULARITY); - buf = buf_new_with_capacity(1024); - buf_add_printf(buf, "complete=%u,timeout=%u," - "running=%u", complete, timeouts, running); - -#define MIN_DIR_REQ_RESPONSES 16 - if (complete >= MIN_DIR_REQ_RESPONSES) { - uint32_t *dltimes; - /* We may have rounded 'completed' up. Here we want to use the - * real value. */ - complete = smartlist_len(dirreq_completed); - dltimes = tor_calloc(complete, sizeof(uint32_t)); - SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { - uint32_t bytes_per_second; - uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time, - &ent->completion_time); - if (time_diff_ == 0) - time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible - * by law of nature or something, but a millisecond - * is a bit greater than "instantly" */ - bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_); - dltimes[ent_sl_idx] = bytes_per_second; - } SMARTLIST_FOREACH_END(ent); - median_uint32(dltimes, complete); /* sorts as a side effect. */ - buf_add_printf(buf, - ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," - "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", - dltimes[0], - dltimes[1*complete/10-1], - dltimes[2*complete/10-1], - dltimes[1*complete/4-1], - dltimes[3*complete/10-1], - dltimes[4*complete/10-1], - dltimes[5*complete/10-1], - dltimes[6*complete/10-1], - dltimes[7*complete/10-1], - dltimes[3*complete/4-1], - dltimes[8*complete/10-1], - dltimes[9*complete/10-1], - dltimes[complete-1]); - tor_free(dltimes); - } - - result = buf_extract(buf, NULL); - - SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, - tor_free(ent)); - smartlist_free(dirreq_completed); - buf_free(buf); - return result; -} - -/** Store a newly allocated comma-separated string in - * *<a>country_str</a> containing entries for all the countries from - * which we've seen enough clients connect as a bridge, directory - * server, or entry guard. The entry format is cc=num where num is the - * number of IPs we've seen connecting from that country, and cc is a - * lowercased country code. *<a>country_str</a> is set to NULL if - * we're not ready to export per country data yet. - * - * Store a newly allocated comma-separated string in <a>ipver_str</a> - * containing entries for clients connecting over IPv4 and IPv6. The - * format is family=num where num is the nubmer of IPs we've seen - * connecting over that protocol family, and family is 'v4' or 'v6'. - * - * Return 0 on success and -1 if we're missing geoip data. */ -int -geoip_get_client_history(geoip_client_action_t action, - char **country_str, char **ipver_str) -{ - unsigned granularity = IP_GRANULARITY; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **cm_ent; - unsigned *counts = NULL; - unsigned total = 0; - unsigned ipv4_count = 0, ipv6_count = 0; - - if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6)) - return -1; - - counts = tor_calloc(n_countries, sizeof(unsigned)); - HT_FOREACH(cm_ent, clientmap, &client_history) { - int country; - if ((*cm_ent)->action != (int)action) - continue; - country = geoip_get_country_by_addr(&(*cm_ent)->addr); - if (country < 0) - country = 0; /** unresolved requests are stored at index 0. */ - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - switch (tor_addr_family(&(*cm_ent)->addr)) { - case AF_INET: - ipv4_count++; - break; - case AF_INET6: - ipv6_count++; - break; - } - } - if (ipver_str) { - smartlist_t *chunks = smartlist_new(); - smartlist_add_asprintf(chunks, "v4=%u", - round_to_next_multiple_of(ipv4_count, granularity)); - smartlist_add_asprintf(chunks, "v6=%u", - round_to_next_multiple_of(ipv6_count, granularity)); - *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - - /* Don't record per country data if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) { - tor_free(counts); - if (country_str) - *country_str = NULL; - return 0; - } - - /* Make a list of c_hist_t */ - entries = smartlist_new(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); - } - } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, c_hist_compare_); - - if (country_str) { - smartlist_t *chunks = smartlist_new(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total); - }); - *country_str = smartlist_join_strings(chunks, ",", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); - tor_free(counts); - - return 0; -} - -/** Return a newly allocated string holding the per-country request history - * for v3 network statuses in a format suitable for an extra-info document, - * or NULL on failure. */ -char * -geoip_get_request_history(void) -{ - smartlist_t *entries, *strings; - char *result; - unsigned granularity = IP_GRANULARITY; - - if (!geoip_countries) - return NULL; - - entries = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) { - uint32_t tot = 0; - c_hist_t *ent; - if ((size_t)c_sl_idx < n_v3_ns_requests_len) - tot = n_v3_ns_requests[c_sl_idx]; - else - tot = 0; - if (!tot) - continue; - ent = tor_malloc_zero(sizeof(c_hist_t)); - strlcpy(ent->country, c->countrycode, sizeof(ent->country)); - ent->total = round_to_next_multiple_of(tot, granularity); - smartlist_add(entries, ent); - } SMARTLIST_FOREACH_END(c); - smartlist_sort(entries, c_hist_compare_); - - strings = smartlist_new(); - SMARTLIST_FOREACH(entries, c_hist_t *, ent, { - smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total); - }); - result = smartlist_join_strings(strings, ",", 0, NULL); - SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent)); - smartlist_free(strings); - smartlist_free(entries); - return result; -} - -/** Start time of directory request stats or 0 if we're not collecting - * directory request statistics. */ -static time_t start_of_dirreq_stats_interval; - -/** Initialize directory request stats. */ -void -geoip_dirreq_stats_init(time_t now) -{ - start_of_dirreq_stats_interval = now; -} - -/** Reset counters for dirreq stats. */ -void -geoip_reset_dirreq_stats(time_t now) -{ - memset(n_v3_ns_requests, 0, - n_v3_ns_requests_len * sizeof(uint32_t)); - { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; - ent = next) { - if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - clientmap_entry_free(this); - } else { - next = HT_NEXT(clientmap, &client_history, ent); - } - } - } - memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); - { - dirreq_map_entry_t **ent, **next, *this; - for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); - tor_free(this); - } - } - start_of_dirreq_stats_interval = now; -} - -/** Stop collecting directory request stats in a way that we can re-start - * doing so in geoip_dirreq_stats_init(). */ -void -geoip_dirreq_stats_term(void) -{ - geoip_reset_dirreq_stats(0); -} - -/** Return a newly allocated string containing the dirreq statistics - * until now, or NULL if we're not collecting dirreq stats. Caller - * must ensure start_of_dirreq_stats_interval is in the past. */ -char * -geoip_format_dirreq_stats(time_t now) -{ - char t[ISO_TIME_LEN+1]; - int i; - char *v3_ips_string = NULL, *v3_reqs_string = NULL, - *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL; - char *result = NULL; - - if (!start_of_dirreq_stats_interval) - return NULL; /* Not initialized. */ - - tor_assert(now >= start_of_dirreq_stats_interval); - - format_iso_time(t, now); - geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL); - v3_reqs_string = geoip_get_request_history(); - -#define RESPONSE_GRANULARITY 8 - for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { - ns_v3_responses[i] = round_uint32_to_next_multiple_of( - ns_v3_responses[i], RESPONSE_GRANULARITY); - } -#undef RESPONSE_GRANULARITY - - v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT); - v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED); - - /* Put everything together into a single string. */ - tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n" - "dirreq-v3-ips %s\n" - "dirreq-v3-reqs %s\n" - "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," - "not-found=%u,not-modified=%u,busy=%u\n" - "dirreq-v3-direct-dl %s\n" - "dirreq-v3-tunneled-dl %s\n", - t, - (unsigned) (now - start_of_dirreq_stats_interval), - v3_ips_string ? v3_ips_string : "", - v3_reqs_string ? v3_reqs_string : "", - ns_v3_responses[GEOIP_SUCCESS], - ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], - ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], - ns_v3_responses[GEOIP_REJECT_NOT_FOUND], - ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], - ns_v3_responses[GEOIP_REJECT_BUSY], - v3_direct_dl_string ? v3_direct_dl_string : "", - v3_tunneled_dl_string ? v3_tunneled_dl_string : ""); - - /* Free partial strings. */ - tor_free(v3_ips_string); - tor_free(v3_reqs_string); - tor_free(v3_direct_dl_string); - tor_free(v3_tunneled_dl_string); - - return result; -} - -/** If 24 hours have passed since the beginning of the current dirreq - * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats - * (possibly overwriting an existing file) and reset counters. Return - * when we would next want to write dirreq stats or 0 if we never want to - * write. */ -time_t -geoip_dirreq_stats_write(time_t now) -{ - char *str = NULL; - - if (!start_of_dirreq_stats_interval) - return 0; /* Not initialized. */ - if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write. */ - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_dirreq_stats_interval); - - /* Generate history string .*/ - str = geoip_format_dirreq_stats(now); - if (! str) - goto done; - - /* Write dirreq-stats string to disk. */ - if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics"); - /* Reset measurement interval start. */ - geoip_reset_dirreq_stats(now); - } - - done: - tor_free(str); - return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; -} - -/** Start time of bridge stats or 0 if we're not collecting bridge - * statistics. */ -static time_t start_of_bridge_stats_interval; - -/** Initialize bridge stats. */ -void -geoip_bridge_stats_init(time_t now) -{ - start_of_bridge_stats_interval = now; -} - -/** Stop collecting bridge stats in a way that we can re-start doing so in - * geoip_bridge_stats_init(). */ -void -geoip_bridge_stats_term(void) -{ - client_history_clear(); - start_of_bridge_stats_interval = 0; -} - -/** Validate a bridge statistics string as it would be written to a - * current extra-info descriptor. Return 1 if the string is valid and - * recent enough, or 0 otherwise. */ -static int -validate_bridge_stats(const char *stats_str, time_t now) -{ - char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], - *eos; - - const char *BRIDGE_STATS_END = "bridge-stats-end "; - const char *BRIDGE_IPS = "bridge-ips "; - const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; - const char *BRIDGE_TRANSPORTS = "bridge-ip-transports "; - const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n"; - const char *tmp; - time_t stats_end_time; - int seconds; - tor_assert(stats_str); - - /* Parse timestamp and number of seconds from - "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); - if (!tmp) - return 0; - tmp += strlen(BRIDGE_STATS_END); - - if (strlen(tmp) < ISO_TIME_LEN + 6) - return 0; - strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); - if (parse_iso_time(stats_end_str, &stats_end_time) < 0) - return 0; - if (stats_end_time < now - (25*60*60) || - stats_end_time > now + (1*60*60)) - return 0; - seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); - if (!eos || seconds < 23*60*60) - return 0; - format_iso_time(stats_start_str, stats_end_time - seconds); - - /* Parse: "bridge-ips CC=N,CC=N,..." */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); - if (!tmp) { - /* Look if there is an empty "bridge-ips" line */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); - if (!tmp) - return 0; - } - - /* Parse: "bridge-ip-transports PT=N,PT=N,..." */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS); - if (!tmp) { - /* Look if there is an empty "bridge-ip-transports" line */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE); - if (!tmp) - return 0; - } - - return 1; -} - -/** Most recent bridge statistics formatted to be written to extra-info - * descriptors. */ -static char *bridge_stats_extrainfo = NULL; - -/** Return a newly allocated string holding our bridge usage stats by country - * in a format suitable for inclusion in an extrainfo document. Return NULL on - * failure. */ -char * -geoip_format_bridge_stats(time_t now) -{ - char *out = NULL; - char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL; - long duration = now - start_of_bridge_stats_interval; - char written[ISO_TIME_LEN+1]; - - if (duration < 0) - return NULL; - if (!start_of_bridge_stats_interval) - return NULL; /* Not initialized. */ - - format_iso_time(written, now); - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); - transport_data = geoip_get_transport_history(); - - tor_asprintf(&out, - "bridge-stats-end %s (%ld s)\n" - "bridge-ips %s\n" - "bridge-ip-versions %s\n" - "bridge-ip-transports %s\n", - written, duration, - country_data ? country_data : "", - ipver_data ? ipver_data : "", - transport_data ? transport_data : ""); - tor_free(country_data); - tor_free(ipver_data); - tor_free(transport_data); - - return out; -} - -/** Return a newly allocated string holding our bridge usage stats by country - * in a format suitable for the answer to a controller request. Return NULL on - * failure. */ -static char * -format_bridge_stats_controller(time_t now) -{ - char *out = NULL, *country_data = NULL, *ipver_data = NULL; - char started[ISO_TIME_LEN+1]; - (void) now; - - format_iso_time(started, start_of_bridge_stats_interval); - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); - - tor_asprintf(&out, - "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s", - started, - country_data ? country_data : "", - ipver_data ? ipver_data : ""); - tor_free(country_data); - tor_free(ipver_data); - return out; -} - -/** Return a newly allocated string holding our bridge usage stats by - * country in a format suitable for inclusion in our heartbeat - * message. Return NULL on failure. */ -char * -format_client_stats_heartbeat(time_t now) -{ - const int n_hours = 6; - char *out = NULL; - int n_clients = 0; - clientmap_entry_t **ent; - unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); - - if (!start_of_bridge_stats_interval) - return NULL; /* Not initialized. */ - - /* count unique IPs */ - HT_FOREACH(ent, clientmap, &client_history) { - /* only count directly connecting clients */ - if ((*ent)->action != GEOIP_CLIENT_CONNECT) - continue; - if ((*ent)->last_seen_in_minutes < cutoff) - continue; - n_clients++; - } - - tor_asprintf(&out, "Heartbeat: " - "In the last %d hours, I have seen %d unique clients.", - n_hours, - n_clients); - - return out; -} - -/** Write bridge statistics to $DATADIR/stats/bridge-stats and return - * when we should next try to write statistics. */ -time_t -geoip_bridge_stats_write(time_t now) -{ - char *val = NULL; - - /* Check if 24 hours have passed since starting measurements. */ - if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) - return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_bridge_stats_interval); - - /* Generate formatted string */ - val = geoip_format_bridge_stats(now); - if (val == NULL) - goto done; - - /* Update the stored value. */ - tor_free(bridge_stats_extrainfo); - bridge_stats_extrainfo = val; - start_of_bridge_stats_interval = now; - - /* Write it to disk. */ - if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "bridge-stats", - bridge_stats_extrainfo, "bridge statistics"); - - /* Tell the controller, "hey, there are clients!" */ - { - char *controller_str = format_bridge_stats_controller(now); - if (controller_str) - control_event_clients_seen(controller_str); - tor_free(controller_str); - } - } - - done: - return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; -} - -/** Try to load the most recent bridge statistics from disk, unless we - * have finished a measurement interval lately, and check whether they - * are still recent enough. */ -static void -load_bridge_stats(time_t now) -{ - char *fname, *contents; - if (bridge_stats_extrainfo) - return; - - fname = get_datadir_fname2("stats", "bridge-stats"); - contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); - if (contents && validate_bridge_stats(contents, now)) { - bridge_stats_extrainfo = contents; - } else { - tor_free(contents); - } - - tor_free(fname); -} - -/** Return most recent bridge statistics for inclusion in extra-info - * descriptors, or NULL if we don't have recent bridge statistics. */ -const char * -geoip_get_bridge_stats_extrainfo(time_t now) -{ - load_bridge_stats(now); - return bridge_stats_extrainfo; -} - -/** Return a new string containing the recent bridge statistics to be returned - * to controller clients, or NULL if we don't have any bridge statistics. */ -char * -geoip_get_bridge_stats_controller(time_t now) -{ - return format_bridge_stats_controller(now); -} - -/** Start time of entry stats or 0 if we're not collecting entry - * statistics. */ -static time_t start_of_entry_stats_interval; - -/** Initialize entry stats. */ -void -geoip_entry_stats_init(time_t now) -{ - start_of_entry_stats_interval = now; -} - -/** Reset counters for entry stats. */ -void -geoip_reset_entry_stats(time_t now) -{ - client_history_clear(); - start_of_entry_stats_interval = now; -} - -/** Stop collecting entry stats in a way that we can re-start doing so in - * geoip_entry_stats_init(). */ -void -geoip_entry_stats_term(void) -{ - geoip_reset_entry_stats(0); -} - -/** Return a newly allocated string containing the entry statistics - * until now, or NULL if we're not collecting entry stats. Caller - * must ensure start_of_entry_stats_interval lies in the past. */ -char * -geoip_format_entry_stats(time_t now) -{ - char t[ISO_TIME_LEN+1]; - char *data = NULL; - char *result; - - if (!start_of_entry_stats_interval) - return NULL; /* Not initialized. */ - - tor_assert(now >= start_of_entry_stats_interval); - - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL); - format_iso_time(t, now); - tor_asprintf(&result, - "entry-stats-end %s (%u s)\n" - "entry-ips %s\n", - t, (unsigned) (now - start_of_entry_stats_interval), - data ? data : ""); - tor_free(data); - return result; -} - -/** If 24 hours have passed since the beginning of the current entry stats - * period, write entry stats to $DATADIR/stats/entry-stats (possibly - * overwriting an existing file) and reset counters. Return when we would - * next want to write entry stats or 0 if we never want to write. */ -time_t -geoip_entry_stats_write(time_t now) -{ - char *str = NULL; - - if (!start_of_entry_stats_interval) - return 0; /* Not initialized. */ - if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write. */ - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_entry_stats_interval); - - /* Generate history string .*/ - str = geoip_format_entry_stats(now); - - /* Write entry-stats string to disk. */ - if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "entry-stats", str, "entry statistics"); - - /* Reset measurement interval start. */ - geoip_reset_entry_stats(now); - } - - done: - tor_free(str); - return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; -} - -/** Helper used to implement GETINFO ip-to-country/... controller command. */ -int -getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - (void)control_conn; - if (!strcmpstart(question, "ip-to-country/")) { - int c; - sa_family_t family; - tor_addr_t addr; - question += strlen("ip-to-country/"); - - if (!strcmp(question, "ipv4-available") || - !strcmp(question, "ipv6-available")) { - family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6; - const int available = geoip_is_loaded(family); - tor_asprintf(answer, "%d", !! available); - return 0; - } - - family = tor_addr_parse(&addr, question); - if (family != AF_INET && family != AF_INET6) { - *errmsg = "Invalid address family"; - return -1; - } - if (!geoip_is_loaded(family)) { - *errmsg = "GeoIP data not loaded"; - return -1; - } - if (family == AF_INET) - c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr)); - else /* AF_INET6 */ - c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr)); - *answer = tor_strdup(geoip_get_country_name(c)); - } - return 0; -} - /** Release all storage held by the GeoIP databases and country list. */ STATIC void clear_geoip_db(void) @@ -1892,28 +500,7 @@ clear_geoip_db(void) void geoip_free_all(void) { - { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - clientmap_entry_free(this); - } - HT_CLEAR(clientmap, &client_history); - } - { - dirreq_map_entry_t **ent, **next, *this; - for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); - tor_free(this); - } - HT_CLEAR(dirreqmap, &dirreq_map); - } - clear_geoip_db(); - tor_free(bridge_stats_extrainfo); - tor_free(n_v3_ns_requests); memset(geoip_digest, 0, sizeof(geoip_digest)); memset(geoip6_digest, 0, sizeof(geoip6_digest)); diff --git a/src/feature/stats/geoip.h b/src/feature/stats/geoip.h index a2175c01e..4adfc10fb 100644 --- a/src/feature/stats/geoip.h +++ b/src/feature/stats/geoip.h @@ -13,96 +13,22 @@ #define TOR_GEOIP_H #include "lib/testsupport/testsupport.h" -#include "core/or/dos.h" - -/** Indicates an action that we might be noting geoip statistics on. - * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing - * the others, we're not. - */ -typedef enum { - /** We've noticed a connection as a bridge relay or entry guard. */ - GEOIP_CLIENT_CONNECT = 0, - /** We've served a networkstatus consensus as a directory server. */ - GEOIP_CLIENT_NETWORKSTATUS = 1, -} geoip_client_action_t; -/** Indicates either a positive reply or a reason for rejectng a network - * status request that will be included in geoip statistics. */ -typedef enum { - /** Request is answered successfully. */ - GEOIP_SUCCESS = 0, - /** V3 network status is not signed by a sufficient number of requested - * authorities. */ - GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, - /** Requested network status object is unavailable. */ - GEOIP_REJECT_UNAVAILABLE = 2, - /** Requested network status not found. */ - GEOIP_REJECT_NOT_FOUND = 3, - /** Network status has not been modified since If-Modified-Since time. */ - GEOIP_REJECT_NOT_MODIFIED = 4, - /** Directory is busy. */ - GEOIP_REJECT_BUSY = 5, -} geoip_ns_response_t; -#define GEOIP_NS_RESPONSE_NUM 6 - -/** Directory requests that we are measuring can be either direct or - * tunneled. */ -typedef enum { - DIRREQ_DIRECT = 0, - DIRREQ_TUNNELED = 1, -} dirreq_type_t; - -/** Possible states for either direct or tunneled directory requests that - * are relevant for determining network status download times. */ -typedef enum { - /** Found that the client requests a network status; applies to both - * direct and tunneled requests; initial state of a request that we are - * measuring. */ - DIRREQ_IS_FOR_NETWORK_STATUS = 0, - /** Finished writing a network status to the directory connection; - * applies to both direct and tunneled requests; completes a direct - * request. */ - DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, - /** END cell sent to circuit that initiated a tunneled request. */ - DIRREQ_END_CELL_SENT = 2, - /** Flushed last cell from queue of the circuit that initiated a - * tunneled request to the outbuf of the OR connection. */ - DIRREQ_CIRC_QUEUE_FLUSHED = 3, - /** Flushed last byte from buffer of the channel belonging to the - * circuit that initiated a tunneled request; completes a tunneled - * request. */ - DIRREQ_CHANNEL_BUFFER_FLUSHED = 4 -} dirreq_state_t; #ifdef GEOIP_PRIVATE STATIC int geoip_parse_entry(const char *line, sa_family_t family); -STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr); -STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); STATIC void clear_geoip_db(void); #endif /* defined(GEOIP_PRIVATE) */ -/** Entry in a map from IP address to the last time we've seen an incoming - * connection from that IP address. Used by bridges only to track which - * countries have them blocked, or the DoS mitigation subsystem if enabled. */ -typedef struct clientmap_entry_t { - HT_ENTRY(clientmap_entry_t) node; - tor_addr_t addr; - /* Name of pluggable transport used by this client. NULL if no - pluggable transport was used. */ - char *transport_name; +int geoip_get_country_by_ipv4(uint32_t ipaddr); +int geoip_get_country_by_ipv6(const struct in6_addr *addr); - /** Time when we last saw this IP address, in MINUTES since the epoch. - * - * (This will run out of space around 4011 CE. If Tor is still in use around - * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */ - unsigned int last_seen_in_minutes:30; - unsigned int action:2; +/** A per-country GeoIP record. */ +typedef struct geoip_country_t { + char countrycode[3]; +} geoip_country_t; - /* This object is used to keep some statistics per client address for the - * DoS mitigation subsystem. */ - dos_client_stats_t dos_stats; -} clientmap_entry_t; +const smartlist_t *geoip_get_countries(void); -int should_record_bridge_info(const or_options_t *options); int geoip_load_file(sa_family_t family, const char *filename); MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); MOCK_DECL(int, geoip_get_n_countries, (void)); @@ -111,47 +37,6 @@ MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); const char *geoip_db_digest(sa_family_t family); MOCK_DECL(country_t, geoip_get_country, (const char *countrycode)); -void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, const char *transport_name, - time_t now); -void geoip_remove_old_clients(time_t cutoff); -clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr, - const char *transport_name, - geoip_client_action_t action); -size_t geoip_client_cache_total_allocation(void); -size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes); - -void geoip_note_ns_response(geoip_ns_response_t response); -char *geoip_get_transport_history(void); -int geoip_get_client_history(geoip_client_action_t action, - char **country_str, char **ipver_str); -char *geoip_get_request_history(void); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg); void geoip_free_all(void); -void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, - dirreq_type_t type); -void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, - dirreq_state_t new_state); - -void geoip_dirreq_stats_init(time_t now); -void geoip_reset_dirreq_stats(time_t now); -char *geoip_format_dirreq_stats(time_t now); -time_t geoip_dirreq_stats_write(time_t now); -void geoip_dirreq_stats_term(void); -void geoip_entry_stats_init(time_t now); -time_t geoip_entry_stats_write(time_t now); -void geoip_entry_stats_term(void); -void geoip_reset_entry_stats(time_t now); -char *geoip_format_entry_stats(time_t now); -void geoip_bridge_stats_init(time_t now); -char *geoip_format_bridge_stats(time_t now); -time_t geoip_bridge_stats_write(time_t now); -void geoip_bridge_stats_term(void); -const char *geoip_get_bridge_stats_extrainfo(time_t); -char *geoip_get_bridge_stats_controller(time_t); -char *format_client_stats_heartbeat(time_t now); - #endif /* !defined(TOR_GEOIP_H) */ diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c new file mode 100644 index 000000000..4af2ff0ec --- /dev/null +++ b/src/feature/stats/geoip_stats.c @@ -0,0 +1,1422 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.c + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. + * + * There are two main kinds of functions in this module: geoip functions, + * which map groups of IPv4 and IPv6 addresses to country codes, and + * statistical functions, which collect statistics about different kinds of + * per-country usage. + * + * The geoip lookup tables are implemented as sorted lists of disjoint address + * ranges, each mapping to a singleton geoip_country_t. These country objects + * are also indexed by their names in a hashtable. + * + * The tables are populated from disk at startup by the geoip_load_file() + * function. For more information on the file format they read, see that + * function. See the scripts and the README file in src/config for more + * information about how those files are generated. + * + * Tor uses GeoIP information in order to implement user requests (such as + * ExcludeNodes {cc}), and to keep track of how much usage relays are getting + * for each country. + */ + +#include "core/or/or.h" + +#include "ht.h" +#include "lib/container/buffers.h" +#include "app/config/config.h" +#include "feature/control/control.h" +#include "feature/client/dnsserv.h" +#include "core/or/dos.h" +#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" +#include "feature/nodelist/routerlist.h" + +#include "lib/container/order.h" +#include "lib/time/tvdiff.h" + +/** Number of entries in n_v3_ns_requests */ +size_t n_v3_ns_requests_len = 0; +/** Array, indexed by country index, of number of v3 networkstatus requests + * received from that country */ +static uint32_t *n_v3_ns_requests; + +/* Total size in bytes of the geoip client history cache. Used by the OOM + * handler. */ +static size_t geoip_client_history_cache_size; + +/* Increment the geoip client history cache size counter with the given bytes. + * This prevents an overflow and set it to its maximum in that case. */ +static inline void +geoip_increment_client_history_cache_size(size_t bytes) +{ + /* This is shockingly high, lets log it so it can be reported. */ + IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) { + geoip_client_history_cache_size = SIZE_MAX; + return; + } + geoip_client_history_cache_size += bytes; +} + +/* Decrement the geoip client history cache size counter with the given bytes. + * This prevents an underflow and set it to 0 in that case. */ +static inline void +geoip_decrement_client_history_cache_size(size_t bytes) +{ + /* Going below 0 means that we either allocated an entry without + * incrementing the counter or we have different sizes when allocating and + * freeing. It shouldn't happened so log it. */ + IF_BUG_ONCE(geoip_client_history_cache_size < bytes) { + geoip_client_history_cache_size = 0; + return; + } + geoip_client_history_cache_size -= bytes; +} + +/** Add 1 to the count of v3 ns requests received from country. */ +static void +increment_v3_ns_request(country_t country) +{ + if (country < 0) + return; + + if ((size_t)country >= n_v3_ns_requests_len) { + /* We need to reallocate the array. */ + size_t new_len; + if (n_v3_ns_requests_len == 0) + new_len = 256; + else + new_len = n_v3_ns_requests_len * 2; + if (new_len <= (size_t)country) + new_len = ((size_t)country)+1; + n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len, + sizeof(uint32_t)); + memset(n_v3_ns_requests + n_v3_ns_requests_len, 0, + sizeof(uint32_t)*(new_len - n_v3_ns_requests_len)); + n_v3_ns_requests_len = new_len; + } + + n_v3_ns_requests[country] += 1; +} + +/** Return 1 if we should collect geoip stats on bridge users, and + * include them in our extrainfo descriptor. Else return 0. */ +int +should_record_bridge_info(const or_options_t *options) +{ + return options->BridgeRelay && options->BridgeRecordUsageByCountry; +} + +/** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, + * so it can hold up to (1u<<30)-1, or 0x3fffffffu. + */ +#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu + +/** Map from client IP address to last time seen. */ +static HT_HEAD(clientmap, clientmap_entry_t) client_history = + HT_INITIALIZER(); + +/** Hashtable helper: compute a hash of a clientmap_entry_t. */ +static inline unsigned +clientmap_entry_hash(const clientmap_entry_t *a) +{ + unsigned h = (unsigned) tor_addr_hash(&a->addr); + + if (a->transport_name) + h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name)); + + return h; +} +/** Hashtable helper: compare two clientmap_entry_t values for equality. */ +static inline int +clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) +{ + if (strcmp_opt(a->transport_name, b->transport_name)) + return 0; + + return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) && + a->action == b->action; +} + +HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, + clientmap_entries_eq) +HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, + clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) + +#define clientmap_entry_free(ent) \ + FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent) + +/** Return the size of a client map entry. */ +static inline size_t +clientmap_entry_size(const clientmap_entry_t *ent) +{ + tor_assert(ent); + return (sizeof(clientmap_entry_t) + + (ent->transport_name ? strlen(ent->transport_name) : 0)); +} + +/** Free all storage held by ent. */ +static void +clientmap_entry_free_(clientmap_entry_t *ent) +{ + if (!ent) + return; + + /* This entry is about to be freed so pass it to the DoS subsystem to see if + * any actions can be taken about it. */ + dos_geoip_entry_about_to_free(ent); + geoip_decrement_client_history_cache_size(clientmap_entry_size(ent)); + + tor_free(ent->transport_name); + tor_free(ent); +} + +/* Return a newly allocated clientmap entry with the given action and address + * that are mandatory. The transport_name can be optional. This can't fail. */ +static clientmap_entry_t * +clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr, + const char *transport_name) +{ + clientmap_entry_t *entry; + + tor_assert(action == GEOIP_CLIENT_CONNECT || + action == GEOIP_CLIENT_NETWORKSTATUS); + tor_assert(addr); + + entry = tor_malloc_zero(sizeof(clientmap_entry_t)); + entry->action = action; + tor_addr_copy(&entry->addr, addr); + if (transport_name) { + entry->transport_name = tor_strdup(transport_name); + } + + /* Allocated and initialized, note down its size for the OOM handler. */ + geoip_increment_client_history_cache_size(clientmap_entry_size(entry)); + + return entry; +} + +/** Clear history of connecting clients used by entry and bridge stats. */ +static void +client_history_clear(void) +{ + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_CONNECT) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + clientmap_entry_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } +} + +/** Note that we've seen a client connect from the IP addr + * at time now. Ignored by all but bridges and directories if + * configured accordingly. */ +void +geoip_note_client_seen(geoip_client_action_t action, + const tor_addr_t *addr, + const char *transport_name, + time_t now) +{ + const or_options_t *options = get_options(); + clientmap_entry_t *ent; + + if (action == GEOIP_CLIENT_CONNECT) { + /* Only remember statistics if the DoS mitigation subsystem is enabled. If + * not, only if as entry guard or as bridge. */ + if (!dos_enabled()) { + if (!options->EntryStatistics && !should_record_bridge_info(options)) { + return; + } + } + } else { + /* Only gather directory-request statistics if configured, and + * forcibly disable them on bridge authorities. */ + if (!options->DirReqStatistics || options->BridgeAuthoritativeDir) + return; + } + + log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.", + safe_str_client(fmt_addr((addr))), + transport_name ? transport_name : "<no transport>"); + + ent = geoip_lookup_client(addr, transport_name, action); + if (! ent) { + ent = clientmap_entry_new(action, addr, transport_name); + HT_INSERT(clientmap, &client_history, ent); + } + if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0) + ent->last_seen_in_minutes = (unsigned)(now/60); + else + ent->last_seen_in_minutes = 0; + + if (action == GEOIP_CLIENT_NETWORKSTATUS) { + int country_idx = geoip_get_country_by_addr(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ + increment_v3_ns_request(country_idx); + } +} + +/** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's + * older than a certain time. */ +static int +remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff) +{ + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { + clientmap_entry_free(ent); + return 1; + } else { + return 0; + } +} + +/** Forget about all clients that haven't connected since cutoff. */ +void +geoip_remove_old_clients(time_t cutoff) +{ + clientmap_HT_FOREACH_FN(&client_history, + remove_old_client_helper_, + &cutoff); +} + +/* Return a client entry object matching the given address, transport name and + * geoip action from the clientmap. NULL if not found. The transport_name can + * be NULL. */ +clientmap_entry_t * +geoip_lookup_client(const tor_addr_t *addr, const char *transport_name, + geoip_client_action_t action) +{ + clientmap_entry_t lookup; + + tor_assert(addr); + + /* We always look for a client connection with no transport. */ + tor_addr_copy(&lookup.addr, addr); + lookup.action = action; + lookup.transport_name = (char *) transport_name; + + return HT_FIND(clientmap, &client_history, &lookup); +} + +/* Cleanup client entries older than the cutoff. Used for the OOM. Return the + * number of bytes freed. If 0 is returned, nothing was freed. */ +static size_t +oom_clean_client_entries(time_t cutoff) +{ + size_t bytes = 0; + clientmap_entry_t **ent, **ent_next; + + for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) { + clientmap_entry_t *entry = *ent; + if (entry->last_seen_in_minutes < (cutoff / 60)) { + ent_next = HT_NEXT_RMV(clientmap, &client_history, ent); + bytes += clientmap_entry_size(entry); + clientmap_entry_free(entry); + } else { + ent_next = HT_NEXT(clientmap, &client_history, ent); + } + } + return bytes; +} + +/* Below this minimum lifetime, the OOM won't cleanup any entries. */ +#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60) +/* The OOM moves the cutoff by that much every run. */ +#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50) + +/* Cleanup the geoip client history cache called from the OOM handler. Return + * the amount of bytes removed. This can return a value below or above + * min_remove_bytes but will stop as oon as the min_remove_bytes has been + * reached. */ +size_t +geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes) +{ + time_t k; + size_t bytes_removed = 0; + + /* Our OOM handler called with 0 bytes to remove is a code flow error. */ + tor_assert(min_remove_bytes != 0); + + /* Set k to the initial cutoff of an entry. We then going to move it by step + * to try to remove as much as we can. */ + k = WRITE_STATS_INTERVAL; + + do { + time_t cutoff; + + /* If k has reached the minimum lifetime, we have to stop else we might + * remove every single entries which would be pretty bad for the DoS + * mitigation subsystem if by just filling the geoip cache, it was enough + * to trigger the OOM and clean every single entries. */ + if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) { + break; + } + + cutoff = now - k; + bytes_removed += oom_clean_client_entries(cutoff); + k -= GEOIP_CLIENT_CACHE_OOM_STEP; + } while (bytes_removed < min_remove_bytes); + + return bytes_removed; +} + +/* Return the total size in bytes of the client history cache. */ +size_t +geoip_client_cache_total_allocation(void) +{ + return geoip_client_history_cache_size; +} + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v3 network status + * for reason reason at time now. */ +void +geoip_note_ns_response(geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + ns_v3_responses[response]++; +} + +/** Do not mention any country from which fewer than this number of IPs have + * connected. This conceivably avoids reporting information that could + * deanonymize users, though analysis is lacking. */ +#define MIN_IPS_TO_NOTE_COUNTRY 1 +/** Do not report any geoip data at all if we have fewer than this number of + * IPs to report about. */ +#define MIN_IPS_TO_NOTE_ANYTHING 1 +/** When reporting geoip data about countries, round up to the nearest + * multiple of this value. */ +#define IP_GRANULARITY 8 + +/** Helper type: used to sort per-country totals by value. */ +typedef struct c_hist_t { + char country[3]; /**< Two-letter country code. */ + unsigned total; /**< Total IP addresses seen in this country. */ +} c_hist_t; + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * geoip_ipv4_entry_t. Sort in descending order of total, and then by country + * code. */ +static int +c_hist_compare_(const void **_a, const void **_b) +{ + const c_hist_t *a = *_a, *b = *_b; + if (a->total > b->total) + return -1; + else if (a->total < b->total) + return 1; + else + return strcmp(a->country, b->country); +} + +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) + +/** Entry in a map from either chan->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * chan->global_identifier of the dir channel (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *dirreq_map_entry_t. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) +{ + return a->dirreq_id == b->dirreq_id && a->type == b->type; +} + +/* DOCDOC dirreq_map_ent_hash */ +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq) +HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) + +/** Helper: Put entry into map of directory requests using + * type and dirreq_id as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX we could switch this to HT_INSERT some time, since it seems that + * this bug doesn't happen. But since this function doesn't seem to be + * critical-path, it's sane to leave it alone. */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using type and dirreq_id as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see type) directory + * request for a v3 network status with unique ID dirreq_id of size + * response_size has started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->type = type; + dirreq_map_put_(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see type) + * directory request with dirreq_id to new_state and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to new_state, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = dirreq_map_get_(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return the bridge-ip-transports string that should be inserted in + * our extra-info descriptor. Return NULL if the bridge-ip-transports + * line should be empty. */ +char * +geoip_get_transport_history(void) +{ + unsigned granularity = IP_GRANULARITY; + /** String hash table (name of transport) -> (number of users). */ + strmap_t *transport_counts = strmap_new(); + + /** Smartlist that contains copies of the names of the transports + that have been used. */ + smartlist_t *transports_used = smartlist_new(); + + /* Special string to signify that no transport was used for this + connection. Pluggable transport names can't have symbols in their + names, so this string will never collide with a real transport. */ + static const char* no_transport_str = "<OR>"; + + clientmap_entry_t **ent; + smartlist_t *string_chunks = smartlist_new(); + char *the_string = NULL; + + /* If we haven't seen any clients yet, return NULL. */ + if (HT_EMPTY(&client_history)) + goto done; + + /** We do the following steps to form the transport history string: + * a) Foreach client that uses a pluggable transport, we increase the + * times that transport was used by one. If the client did not use + * a transport, we increase the number of times someone connected + * without obfuscation. + * b) Foreach transport we observed, we write its transport history + * string and push it to string_chunks. So, for example, if we've + * seen 665 obfs2 clients, we write "obfs2=665". + * c) We concatenate string_chunks to form the final string. + */ + + log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.", + HT_SIZE(&client_history)); + + /* Loop through all clients. */ + HT_FOREACH(ent, clientmap, &client_history) { + uintptr_t val; + void *ptr; + const char *transport_name = (*ent)->transport_name; + if (!transport_name) + transport_name = no_transport_str; + + /* Increase the count for this transport name. */ + ptr = strmap_get(transport_counts, transport_name); + val = (uintptr_t)ptr; + val++; + ptr = (void*)val; + strmap_set(transport_counts, transport_name, ptr); + + /* If it's the first time we see this transport, note it. */ + if (val == 1) + smartlist_add_strdup(transports_used, transport_name); + + log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. " + "I've now seen %d clients.", + safe_str_client(fmt_addr(&(*ent)->addr)), + transport_name ? transport_name : "<no transport>", + (int)val); + } + + /* Sort the transport names (helps with unit testing). */ + smartlist_sort_strings(transports_used); + + /* Loop through all seen transports. */ + SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) { + void *transport_count_ptr = strmap_get(transport_counts, transport_name); + uintptr_t transport_count = (uintptr_t) transport_count_ptr; + + log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.", + ((uint64_t)transport_count), transport_name); + + smartlist_add_asprintf(string_chunks, "%s=%"PRIu64, + transport_name, + (round_uint64_to_next_multiple_of( + (uint64_t)transport_count, + granularity))); + } SMARTLIST_FOREACH_END(transport_name); + + the_string = smartlist_join_strings(string_chunks, ",", 0, NULL); + + log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string); + + done: + strmap_free(transport_counts, NULL); + SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s)); + smartlist_free(transports_used); + SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s)); + smartlist_free(string_chunks); + + return the_string; +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(dirreq_type_t type) +{ + char *result = NULL; + buf_t *buf = NULL; + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + dirreq_map_entry_t **ptr, **next; + struct timeval now; + + tor_gettimeofday(&now); + dirreq_completed = smartlist_new(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + dirreq_map_entry_t *ent = *ptr; + if (ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); + } + } + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + buf = buf_new_with_capacity(1024); + buf_add_printf(buf, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_calloc(complete, sizeof(uint32_t)); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff_ == 0) + time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a millisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + buf_add_printf(buf, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + tor_free(dltimes); + } + + result = buf_extract(buf, NULL); + + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + buf_free(buf); + return result; +} + +/** Store a newly allocated comma-separated string in + * *<a>country_str</a> containing entries for all the countries from + * which we've seen enough clients connect as a bridge, directory + * server, or entry guard. The entry format is cc=num where num is the + * number of IPs we've seen connecting from that country, and cc is a + * lowercased country code. *<a>country_str</a> is set to NULL if + * we're not ready to export per country data yet. + * + * Store a newly allocated comma-separated string in <a>ipver_str</a> + * containing entries for clients connecting over IPv4 and IPv6. The + * format is family=num where num is the nubmer of IPs we've seen + * connecting over that protocol family, and family is 'v4' or 'v6'. + * + * Return 0 on success and -1 if we're missing geoip data. */ +int +geoip_get_client_history(geoip_client_action_t action, + char **country_str, char **ipver_str) +{ + unsigned granularity = IP_GRANULARITY; + smartlist_t *entries = NULL; + int n_countries = geoip_get_n_countries(); + int i; + clientmap_entry_t **cm_ent; + unsigned *counts = NULL; + unsigned total = 0; + unsigned ipv4_count = 0, ipv6_count = 0; + + if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6)) + return -1; + + counts = tor_calloc(n_countries, sizeof(unsigned)); + HT_FOREACH(cm_ent, clientmap, &client_history) { + int country; + if ((*cm_ent)->action != (int)action) + continue; + country = geoip_get_country_by_addr(&(*cm_ent)->addr); + if (country < 0) + country = 0; /** unresolved requests are stored at index 0. */ + tor_assert(0 <= country && country < n_countries); + ++counts[country]; + ++total; + switch (tor_addr_family(&(*cm_ent)->addr)) { + case AF_INET: + ipv4_count++; + break; + case AF_INET6: + ipv6_count++; + break; + } + } + if (ipver_str) { + smartlist_t *chunks = smartlist_new(); + smartlist_add_asprintf(chunks, "v4=%u", + round_to_next_multiple_of(ipv4_count, granularity)); + smartlist_add_asprintf(chunks, "v6=%u", + round_to_next_multiple_of(ipv6_count, granularity)); + *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL); + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + + /* Don't record per country data if we haven't seen enough IPs. */ + if (total < MIN_IPS_TO_NOTE_ANYTHING) { + tor_free(counts); + if (country_str) + *country_str = NULL; + return 0; + } + + /* Make a list of c_hist_t */ + entries = smartlist_new(); + for (i = 0; i < n_countries; ++i) { + unsigned c = counts[i]; + const char *countrycode; + c_hist_t *ent; + /* Only report a country if it has a minimum number of IPs. */ + if (c >= MIN_IPS_TO_NOTE_COUNTRY) { + c = round_to_next_multiple_of(c, granularity); + countrycode = geoip_get_country_name(i); + ent = tor_malloc(sizeof(c_hist_t)); + strlcpy(ent->country, countrycode, sizeof(ent->country)); + ent->total = c; + smartlist_add(entries, ent); + } + } + /* Sort entries. Note that we must do this _AFTER_ rounding, or else + * the sort order could leak info. */ + smartlist_sort(entries, c_hist_compare_); + + if (country_str) { + smartlist_t *chunks = smartlist_new(); + SMARTLIST_FOREACH(entries, c_hist_t *, ch, { + smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total); + }); + *country_str = smartlist_join_strings(chunks, ",", 0, NULL); + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + + SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); + smartlist_free(entries); + tor_free(counts); + + return 0; +} + +/** Return a newly allocated string holding the per-country request history + * for v3 network statuses in a format suitable for an extra-info document, + * or NULL on failure. */ +char * +geoip_get_request_history(void) +{ + smartlist_t *entries, *strings; + char *result; + unsigned granularity = IP_GRANULARITY; + + entries = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(geoip_get_countries(), const geoip_country_t *, c) { + uint32_t tot = 0; + c_hist_t *ent; + if ((size_t)c_sl_idx < n_v3_ns_requests_len) + tot = n_v3_ns_requests[c_sl_idx]; + else + tot = 0; + if (!tot) + continue; + ent = tor_malloc_zero(sizeof(c_hist_t)); + strlcpy(ent->country, c->countrycode, sizeof(ent->country)); + ent->total = round_to_next_multiple_of(tot, granularity); + smartlist_add(entries, ent); + } SMARTLIST_FOREACH_END(c); + smartlist_sort(entries, c_hist_compare_); + + strings = smartlist_new(); + SMARTLIST_FOREACH(entries, c_hist_t *, ent, { + smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total); + }); + result = smartlist_join_strings(strings, ",", 0, NULL); + SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent)); + smartlist_free(strings); + smartlist_free(entries); + return result; +} + +/** Start time of directory request stats or 0 if we're not collecting + * directory request statistics. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ +void +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + +/** Reset counters for dirreq stats. */ +void +geoip_reset_dirreq_stats(time_t now) +{ + memset(n_v3_ns_requests, 0, + n_v3_ns_requests_len * sizeof(uint32_t)); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + clientmap_entry_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } + } + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + } + start_of_dirreq_stats_interval = now; +} + +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ +void +geoip_dirreq_stats_term(void) +{ + geoip_reset_dirreq_stats(0); +} + +/** Return a newly allocated string containing the dirreq statistics + * until now, or NULL if we're not collecting dirreq stats. Caller + * must ensure start_of_dirreq_stats_interval is in the past. */ +char * +geoip_format_dirreq_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + int i; + char *v3_ips_string = NULL, *v3_reqs_string = NULL, + *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL; + char *result = NULL; + + if (!start_of_dirreq_stats_interval) + return NULL; /* Not initialized. */ + + tor_assert(now >= start_of_dirreq_stats_interval); + + format_iso_time(t, now); + geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL); + v3_reqs_string = geoip_get_request_history(); + +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + + v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT); + v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED); + + /* Put everything together into a single string. */ + tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n" + "dirreq-v3-ips %s\n" + "dirreq-v3-reqs %s\n" + "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n" + "dirreq-v3-direct-dl %s\n" + "dirreq-v3-tunneled-dl %s\n", + t, + (unsigned) (now - start_of_dirreq_stats_interval), + v3_ips_string ? v3_ips_string : "", + v3_reqs_string ? v3_reqs_string : "", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY], + v3_direct_dl_string ? v3_direct_dl_string : "", + v3_tunneled_dl_string ? v3_tunneled_dl_string : ""); + + /* Free partial strings. */ + tor_free(v3_ips_string); + tor_free(v3_reqs_string); + tor_free(v3_direct_dl_string); + tor_free(v3_tunneled_dl_string); + + return result; +} + +/** If 24 hours have passed since the beginning of the current dirreq + * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write dirreq stats or 0 if we never want to + * write. */ +time_t +geoip_dirreq_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + + /* Generate history string .*/ + str = geoip_format_dirreq_stats(now); + if (! str) + goto done; + + /* Write dirreq-stats string to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics"); + /* Reset measurement interval start. */ + geoip_reset_dirreq_stats(now); + } + + done: + tor_free(str); + return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Start time of bridge stats or 0 if we're not collecting bridge + * statistics. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Stop collecting bridge stats in a way that we can re-start doing so in + * geoip_bridge_stats_init(). */ +void +geoip_bridge_stats_term(void) +{ + client_history_clear(); + start_of_bridge_stats_interval = 0; +} + +/** Validate a bridge statistics string as it would be written to a + * current extra-info descriptor. Return 1 if the string is valid and + * recent enough, or 0 otherwise. */ +static int +validate_bridge_stats(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *eos; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *BRIDGE_TRANSPORTS = "bridge-ip-transports "; + const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n"; + const char *tmp; + time_t stats_end_time; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return 0; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return 0; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return 0; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return 0; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return 0; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (!tmp) { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return 0; + } + + /* Parse: "bridge-ip-transports PT=N,PT=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS); + if (!tmp) { + /* Look if there is an empty "bridge-ip-transports" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE); + if (!tmp) + return 0; + } + + return 1; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for inclusion in an extrainfo document. Return NULL on + * failure. */ +char * +geoip_format_bridge_stats(time_t now) +{ + char *out = NULL; + char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL; + long duration = now - start_of_bridge_stats_interval; + char written[ISO_TIME_LEN+1]; + + if (duration < 0) + return NULL; + if (!start_of_bridge_stats_interval) + return NULL; /* Not initialized. */ + + format_iso_time(written, now); + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); + transport_data = geoip_get_transport_history(); + + tor_asprintf(&out, + "bridge-stats-end %s (%ld s)\n" + "bridge-ips %s\n" + "bridge-ip-versions %s\n" + "bridge-ip-transports %s\n", + written, duration, + country_data ? country_data : "", + ipver_data ? ipver_data : "", + transport_data ? transport_data : ""); + tor_free(country_data); + tor_free(ipver_data); + tor_free(transport_data); + + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for the answer to a controller request. Return NULL on + * failure. */ +static char * +format_bridge_stats_controller(time_t now) +{ + char *out = NULL, *country_data = NULL, *ipver_data = NULL; + char started[ISO_TIME_LEN+1]; + (void) now; + + format_iso_time(started, start_of_bridge_stats_interval); + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); + + tor_asprintf(&out, + "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s", + started, + country_data ? country_data : "", + ipver_data ? ipver_data : ""); + tor_free(country_data); + tor_free(ipver_data); + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by + * country in a format suitable for inclusion in our heartbeat + * message. Return NULL on failure. */ +char * +format_client_stats_heartbeat(time_t now) +{ + const int n_hours = 6; + char *out = NULL; + int n_clients = 0; + clientmap_entry_t **ent; + unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + + if (!start_of_bridge_stats_interval) + return NULL; /* Not initialized. */ + + /* count unique IPs */ + HT_FOREACH(ent, clientmap, &client_history) { + /* only count directly connecting clients */ + if ((*ent)->action != GEOIP_CLIENT_CONNECT) + continue; + if ((*ent)->last_seen_in_minutes < cutoff) + continue; + n_clients++; + } + + tor_asprintf(&out, "Heartbeat: " + "In the last %d hours, I have seen %d unique clients.", + n_hours, + n_clients); + + return out; +} + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *val = NULL; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + /* Generate formatted string */ + val = geoip_format_bridge_stats(now); + if (val == NULL) + goto done; + + /* Update the stored value. */ + tor_free(bridge_stats_extrainfo); + bridge_stats_extrainfo = val; + start_of_bridge_stats_interval = now; + + /* Write it to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "bridge-stats", + bridge_stats_extrainfo, "bridge statistics"); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } + } + + done: + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately, and check whether they + * are still recent enough. */ +static void +load_bridge_stats(time_t now) +{ + char *fname, *contents; + if (bridge_stats_extrainfo) + return; + + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents && validate_bridge_stats(contents, now)) { + bridge_stats_extrainfo = contents; + } else { + tor_free(contents); + } + + tor_free(fname); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return a new string containing the recent bridge statistics to be returned + * to controller clients, or NULL if we don't have any bridge statistics. */ +char * +geoip_get_bridge_stats_controller(time_t now) +{ + return format_bridge_stats_controller(now); +} + +/** Start time of entry stats or 0 if we're not collecting entry + * statistics. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Reset counters for entry stats. */ +void +geoip_reset_entry_stats(time_t now) +{ + client_history_clear(); + start_of_entry_stats_interval = now; +} + +/** Stop collecting entry stats in a way that we can re-start doing so in + * geoip_entry_stats_init(). */ +void +geoip_entry_stats_term(void) +{ + geoip_reset_entry_stats(0); +} + +/** Return a newly allocated string containing the entry statistics + * until now, or NULL if we're not collecting entry stats. Caller + * must ensure start_of_entry_stats_interval lies in the past. */ +char * +geoip_format_entry_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + char *data = NULL; + char *result; + + if (!start_of_entry_stats_interval) + return NULL; /* Not initialized. */ + + tor_assert(now >= start_of_entry_stats_interval); + + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL); + format_iso_time(t, now); + tor_asprintf(&result, + "entry-stats-end %s (%u s)\n" + "entry-ips %s\n", + t, (unsigned) (now - start_of_entry_stats_interval), + data ? data : ""); + tor_free(data); + return result; +} + +/** If 24 hours have passed since the beginning of the current entry stats + * period, write entry stats to $DATADIR/stats/entry-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write entry stats or 0 if we never want to write. */ +time_t +geoip_entry_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_entry_stats_interval) + return 0; /* Not initialized. */ + if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + /* Generate history string .*/ + str = geoip_format_entry_stats(now); + + /* Write entry-stats string to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "entry-stats", str, "entry statistics"); + + /* Reset measurement interval start. */ + geoip_reset_entry_stats(now); + } + + done: + tor_free(str); + return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Release all storage held in this file. */ +void +geoip_stats_free_all(void) +{ + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + clientmap_entry_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); + } + + tor_free(bridge_stats_extrainfo); + tor_free(n_v3_ns_requests); +} diff --git a/src/feature/stats/geoip_stats.h b/src/feature/stats/geoip_stats.h new file mode 100644 index 000000000..97011d62c --- /dev/null +++ b/src/feature/stats/geoip_stats.h @@ -0,0 +1,139 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip_stats.h + * \brief Header file for geoip_stats.c. + **/ + +#ifndef TOR_GEOIP_STATS_H +#define TOR_GEOIP_STATS_H + +#include "core/or/dos.h" + +/** Indicates an action that we might be noting geoip statistics on. + * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing + * the others, we're not. + */ +typedef enum { + /** We've noticed a connection as a bridge relay or entry guard. */ + GEOIP_CLIENT_CONNECT = 0, + /** We've served a networkstatus consensus as a directory server. */ + GEOIP_CLIENT_NETWORKSTATUS = 1, +} geoip_client_action_t; +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; + +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the channel belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_CHANNEL_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +/** Entry in a map from IP address to the last time we've seen an incoming + * connection from that IP address. Used by bridges only to track which + * countries have them blocked, or the DoS mitigation subsystem if enabled. */ +typedef struct clientmap_entry_t { + HT_ENTRY(clientmap_entry_t) node; + tor_addr_t addr; + /* Name of pluggable transport used by this client. NULL if no + pluggable transport was used. */ + char *transport_name; + + /** Time when we last saw this IP address, in MINUTES since the epoch. + * + * (This will run out of space around 4011 CE. If Tor is still in use around + * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; + + /* This object is used to keep some statistics per client address for the + * DoS mitigation subsystem. */ + dos_client_stats_t dos_stats; +} clientmap_entry_t; + +int should_record_bridge_info(const or_options_t *options); + +void geoip_note_client_seen(geoip_client_action_t action, + const tor_addr_t *addr, const char *transport_name, + time_t now); +void geoip_remove_old_clients(time_t cutoff); +clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr, + const char *transport_name, + geoip_client_action_t action); +size_t geoip_client_cache_total_allocation(void); +size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes); + +void geoip_note_ns_response(geoip_ns_response_t response); +char *geoip_get_transport_history(void); +int geoip_get_client_history(geoip_client_action_t action, + char **country_str, char **ipver_str); +char *geoip_get_request_history(void); +void geoip_stats_free_all(void); + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +void geoip_reset_dirreq_stats(time_t now); +char *geoip_format_dirreq_stats(time_t now); +time_t geoip_dirreq_stats_write(time_t now); +void geoip_dirreq_stats_term(void); +void geoip_entry_stats_init(time_t now); +time_t geoip_entry_stats_write(time_t now); +void geoip_entry_stats_term(void); +void geoip_reset_entry_stats(time_t now); +char *geoip_format_entry_stats(time_t now); +void geoip_bridge_stats_init(time_t now); +char *geoip_format_bridge_stats(time_t now); +time_t geoip_bridge_stats_write(time_t now); +void geoip_bridge_stats_term(void); +const char *geoip_get_bridge_stats_extrainfo(time_t); +char *geoip_get_bridge_stats_controller(time_t); +char *format_client_stats_heartbeat(time_t now); + +#endif /* !defined(TOR_GEOIP_STATS_H) */ diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index f05b0fc79..7f7a00abd 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -33,6 +33,7 @@ #include "feature/nodelist/networkstatus.h" #include "core/proto/proto_http.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/dirvote.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_dos.c b/src/test/test_dos.c index b411e7b38..40a4c6ba2 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -9,7 +9,7 @@ #include "core/or/dos.h" #include "core/or/circuitlist.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "core/or/channel.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" @@ -500,4 +500,3 @@ struct testcase_t dos_tests[] = { NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c index 9df8ea798..8f13939db 100644 --- a/src/test/test_geoip.c +++ b/src/test/test_geoip.c @@ -11,6 +11,7 @@ #include "core/or/or.h" #include "app/config/config.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "test/test.h" /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs

1 0

[tor/master] Remove excess dependencies from geoip.c
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit fa32574bdb7381f02abc68ad917ad79f354938a4 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 10:15:39 2018 -0400 Remove excess dependencies from geoip.c --- src/app/config/config.c | 24 ++++++++++++++++++++---- src/feature/stats/geoip.c | 41 +++++++++++++++++++++++++---------------- src/feature/stats/geoip.h | 12 +++++++++--- src/test/test_geoip.c | 14 ++++++++------ 4 files changed, 62 insertions(+), 29 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 0909ced50..53cc9c0ac 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -96,6 +96,7 @@ #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nickname.h" #include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" @@ -8348,6 +8349,11 @@ config_load_geoip_file_(sa_family_t family, const char *fname, const char *default_fname) { + const or_options_t *options = get_options(); + const char *msg = ""; + int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO; + int r; + #ifdef _WIN32 char *free_fname = NULL; /* Used to hold any temporary-allocated value */ /* XXXX Don't use this "<default>" junk; make our filename options @@ -8357,12 +8363,16 @@ config_load_geoip_file_(sa_family_t family, tor_asprintf(&free_fname, "%s\\%s", conf_root, default_fname); fname = free_fname; } - geoip_load_file(family, fname); + r = geoip_load_file(family, fname, severity); tor_free(free_fname); #else /* !(defined(_WIN32)) */ (void)default_fname; - geoip_load_file(family, fname); + r = geoip_load_file(family, fname, severity); #endif /* defined(_WIN32) */ + + if (r < 0 && severity == LOG_WARN) { + log_warn(LD_GENERAL, "%s", msg); + } } /** Load geoip files for IPv4 and IPv6 if <a>options</a> and @@ -8376,13 +8386,19 @@ config_maybe_load_geoip_files_(const or_options_t *options, if (options->GeoIPFile && ((!old_options || !opt_streq(old_options->GeoIPFile, options->GeoIPFile)) - || !geoip_is_loaded(AF_INET))) + || !geoip_is_loaded(AF_INET))) { config_load_geoip_file_(AF_INET, options->GeoIPFile, "geoip"); + /* Okay, now we need to maybe change our mind about what is in + * which country. We do this for IPv4 only since that's what we + * store in node->country. */ + refresh_all_country_info(); + } if (options->GeoIPv6File && ((!old_options || !opt_streq(old_options->GeoIPv6File, options->GeoIPv6File)) - || !geoip_is_loaded(AF_INET6))) + || !geoip_is_loaded(AF_INET6))) { config_load_geoip_file_(AF_INET6, options->GeoIPv6File, "geoip6"); + } } /** Initialize cookie authentication (used so far by the ControlPort diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip.c index 885f26bc3..496615080 100644 --- a/src/feature/stats/geoip.c +++ b/src/feature/stats/geoip.c @@ -28,16 +28,32 @@ */ #define GEOIP_PRIVATE -#include "core/or/or.h" -#include "ht.h" -#include "lib/container/buffers.h" -#include "app/config/config.h" // XXXX -#include "feature/stats/geoip.h" -#include "feature/nodelist/routerlist.h" // XXXX +#include "lib/cc/torint.h" +/** A signed integer representing a country code. */ +typedef int16_t country_t; // XXXX duplicate in or.h + +#include "feature/stats/geoip.h" +#include "lib/container/map.h" #include "lib/container/order.h" +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/ctime/di_ops.h" +#include "lib/encoding/binascii.h" +#include "lib/fs/files.h" +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/net/address.h" //???? +#include "lib/net/inaddr.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/compat_string.h" +#include "lib/string/scanf.h" +#include "lib/string/util_string.h" #include "lib/time/tvdiff.h" +#include <stdio.h> +#include <string.h> + static void init_geoip_countries(void); /** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */ @@ -305,19 +321,16 @@ init_geoip_countries(void) * with '#' (comments). */ int -geoip_load_file(sa_family_t family, const char *filename) +geoip_load_file(sa_family_t family, const char *filename, int severity) { FILE *f; - const char *msg = ""; - const or_options_t *options = get_options(); - int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO; crypto_digest_t *geoip_digest_env = NULL; tor_assert(family == AF_INET || family == AF_INET6); if (!(f = tor_fopen_cloexec(filename, "r"))) { - log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s", - filename, msg); + log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s.", + filename); return -1; } if (!geoip_countries) @@ -357,10 +370,6 @@ geoip_load_file(sa_family_t family, const char *filename) * our extra-info descriptors. */ if (family == AF_INET) { smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_); - /* Okay, now we need to maybe change our mind about what is in - * which country. We do this for IPv4 only since that's what we - * store in node->country. */ - refresh_all_country_info(); crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN); } else { /* AF_INET6 */ diff --git a/src/feature/stats/geoip.h b/src/feature/stats/geoip.h index 4adfc10fb..1ca04cdff 100644 --- a/src/feature/stats/geoip.h +++ b/src/feature/stats/geoip.h @@ -12,13 +12,18 @@ #ifndef TOR_GEOIP_H #define TOR_GEOIP_H +#include "lib/net/nettypes.h" #include "lib/testsupport/testsupport.h" +#include "lib/cc/torint.h" #ifdef GEOIP_PRIVATE STATIC int geoip_parse_entry(const char *line, sa_family_t family); STATIC void clear_geoip_db(void); #endif /* defined(GEOIP_PRIVATE) */ +struct in6_addr; +struct tor_addr_t; + int geoip_get_country_by_ipv4(uint32_t ipaddr); int geoip_get_country_by_ipv6(const struct in6_addr *addr); @@ -27,10 +32,11 @@ typedef struct geoip_country_t { char countrycode[3]; } geoip_country_t; -const smartlist_t *geoip_get_countries(void); +struct smartlist_t; +const struct smartlist_t *geoip_get_countries(void); -int geoip_load_file(sa_family_t family, const char *filename); -MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); +int geoip_load_file(sa_family_t family, const char *filename, int severity); +MOCK_DECL(int, geoip_get_country_by_addr, (const struct tor_addr_t *addr)); MOCK_DECL(int, geoip_get_n_countries, (void)); const char *geoip_get_country_name(country_t num); MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c index 8f13939db..5e2849147 100644 --- a/src/test/test_geoip.c +++ b/src/test/test_geoip.c @@ -404,7 +404,8 @@ test_geoip_load_file(void *arg) /* A nonexistant filename should fail. */ tt_int_op(-1, OP_EQ, - geoip_load_file(AF_INET, "/you/did/not/put/a/file/here/I/hope")); + geoip_load_file(AF_INET, "/you/did/not/put/a/file/here/I/hope", + LOG_INFO)); /* We start out with only "Ningunpartia" in the database. */ tt_int_op(1, OP_EQ, geoip_get_n_countries()); @@ -418,7 +419,7 @@ test_geoip_load_file(void *arg) const char *fname = get_fname("geoip"); tt_int_op(0, OP_EQ, write_str_to_file(fname, GEOIP_CONTENT, 1)); - int rv = geoip_load_file(AF_INET, fname); + int rv = geoip_load_file(AF_INET, fname, LOG_WARN); if (rv != 0) { TT_GRIPE(("Unable to load geoip from %s", escaped(fname))); } @@ -468,7 +469,8 @@ test_geoip6_load_file(void *arg) /* A nonexistant filename should fail. */ tt_int_op(-1, OP_EQ, - geoip_load_file(AF_INET6, "/you/did/not/put/a/file/here/I/hope")); + geoip_load_file(AF_INET6, "/you/did/not/put/a/file/here/I/hope", + LOG_INFO)); /* Any lookup attempt should say "-1" because we have no info */ tor_inet_pton(AF_INET6, "2001:4860:4860::8888", &iaddr6); @@ -494,7 +496,7 @@ test_geoip6_load_file(void *arg) "2001:4878:205::,2001:4878:214:ffff:ffff:ffff:ffff:ffff,US\n"; tt_int_op(0, OP_EQ, write_str_to_file(fname6, CONTENT, 1)); - tt_int_op(0, OP_EQ, geoip_load_file(AF_INET6, fname6)); + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET6, fname6, LOG_WARN)); /* Check that we loaded some countries; this will fail if there are ever * fewer than 5 countries in our test data above. */ @@ -546,11 +548,11 @@ test_geoip_load_2nd_file(void *arg) tt_int_op(0, OP_EQ, write_str_to_file(fname_empty, "\n", 1)); /* Load 1st geoip file */ - tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_geoip)); + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_geoip, LOG_WARN)); /* Load 2nd geoip (empty) file */ /* It has to be the same IP address family */ - tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_empty)); + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_empty, LOG_WARN)); /* Check that there is no geoip information for 8.8.8.8, */ /* since loading the empty 2nd file should have delete it. */

1 0

[tor/master] Merge branch 'split_stats'
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit 9e65e7a36f6f7b18164d0e76fe25f0c32b31aaec Merge: 09223ad83 316453065 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 16:26:06 2018 -0400 Merge branch 'split_stats' .gitignore | 2 + Makefile.am | 2 + changes/ticket27892 | 3 + src/app/config/config.c | 28 +- src/app/main/main.c | 6 +- src/core/include.am | 12 +- src/core/mainloop/connection.c | 2 +- src/core/mainloop/mainloop.c | 3 +- src/core/or/channel.c | 23 +- src/core/or/circuitbuild.c | 2 +- src/core/or/circuitlist.c | 1 + src/core/or/circuituse.c | 2 +- src/core/or/connection_edge.c | 1 + src/core/or/connection_or.c | 2 +- src/core/or/dos.c | 10 +- src/core/or/or.h | 4 +- src/core/or/policies.c | 2 +- src/core/or/relay.c | 2 +- src/core/or/status.c | 2 +- src/feature/control/control.c | 5 +- src/feature/control/getinfo_geoip.c | 45 +++ src/feature/control/getinfo_geoip.h | 14 + src/feature/dircache/dircache.c | 2 +- src/feature/dircache/dirserv.c | 2 +- src/feature/dirclient/dirclient.c | 2 +- src/feature/dircommon/directory.c | 2 +- src/feature/nodelist/nodelist.c | 2 +- src/feature/nodelist/routerset.c | 2 +- src/feature/relay/router.c | 3 +- src/feature/rend/rendservice.c | 2 +- src/feature/stats/{geoip.c => geoip_stats.c} | 539 ++------------------------- src/feature/stats/{geoip.h => geoip_stats.h} | 30 +- src/feature/stats/predict_ports.c | 311 ++++++++++++++++ src/feature/stats/predict_ports.h | 30 ++ src/feature/stats/rephist.c | 309 +-------------- src/feature/stats/rephist.h | 14 - src/include.am | 1 + src/lib/geoip/.may_include | 13 + src/lib/geoip/country.h | 14 + src/lib/geoip/geoip.c | 510 +++++++++++++++++++++++++ src/lib/geoip/geoip.h | 50 +++ src/lib/geoip/include.am | 17 + src/test/test_config.c | 2 +- src/test/test_dir_handle_get.c | 3 +- src/test/test_dos.c | 3 +- src/test/test_geoip.c | 17 +- src/test/test_options.c | 2 +- src/test/test_routerset.c | 2 +- src/test/testing_common.c | 1 + 49 files changed, 1157 insertions(+), 901 deletions(-)

1 0

[tor/master] Detect an unlikely integer overflow.
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit b058f64cc002b44e6dd48616ca3163a01c3f3e14 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 16:30:02 2018 -0400 Detect an unlikely integer overflow. --- src/feature/stats/geoip_stats.c | 5 ++++- src/lib/geoip/country.h | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index 3e647bd46..1a4f8ddfb 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -265,7 +265,10 @@ geoip_note_client_seen(geoip_client_action_t action, int country_idx = geoip_get_country_by_addr(addr); if (country_idx < 0) country_idx = 0; /** unresolved requests are stored at index 0. */ - increment_v3_ns_request(country_idx); + IF_BUG_ONCE(country_idx > COUNTRY_MAX) { + return; + } + increment_v3_ns_request((country_t) country_idx); } } diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h index e4ad0752b..080c15602 100644 --- a/src/lib/geoip/country.h +++ b/src/lib/geoip/country.h @@ -11,4 +11,6 @@ /** A signed integer representing a country code. */ typedef int16_t country_t; +#define COUNTRY_MAX INT16_MAX + #endif

1 0

[tor/master] Changes file for splitting stats
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit 013a1238336ab860a9c9998e1aec6bc211f33fb3 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 10:30:17 2018 -0400 Changes file for splitting stats --- changes/ticket27892 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/ticket27892 b/changes/ticket27892 new file mode 100644 index 000000000..da7274c73 --- /dev/null +++ b/changes/ticket27892 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Split the non-statistics-related parts from the rephist.c and geoip.c + modules. Closes ticket 27892.

1 0

[tor/master] Mark a variable static.
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit 316453065dfae10dde5afce15feafbdc5c8562a9 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 11:59:39 2018 -0400 Mark a variable static. --- src/feature/stats/geoip_stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index d1a5dd79b..3e647bd46 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -43,7 +43,7 @@ #include "lib/time/tvdiff.h" /** Number of entries in n_v3_ns_requests */ -size_t n_v3_ns_requests_len = 0; +static size_t n_v3_ns_requests_len = 0; /** Array, indexed by country index, of number of v3 networkstatus requests * received from that country */ static uint32_t *n_v3_ns_requests;

1 0

[tor/master] geoip: fix windows compilation
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit 34402ece8c7483ba0a206af35c1bae3d79e9b63c Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 11:02:58 2018 -0400 geoip: fix windows compilation --- src/lib/geoip/.may_include | 1 + src/lib/geoip/geoip.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/lib/geoip/.may_include b/src/lib/geoip/.may_include index 63207125b..b1ee2dcfe 100644 --- a/src/lib/geoip/.may_include +++ b/src/lib/geoip/.may_include @@ -1,3 +1,4 @@ +orconfig.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h diff --git a/src/lib/geoip/geoip.h b/src/lib/geoip/geoip.h index 3c1238c31..6ef27d66d 100644 --- a/src/lib/geoip/geoip.h +++ b/src/lib/geoip/geoip.h @@ -12,8 +12,10 @@ #ifndef TOR_GEOIP_H #define TOR_GEOIP_H +#include "orconfig.h" #include "lib/net/nettypes.h" #include "lib/testsupport/testsupport.h" +#include "lib/net/inaddr_st.h" #include "lib/geoip/country.h" #ifdef GEOIP_PRIVATE

1 0

[tor/master] Always initialize the periodic events list.
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit d0b2b5a20269c050d5346a869940b55ff2add190 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 12:44:20 2018 -0400 Always initialize the periodic events list. Various places in our code try to activate these events or check their status, so we should make sure they're initialized as early as possible. Fixes bug 27861; bugfix on 0.3.5.1-alpha. --- changes/bug27861 | 4 ++++ src/core/mainloop/mainloop.c | 11 ++++++----- src/test/test_periodic_event.c | 8 ++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/changes/bug27861 b/changes/bug27861 new file mode 100644 index 000000000..377d68e32 --- /dev/null +++ b/changes/bug27861 @@ -0,0 +1,4 @@ + o Major bugfixes (initialization, crash): + - Fix an assertion crash that would stop Tor from starting up if + the code tried to activate a periodic event too early. Fixes bug + 27861; bugfix on 0.3.5.1-alpha. diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 4a9da43c9..6e7033ec4 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1535,7 +1535,9 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data) STATIC void initialize_periodic_events(void) { - tor_assert(periodic_events_initialized == 0); + if (periodic_events_initialized) + return; + periodic_events_initialized = 1; /* Set up all periodic events. We'll launch them by roles. */ @@ -2682,6 +2684,8 @@ dns_servers_relaunch_checks(void) void initialize_mainloop_events(void) { + initialize_periodic_events(); + if (!schedule_active_linked_connections_event) { schedule_active_linked_connections_event = mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL); @@ -2699,10 +2703,7 @@ do_main_loop(void) /* initialize the periodic events first, so that code that depends on the * events being present does not assert. */ - if (! periodic_events_initialized) { - initialize_periodic_events(); - } - + initialize_periodic_events(); initialize_mainloop_events(); /* set up once-a-second callback. */ diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 7804a9d8f..86dedd85d 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -87,15 +87,19 @@ test_pe_launch(void *arg) item->fn = dumb_event_fn; } - /* Lets make sure that before intialization, we can't scan the periodic - * events list and launch them. Lets try by being a Client. */ options = get_options_mutable(); options->SocksPort_set = 1; periodic_events_on_new_options(options); +#if 0 + /* Lets make sure that before intialization, we can't scan the periodic + * events list and launch them. Lets try by being a Client. */ + /* XXXX We make sure these events are initialized now way earlier than we + * did before. */ for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); } +#endif initialize_periodic_events();

1 0

[tor/master] Merge branch 'ticket_27861'
by nickm＠torproject.org 27 Sep '18

27 Sep '18

commit 09223ad83b09e75494863bacb3f8004c945ef015 Merge: de0b07c63 d0b2b5a20 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 27 16:24:47 2018 -0400 Merge branch 'ticket_27861' changes/bug27861 | 4 ++++ src/core/mainloop/mainloop.c | 11 ++++++----- src/test/test_periodic_event.c | 8 ++++++-- 3 files changed, 16 insertions(+), 7 deletions(-)

1 0