April 2014 - tor-commits - lists.torproject.org

[tor/master] Improvements to #11553 fix based on review
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit a770b74501a3faf6483c40735b70adae6fb95187 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 23 12:39:01 2014 -0400 Improvements to #11553 fix based on review Use a per-channel ratelim_t to control the rate at which we report failures for each channel. Explain why I picked N=32. Never return a zero circID. Thanks to Andrea and to cypherpunks. --- src/or/channel.c | 3 +++ src/or/channel.h | 6 ++++-- src/or/circuitbuild.c | 19 +++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/or/channel.c b/src/or/channel.c index dfef703..691da17 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -731,6 +731,9 @@ channel_init(channel_t *chan) /* Init timestamp */ chan->timestamp_last_added_nonpadding = time(NULL); + /* Warn about exhausted circuit IDs no more than hourly. */ + chan->last_warned_circ_ids_exhausted.rate = 3600; + /* Initialize queues. */ TOR_SIMPLEQ_INIT(&chan->incoming_queue); TOR_SIMPLEQ_INIT(&chan->outgoing_queue); diff --git a/src/or/channel.h b/src/or/channel.h index e63c949..63da0c1 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -148,8 +148,6 @@ struct channel_s { ENUM_BF(circ_id_type_t) circ_id_type:2; /** DOCDOC*/ unsigned wide_circ_ids:1; - /** Have we logged a warning about circID exhaustion on this channel? */ - unsigned warned_circ_ids_exhausted:1; /* For how many circuits are we n_chan? What about p_chan? */ unsigned int num_n_circuits, num_p_circuits; @@ -178,6 +176,10 @@ struct channel_s { */ unsigned int is_local:1; + /** Have we logged a warning about circID exhaustion on this channel? + * If so, when? */ + ratelim_t last_warned_circ_ids_exhausted; + /** Channel timestamps for cell channels */ time_t timestamp_client; /* Client used this, according to relay.c */ time_t timestamp_drained; /* Output queue empty */ diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 7b852ff..8d6aad6 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -102,6 +102,12 @@ channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, static circid_t get_unique_circ_id_by_chan(channel_t *chan) { +/* This number is chosen somewhat arbitrarily; see comment below for more + * info. When the space is 80% full, it gives a one-in-a-million failure + * chance; when the space is 90% full, it gives a one-in-850 chance; and when + * the space is 95% full, it gives a one-in-26 failure chance. That seems + * okay, though you could make a case IMO for anything between N=32 and + * N=256. */ #define MAX_CIRCID_ATTEMPTS 64 circid_t test_circ_id; @@ -137,19 +143,20 @@ get_unique_circ_id_by_chan(channel_t *chan) * whole circuit ID space every time we extend a circuit, which is * not so great either. */ - if (! chan->warned_circ_ids_exhausted) { - chan->warned_circ_ids_exhausted = 1; - log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " + log_fn_ratelim(&chan->last_warned_circ_ids_exhausted, LOG_WARN, + LD_CIRC,"No unused circIDs found on channel %s wide " "circID support, with %u inbound and %u outbound circuits. " "Failing a circuit.", chan->wide_circ_ids ? "with" : "without", chan->num_p_circuits, chan->num_n_circuits); - } return 0; } - crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); - test_circ_id &= mask; + do { + crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); + test_circ_id &= mask; + } while (test_circ_id == 0); + test_circ_id |= high_bit; } while (circuit_id_in_use_on_channel(test_circ_id, chan)); return test_circ_id;

1 0

[tor/master] Merge remote-tracking branch 'public/bug11553_024' into bug11553_025
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit 17ad309d33561ee255cac70bdb9a19803f2d8c08 Merge: fd9961d a770b74 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 23 12:44:18 2014 -0400 Merge remote-tracking branch 'public/bug11553_024' into bug11553_025 Conflicts: src/or/circuitbuild.c src/or/channel.c | 3 +++ src/or/channel.h | 6 ++++-- src/or/circuitbuild.c | 19 +++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --cc src/or/channel.h index de19fad,63da0c1..bd9a02f --- a/src/or/channel.h +++ b/src/or/channel.h @@@ -146,16 -145,14 +146,14 @@@ struct channel_s * When we send CREATE cells along this connection, which half of the * space should we use? */ - ENUM_BF(circ_id_type_t) circ_id_type:2; + circ_id_type_bitfield_t circ_id_type:2; /** DOCDOC*/ unsigned wide_circ_ids:1; - /** Have we logged a warning about circID exhaustion on this channel? */ - unsigned warned_circ_ids_exhausted:1; - /* For how many circuits are we n_chan? What about p_chan? */ + /** For how many circuits are we n_chan? What about p_chan? */ unsigned int num_n_circuits, num_p_circuits; - /* + /** * True iff this channel shouldn't get any new circs attached to it, * because the connection is too old, or because there's a better one. * More generally, this flag is used to note an unhealthy connection; diff --cc src/or/circuitbuild.c index 550ed1c,8d6aad6..9e11a0b --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@@ -87,9 -102,14 +87,15 @@@ channel_connect_for_circuit(const tor_a static circid_t get_unique_circ_id_by_chan(channel_t *chan) { + /* This number is chosen somewhat arbitrarily; see comment below for more + * info. When the space is 80% full, it gives a one-in-a-million failure + * chance; when the space is 90% full, it gives a one-in-850 chance; and when + * the space is 95% full, it gives a one-in-26 failure chance. That seems + * okay, though you could make a case IMO for anything between N=32 and + * N=256. */ #define MAX_CIRCID_ATTEMPTS 64 - + int in_use; + unsigned n_with_circ = 0, n_pending_destroy = 0; circid_t test_circ_id; circid_t attempts=0; circid_t high_bit, max_range, mask; @@@ -123,30 -143,22 +129,31 @@@ * whole circuit ID space every time we extend a circuit, which is * not so great either. */ - if (! chan->warned_circ_ids_exhausted) { - chan->warned_circ_ids_exhausted = 1; - log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " + log_fn_ratelim(&chan->last_warned_circ_ids_exhausted, LOG_WARN, + LD_CIRC,"No unused circIDs found on channel %s wide " "circID support, with %u inbound and %u outbound circuits. " + "Found %u circuit IDs in use by circuits, and %u with " + "pending destroy cells." "Failing a circuit.", chan->wide_circ_ids ? "with" : "without", - chan->num_p_circuits, chan->num_n_circuits); + chan->num_p_circuits, chan->num_n_circuits, + n_with_circ, n_pending_destroy); return 0; } - crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); - test_circ_id &= mask; + do { + crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); + test_circ_id &= mask; + } while (test_circ_id == 0); + test_circ_id |= high_bit; - } while (circuit_id_in_use_on_channel(test_circ_id, chan)); + + in_use = circuit_id_in_use_on_channel(test_circ_id, chan); + if (in_use == 1) + ++n_with_circ; + else if (in_use == 2) + ++n_pending_destroy; + } while (in_use); return test_circ_id; }

1 0

[tor/master] Merge remote-tracking branch 'public/bug11553_024' into bug11553_025
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit fd9961d220504f81204ab5ad0204cfec8abdb75e Merge: 47a0c10 985deaa Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 18 13:23:44 2014 -0400 Merge remote-tracking branch 'public/bug11553_024' into bug11553_025 src/or/circuitbuild.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)

1 0

r26732: {website} use page macro, not direct links. (website/trunk/docs/en)
by Andrew Lewman 24 Apr '14

24 Apr '14

Author: phobos Date: 2014-04-24 14:45:24 +0000 (Thu, 24 Apr 2014) New Revision: 26732 Modified: website/trunk/docs/en/faq.wml Log: use page macro, not direct links. Modified: website/trunk/docs/en/faq.wml =================================================================== --- website/trunk/docs/en/faq.wml 2014-04-24 14:42:53 UTC (rev 26731) +++ website/trunk/docs/en/faq.wml 2014-04-24 14:45:24 UTC (rev 26732) @@ -1342,7 +1342,7 @@ Unfortunately, Torbutton only works with Firefox right now, and without -<a href="https://www.torproject.org/docs/torbutton/en/design/">Torbutton's +<a href="<page docs/torbutton/en/design/>">Torbutton's extensive privacy fixes</a> there are many ways for websites or other attackers to recognize you, track you back to your IP address, and so on. @@ -1398,13 +1398,11 @@ When you use Tor, you are sending queries through exit relays that are -also -shared by thousands of other users. Tor users typically see this message -when many Tor users are querying Google in a short period of time. -Google -interprets the high volume of traffic from a single IP address (the exit -relay you happened to pick) as somebody trying to "crawl" their website, -so it slows down traffic from that IP address for a short time. +also shared by thousands of other users. Tor users typically see this +message when many Tor users are querying Google in a short period of time. +Google interprets the high volume of traffic from a single IP address +(the exit relay you happened to pick) as somebody trying to "crawl" their +website, so it slows down traffic from that IP address for a short time. An alternate explanation is that Google tries to detect certain @@ -1416,8 +1414,7 @@ -To our knowledge, Google is not doing anything intentionally -specifically +To our knowledge, Google is not doing anything intentionally specifically to deter or block Tor use. The error message about an infected machine should clear up again after a short time.

1 0

r26731: {website} update more references to torbutton (in website/trunk: . docs/en)
by Andrew Lewman 24 Apr '14

24 Apr '14

Author: phobos Date: 2014-04-24 14:42:53 +0000 (Thu, 24 Apr 2014) New Revision: 26731 Modified: website/trunk/Makefile website/trunk/docs/en/documentation.wml website/trunk/docs/en/faq.wml Log: update more references to torbutton Modified: website/trunk/Makefile =================================================================== --- website/trunk/Makefile 2014-04-24 14:39:20 UTC (rev 26730) +++ website/trunk/Makefile 2014-04-24 14:42:53 UTC (rev 26731) @@ -14,7 +14,7 @@ export DEVTAG=tor-0.2.5.3-alpha WMLBASE=. -SUBDIRS=docs eff projects press about download getinvolved donate torbutton +SUBDIRS=docs eff projects press about download getinvolved donate include $(WMLBASE)/Makefile.local include $(WMLBASE)/Makefile.common @@ -36,8 +36,6 @@ $(MAKE) -C "$@" WMLBASE=.. donate: $(MAKE) -C "$@" WMLBASE=.. -torbutton: - $(MAKE) -C "$@" WMLBASE=.. mirrors: ./update-mirrors.pl translations: @@ -49,4 +47,4 @@ # XXX: this also depends on all subs' wmlfiles. How to fix? #translation-status.html.en: $(LANGS) $(WMIFILES) $(WMLFILES) -.PHONY: docs eff projects press about download getinvolved donate torbutton +.PHONY: docs eff projects press about download getinvolved donate Modified: website/trunk/docs/en/documentation.wml =================================================================== --- website/trunk/docs/en/documentation.wml 2014-04-24 14:39:20 UTC (rev 26730) +++ website/trunk/docs/en/documentation.wml 2014-04-24 14:42:53 UTC (rev 26731) @@ -18,7 +18,7 @@ Mac OS X</a></li> <li><a href="<page docs/tor-doc-unix>">Installing Tor on Linux/BSD/Unix</a></li> - <li><a href="<page torbutton/index>">Installing + <li><a href="<page docs/torbutton/index>">Installing Torbutton for Tor</a></li> <li><a href="<page docs/tor-doc-relay>">Configuring a Tor relay</a></li> Modified: website/trunk/docs/en/faq.wml =================================================================== --- website/trunk/docs/en/faq.wml 2014-04-24 14:39:20 UTC (rev 26730) +++ website/trunk/docs/en/faq.wml 2014-04-24 14:42:53 UTC (rev 26731) @@ -1342,7 +1342,7 @@ Unfortunately, Torbutton only works with Firefox right now, and without -<a href="https://www.torproject.org/torbutton/en/design/">Torbutton's +<a href="https://www.torproject.org/docs/torbutton/en/design/">Torbutton's extensive privacy fixes</a> there are many ways for websites or other attackers to recognize you, track you back to your IP address, and so on.

1 0

r26730: {website} move manpages to docs, move torbutton to docs, update rewrit (in website/trunk: . docs)
by Andrew Lewman 24 Apr '14

24 Apr '14

Author: phobos Date: 2014-04-24 14:39:20 +0000 (Thu, 24 Apr 2014) New Revision: 26730 Added: website/trunk/docs/manpages/ website/trunk/docs/torbutton/ Removed: website/trunk/manpages/ website/trunk/torbutton/ Modified: website/trunk/.htaccess Log: move manpages to docs, move torbutton to docs, update rewrite rules for new paths. Modified: website/trunk/.htaccess =================================================================== --- website/trunk/.htaccess 2014-04-24 04:11:40 UTC (rev 26729) +++ website/trunk/.htaccess 2014-04-24 14:39:20 UTC (rev 26730) @@ -9,12 +9,12 @@ RewriteRule ^tor-manual(.*) /docs/tor-manual$1 [R=301,L] # Torbutton -RewriteRule ^torbutton/design(.*) /torbutton/en/design$1 [R=301,L] +RewriteRule ^torbutton/design(.*) /docs/torbutton/en/design$1 [R=301,L] # Tor Browser RewriteRule ^torbrowser/dist/(.*) /dist/torbrowser/$1 [R=301,L] -RewriteRule ^torbrowser/$ /projects/torbrowser.html.en [R=301,L] -RewriteRule ^torbrowser(.*) /projects/torbrowser$1 [R=301,L] +#RewriteRule ^torbrowser/$ /projects/torbrowser.html.en [R=301,L] +RewriteRule ^torbrowser(.*) /projects/torbrowser.html.en [R=301,L] # Overview RewriteRule ^overview(.*) /about/overview$1 [R=301,L]

1 0

[tor/master] get_total_system_memory(): see how much RAM we have
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit aca05fc5c08d6296c4f93850a0256bfd96890b18 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Apr 3 11:46:01 2014 -0400 get_total_system_memory(): see how much RAM we have --- configure.ac | 2 + src/common/compat.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/common/compat.h | 2 + src/or/main.c | 1 + src/test/test_util.c | 29 ++++++++++++ 5 files changed, 153 insertions(+) diff --git a/configure.ac b/configure.ac index 6e41041..d8543b0 100644 --- a/configure.ac +++ b/configure.ac @@ -353,6 +353,7 @@ AC_CHECK_FUNCS( strtok_r \ strtoull \ sysconf \ + sysctl \ uname \ vasprintf \ _vscprintf @@ -877,6 +878,7 @@ AC_CHECK_HEADERS( sys/prctl.h \ sys/resource.h \ sys/socket.h \ + sys/sysctl.h \ sys/syslimits.h \ sys/time.h \ sys/types.h \ diff --git a/src/common/compat.c b/src/common/compat.c index 135f2c9..58dd7f5 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -38,6 +38,9 @@ #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif +#ifdef HAVE_SYS_SYSCTL_H +#include <sys/sysctl.h> +#endif #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif @@ -3311,3 +3314,119 @@ format_win32_error(DWORD err) } #endif +#if defined(HW_PHYSMEM64) +/* This appears to be an OpenBSD thing */ +#define INT64_HW_MEM HW_PHYSMEM64 +#elif defined(HW_MEMSIZE) +/* OSX defines this one */ +#define INT64_HW_MEM HW_MEMSIZE +#endif + +/** + * Helper: try to detect the total system memory, and return it. On failure, + * return 0. + */ +static uint64_t +get_total_system_memory_impl(void) +{ +#if defined(__linux__) + /* On linux, sysctl is deprecated. Because proc is so awesome that you + * shouldn't _want_ to write portable code, I guess? */ + unsigned long long result=0; + int fd = -1; + char *s = NULL; + const char *cp; + size_t file_size=0; + if (-1 == (fd = tor_open_cloexec("/proc/meminfo",O_RDONLY,0))) + return 0; + s = read_file_to_str_until_eof(fd, 65536, &file_size); + if (!s) + goto err; + cp = strstr(s, "MemTotal:"); + if (!cp) + goto err; + /* Use the system sscanf so that space will match a wider number of space */ + if (sscanf(cp, "MemTotal: %llu kB\n", &result) != 1) + goto err; + + close(fd); + tor_free(s); + return result * 1024; + + err: + tor_free(s); + close(fd); + return 0; +#elif defined (_WIN32) + /* Windows has MEMORYSTATUSEX; pretty straightforward. */ + MEMORYSTATUSEX ms; + memset(&ms, 0, sizeof(ms)); + ms.dwLength = sizeof(ms); + if (! GlobalMemoryStatusEx(&ms)) + return 0; + + return ms.ullTotalPhys; + +#elif defined(HAVE_SYSCTL) && defined(INT64_HW_MEM) + /* On many systems, HW_PYHSMEM is clipped to 32 bits; let's use a better + * variant if we know about it. */ + uint64_t memsize = 0; + size_t len = sizeof(memsize); + int mib[2] = {CTL_HW, INT64_HW_MEM}; + if (sysctl(mib,2,&memsize,&len,NULL,0)) + return 0; + + return memsize; + +#elif defined(HAVE_SYSCTL) && defined(HW_PHYSMEM) + /* On some systems (like FreeBSD I hope) you can use a size_t with + * HW_PHYSMEM. */ + size_t memsize=0; + size_t len = sizeof(memsize); + int mib[2] = {CTL_HW, HW_USERMEM}; + if (sysctl(mib,2,&memsize,&len,NULL,0)) + return -1; + + return memsize; + +#else + /* I have no clue. */ + return 0; +#endif +} + +/** + * Try to find out how much physical memory the system has. On success, + * return 0 and set *mem_out to that value. On failure, return -1. + */ +int +get_total_system_memory(size_t *mem_out) +{ + static size_t mem_cached=0; + uint64_t m = get_total_system_memory_impl(); + if (0 == m) { + /* We couldn't find our memory total */ + if (0 == mem_cached) { + /* We have no cached value either */ + *mem_out = 0; + return -1; + } + + *mem_out = mem_cached; + return 0; + } + +#if SIZE_T_MAX != UINT64_MAX + if (m > SIZE_T_MAX) { + /* I think this could happen if we're a 32-bit Tor running on a 64-bit + * system: we could have more system memory than would fit in a + * size_t. */ + m = SIZE_T_MAX; + } +#endif + + *mem_out = mem_cached = (size_t) m; + + return -1; +} + diff --git a/src/common/compat.h b/src/common/compat.h index bb88818..01faf14 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -637,6 +637,8 @@ char *make_path_absolute(char *fname); char **get_environment(void); +int get_total_system_memory(size_t *mem_out); + int spawn_func(void (*func)(void *), void *data); void spawn_exit(void) ATTR_NORETURN; diff --git a/src/or/main.c b/src/or/main.c index feca35c..8c75a3b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2751,6 +2751,7 @@ sandbox_init_filter(void) "/dev/srandom", 0, "/dev/urandom", 0, "/dev/random", 0, + "/proc/meminfo", 0, NULL, 0 ); diff --git a/src/test/test_util.c b/src/test/test_util.c index a471b8e..20e6585 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -3606,6 +3606,34 @@ test_util_socketpair(void *arg) tor_close_socket(fds[1]); } +static void +test_util_max_mem(void *arg) +{ + size_t memory1, memory2; + int r, r2; + (void) arg; + + r = get_total_system_memory(&memory1); + r2 = get_total_system_memory(&memory2); + tt_int_op(r, ==, r2); + tt_int_op(memory2, ==, memory1); + + TT_BLATHER(("System memory: "U64_FORMAT, U64_PRINTF_ARG(memory1))); + + if (r==0) { + /* You have at least a megabyte. */ + tt_int_op(memory1, >, (1<<20)); + } else { + /* You do not have a petabyte. */ +#if SIZEOF_SIZE_T == SIZEOF_UINT64_T + tt_int_op(memory1, <, (U64_LITERAL(1)<<50)); +#endif + } + + done: + ; +} + struct testcase_t util_tests[] = { UTIL_LEGACY(time), UTIL_TEST(parse_http_time, 0), @@ -3669,6 +3697,7 @@ struct testcase_t util_tests[] = { (void*)"0" }, { "socketpair_ersatz", test_util_socketpair, TT_FORK, &socketpair_setup, (void*)"1" }, + UTIL_TEST(max_mem, 0), END_OF_TESTCASES };

1 0

[tor/master] Change the logic for the default for MaxMemInQueues
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit 17ecd04fde2fd98b0cca3afb251b7173e22d3f42 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Apr 3 12:06:44 2014 -0400 Change the logic for the default for MaxMemInQueues If we can't detect the physical memory, the new default is 8 GB on 64-bit architectures, and 1 GB on 32-bit architectures. If we *can* detect the physical memory, the new default is CLAMP(256 MB, phys_mem * 0.75, MAX_DFLT) where MAX_DFLT is 8 GB on 64-bit architectures and 2 GB on 32-bit architectures. You can still override the default by hand. The logic here is simply trying to choose a lower default value on systems with less than 12 GB of physical RAM. --- changes/bug11396 | 7 ++++++ doc/tor.1.txt | 3 ++- src/or/config.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++++----- src/or/or.h | 5 +++- 4 files changed, 81 insertions(+), 8 deletions(-) diff --git a/changes/bug11396 b/changes/bug11396 new file mode 100644 index 0000000..af94be0 --- /dev/null +++ b/changes/bug11396 @@ -0,0 +1,7 @@ + o Minor features (security): + + - If you don't specify MaxMemInQueues yourself, Tor now tries to + pick a good value based on your total system memory. Previously, + the default was always 8 GB. You can still override the default by + setting MaxMemInQueues yourself. Resolves ticket 11396. + diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 1cc8f84..319921f 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1752,7 +1752,8 @@ is non-zero): it has recovered at least 10% of this memory. Do not set this option too low, or your relay may be unreliable under load. This option only affects some queues, so the actual process size will be larger than - this. (Default: 8GB) + this. If this option is set to 0, Tor will try to pick a reasonable + default based on your system's physical memory. (Default: 0) DIRECTORY SERVER OPTIONS ------------------------ diff --git a/src/or/config.c b/src/or/config.c index 4a6b301..78883cc 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -307,7 +307,7 @@ static config_var_t option_vars_[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInQueues, MEMUNIT, "8 GB"), + VAR("MaxMeminQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), @@ -565,6 +565,8 @@ static void config_maybe_load_geoip_files_(const or_options_t *options, static int options_validate_cb(void *old_options, void *options, void *default_options, int from_setconf, char **msg); +static uint64_t compute_real_max_mem_in_queues(const uint64_t val, + int log_guess); /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -2793,11 +2795,9 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInQueues < (256 << 20)) { - log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " - "Ideally, have it as large as you can afford."); - options->MaxMemInQueues = (256 << 20); - } + options->MaxMemInQueues = + compute_real_max_mem_in_queues(options->MaxMemInQueues_raw, + server_mode(options)); options->AllowInvalid_ = 0; @@ -3547,6 +3547,68 @@ options_validate(or_options_t *old_options, or_options_t *options, #undef COMPLAIN } +/* Given the value that the user has set for MaxMemInQueues, compute the + * actual maximum value. We clip this value if it's too low, and autodetect + * it if it's set to 0. */ +static uint64_t +compute_real_max_mem_in_queues(const uint64_t val, int log_guess) +{ + uint64_t result; + + if (val == 0) { +#define ONE_GIGABYTE (U64_LITERAL(1) << 30) +#define ONE_MEGABYTE (U64_LITERAL(1) << 20) +#if SIZEOF_VOID_P >= 8 +#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE) +#else +#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE) +#endif + /* The user didn't pick a memory limit. Choose a very large one + * that is still smaller than the system memory */ + static int notice_sent = 0; + size_t ram = 0; + if (get_total_system_memory(&ram) < 0) { + /* We couldn't determine our total system memory! */ +#if SIZEOF_VOID_P >= 8 + /* 64-bit system. Let's hope for 8 GB. */ + result = 8 * ONE_GIGABYTE; +#else + /* (presumably) 32-bit system. Let's hope for 1 GB. */ + result = ONE_GIGABYTE; +#endif + } else { + /* We detected it, so let's pick 3/4 of the total RAM as our limit. */ + const uint64_t avail = (ram / 4) * 3; + + /* Make sure it's in range from 0.25 GB to 8 GB. */ + if (avail > MAX_DEFAULT_MAXMEM) { + /* If you want to use more than this much RAM, you need to configure + it yourself */ + result = MAX_DEFAULT_MAXMEM; + } else if (avail < ONE_GIGABYTE / 4) { + result = ONE_GIGABYTE / 4; + } else { + result = avail; + } + } + if (log_guess && ! notice_sent) { + log_notice(LD_CONFIG, "%sMaxMemInQueues is set to "U64_FORMAT" MB. " + "You can override this by setting MaxMemInQueues by hand.", + ram ? "Based on detected system memory, " : "", + U64_PRINTF_ARG(result / ONE_MEGABYTE)); + notice_sent = 1; + } + return result; + } else if (val < ONE_GIGABYTE / 4) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " + "Ideally, have it as large as you can afford."); + return ONE_GIGABYTE / 4; + } else { + /* The value was fine all along */ + return val; + } +} + /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL * equal strings. */ static int diff --git a/src/or/or.h b/src/or/or.h index 38ab176..fa5a323 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3474,7 +3474,10 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInQueues; /**< If we have more memory than this allocated + /* MaxMemInQueues value as input by the user. We clean this up to be + * MaxMemInQueues. */ + uint64_t MaxMemInQueues_raw; + uint64_t MaxMemInQueues;/**< If we have more memory than this allocated * for queues and buffers, run the OOM handler */ /** @name port booleans

1 0

[tor/master] Expose the real maxmeminqueues via a GETINFO
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit e3af72647db51e99186b9f284066dedcdc8c10d6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Apr 4 10:33:01 2014 -0400 Expose the real maxmeminqueues via a GETINFO That is, GETINFO limits/max-mem-in-queues --- changes/bug11396 | 4 ++++ src/or/control.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/changes/bug11396 b/changes/bug11396 index af94be0..fd26329 100644 --- a/changes/bug11396 +++ b/changes/bug11396 @@ -5,3 +5,7 @@ the default was always 8 GB. You can still override the default by setting MaxMemInQueues yourself. Resolves ticket 11396. + o Minor features (controller): + - Because of the fix for ticket 11396, the real limit for memory + usage may no longer match the configured MaxMemInQueues value. + The real limit is now exposed via GETINFO limits/max-mem-in-queues. diff --git a/src/or/control.c b/src/or/control.c index 23e2054..4d41e10 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1504,6 +1504,9 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, int max_fds=-1; set_max_file_descriptors(0, &max_fds); tor_asprintf(answer, "%d", max_fds); + } else if (!strcmp(question, "limits/max-mem-in-queues")) { + tor_asprintf(answer, U64_FORMAT, + U64_PRINTF_ARG(get_options()->MaxMemInQueues)); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { @@ -2184,6 +2187,7 @@ static const getinfo_item_t getinfo_items[] = { ITEM("process/user", misc, "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), + ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),

1 0

[tor/master] Merge branch 'bug11396_v2_squashed'
by nickm＠torproject.org 24 Apr '14

24 Apr '14

commit 67aa3685e7321322cbbc2bef7f87c9a885819af8 Merge: aa1ad30 e3af726 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Apr 24 10:31:38 2014 -0400 Merge branch 'bug11396_v2_squashed' Conflicts: src/or/main.c changes/bug11396 | 11 +++++ configure.ac | 2 + doc/tor.1.txt | 3 +- src/common/compat.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/common/compat.h | 2 + src/or/config.c | 74 ++++++++++++++++++++++++++++--- src/or/control.c | 4 ++ src/or/main.c | 1 + src/or/or.h | 5 ++- src/test/test_util.c | 29 ++++++++++++ 10 files changed, 242 insertions(+), 8 deletions(-) diff --cc src/or/control.c index 2815b79,4d41e10..d571900 mode 100755,100644..100755 --- a/src/or/control.c +++ b/src/or/control.c diff --cc src/or/main.c index 4770b7e,8c75a3b..6713d80 --- a/src/or/main.c +++ b/src/or/main.c @@@ -2726,95 -2721,46 +2726,96 @@@ init_addrinfo(void static sandbox_cfg_t* sandbox_init_filter(void) { + const or_options_t *options = get_options(); sandbox_cfg_t *cfg = sandbox_cfg_new(); + int i; sandbox_cfg_allow_openat_filename(&cfg, - get_datadir_fname("cached-status"), 1); + get_datadir_fname("cached-status")); sandbox_cfg_allow_open_filename_array(&cfg, - get_datadir_fname("cached-certs"), 1, - get_datadir_fname("cached-certs.tmp"), 1, - get_datadir_fname("cached-consensus"), 1, - get_datadir_fname("unverified-consensus"), 1, - get_datadir_fname("unverified-consensus.tmp"), 1, - get_datadir_fname("cached-microdesc-consensus"), 1, - get_datadir_fname("cached-microdesc-consensus.tmp"), 1, - get_datadir_fname("cached-microdescs"), 1, - get_datadir_fname("cached-microdescs.tmp"), 1, - get_datadir_fname("cached-microdescs.new"), 1, - get_datadir_fname("cached-microdescs.new.tmp"), 1, - get_datadir_fname("unverified-microdesc-consensus"), 1, - get_datadir_fname("cached-descriptors"), 1, - get_datadir_fname("cached-descriptors.new"), 1, - get_datadir_fname("cached-descriptors.tmp"), 1, - get_datadir_fname("cached-descriptors.new.tmp"), 1, - get_datadir_fname("cached-descriptors.tmp.tmp"), 1, - get_datadir_fname("cached-extrainfo"), 1, - get_datadir_fname("state.tmp"), 1, - get_datadir_fname("unparseable-desc.tmp"), 1, - get_datadir_fname("unparseable-desc"), 1, - "/dev/srandom", 0, - "/dev/urandom", 0, - "/dev/random", 0, - "/proc/meminfo", 0, + get_datadir_fname("cached-certs"), + get_datadir_fname("cached-certs.tmp"), + get_datadir_fname("cached-consensus"), + get_datadir_fname("cached-consensus.tmp"), + get_datadir_fname("unverified-consensus"), + get_datadir_fname("unverified-consensus.tmp"), + get_datadir_fname("unverified-microdesc-consensus"), + get_datadir_fname("unverified-microdesc-consensus.tmp"), + get_datadir_fname("cached-microdesc-consensus"), + get_datadir_fname("cached-microdesc-consensus.tmp"), + get_datadir_fname("cached-microdescs"), + get_datadir_fname("cached-microdescs.tmp"), + get_datadir_fname("cached-microdescs.new"), + get_datadir_fname("cached-microdescs.new.tmp"), + get_datadir_fname("cached-descriptors"), + get_datadir_fname("cached-descriptors.new"), + get_datadir_fname("cached-descriptors.tmp"), + get_datadir_fname("cached-descriptors.new.tmp"), + get_datadir_fname("cached-descriptors.tmp.tmp"), + get_datadir_fname("cached-extrainfo"), + get_datadir_fname("cached-extrainfo.new"), + get_datadir_fname("cached-extrainfo.tmp"), + get_datadir_fname("cached-extrainfo.new.tmp"), + get_datadir_fname("cached-extrainfo.tmp.tmp"), + get_datadir_fname("state.tmp"), + get_datadir_fname("unparseable-desc.tmp"), + get_datadir_fname("unparseable-desc"), + get_datadir_fname("v3-status-votes"), + get_datadir_fname("v3-status-votes.tmp"), + tor_strdup("/dev/srandom"), + tor_strdup("/dev/urandom"), + tor_strdup("/dev/random"), + tor_strdup("/etc/hosts"), ++ tor_strdup("/proc/meminfo"), NULL, 0 ); + if (options->ServerDNSResolvConfFile) + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->ServerDNSResolvConfFile)); + else + sandbox_cfg_allow_open_filename(&cfg, tor_strdup("/etc/resolv.conf")); + + for (i = 0; i < 2; ++i) { + if (get_torrc_fname(i)) { + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(get_torrc_fname(i))); + } + } + +#define RENAME_SUFFIX(name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname(name suffix), \ + get_datadir_fname(name)) + +#define RENAME_SUFFIX2(prefix, name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname2(prefix, name suffix), \ + get_datadir_fname2(prefix, name)) + + RENAME_SUFFIX("cached-certs", ".tmp"); + RENAME_SUFFIX("cached-consensus", ".tmp"); + RENAME_SUFFIX("unverified-consensus", ".tmp"); + RENAME_SUFFIX("unverified-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".new"); + RENAME_SUFFIX("cached-microdescs.new", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".new"); + RENAME_SUFFIX("cached-descriptors.new", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".new"); + RENAME_SUFFIX("cached-extrainfo.new", ".tmp"); + RENAME_SUFFIX("state", ".tmp"); + RENAME_SUFFIX("unparseable-desc", ".tmp"); + RENAME_SUFFIX("v3-status-votes", ".tmp"); sandbox_cfg_allow_stat_filename_array(&cfg, - get_datadir_fname(NULL), 1, - get_datadir_fname("lock"), 1, - get_datadir_fname("state"), 1, - get_datadir_fname("router-stability"), 1, - get_datadir_fname("cached-extrainfo.new"), 1, + get_datadir_fname(NULL), + get_datadir_fname("lock"), + get_datadir_fname("state"), + get_datadir_fname("router-stability"), + get_datadir_fname("cached-extrainfo.new"), NULL, 0 );

1 0