commit 31508a0abccfee1cda0869a9a7d22df74d6b67b7 Author: Alexander Færøy ahf@torproject.org Date: Tue Apr 3 18:26:52 2018 +0200
Use less memory for MaxMemInQueues for machines with more than 8 GB of RAM.
This patch changes the algorithm of compute_real_max_mem_in_queues() to use 0.4 * RAM iff the system has more than or equal to 8 GB of RAM, but will continue to use the old value of 0.75 * RAM if the system have less than * GB of RAM available.
This patch also adds tests for compute_real_max_mem_in_queues().
See: https://bugs.torproject.org/24782 --- changes/bug24782 | 4 +++ src/or/config.c | 31 +++++++++++++------ src/or/config.h | 7 +++++ src/test/test_config.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 10 deletions(-)
diff --git a/changes/bug24782 b/changes/bug24782 new file mode 100644 index 000000000..59bbdad12 --- /dev/null +++ b/changes/bug24782 @@ -0,0 +1,4 @@ + o Minor features (config options): + - Change the way the default value for MaxMemInQueues is calculated. We now + use 0.4 * RAM if the system have 8 GB RAM or more, otherwise we use the + former value of 0.75 * RAM. Closes ticket 24782. diff --git a/src/or/config.c b/src/or/config.c index 1f8ff5e3a..6b8885521 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4528,11 +4528,6 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess) if (val == 0) { #define ONE_GIGABYTE (U64_LITERAL(1) << 30) #define ONE_MEGABYTE (U64_LITERAL(1) << 20) -#if SIZEOF_VOID_P >= 8 -#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE) -#else -#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE) -#endif /* The user didn't pick a memory limit. Choose a very large one * that is still smaller than the system memory */ static int notice_sent = 0; @@ -4547,14 +4542,30 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess) result = ONE_GIGABYTE; #endif /* SIZEOF_VOID_P >= 8 */ } else { - /* We detected it, so let's pick 3/4 of the total RAM as our limit. */ - const uint64_t avail = (ram / 4) * 3; + /* We detected the amount of memory available. */ + uint64_t avail = 0; + + if (ram >= (8 * ONE_GIGABYTE)) { + /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues + * to 0.4 * RAM. The idea behind this value is that the amount of RAM + * is more than enough for a single relay and should allow the relay + * operator to run two relays if they have additional bandwidth + * available. + */ + avail = (ram / 5) * 2; + } else { + /* If we have less than 8 GB of RAM available, we use the "old" default + * for MaxMemInQueues of 0.75 * RAM. + */ + avail = (ram / 4) * 3; + }
- /* Make sure it's in range from 0.25 GB to 8 GB. */ - if (avail > MAX_DEFAULT_MAXMEM) { + /* Make sure it's in range from 0.25 GB to 8 GB for 64-bit and 0.25 to 2 + * GB for 32-bit. */ + if (avail > MAX_DEFAULT_MEMORY_QUEUE_SIZE) { /* If you want to use more than this much RAM, you need to configure it yourself */ - result = MAX_DEFAULT_MAXMEM; + result = MAX_DEFAULT_MEMORY_QUEUE_SIZE; } else if (avail < ONE_GIGABYTE / 4) { result = ONE_GIGABYTE / 4; } else { diff --git a/src/or/config.h b/src/or/config.h index f6a659d0c..1d3c27217 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -22,6 +22,13 @@ * expose more information than we're comfortable with. */ #define MIN_HEARTBEAT_PERIOD (30*60)
+/** Maximum default value for MaxMemInQueues, in bytes. */ +#if SIZEOF_VOID_P >= 8 +#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (U64_LITERAL(8) << 30) +#else +#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (U64_LITERAL(2) << 30) +#endif + MOCK_DECL(const char*, get_dirportfrontpage, (void)); MOCK_DECL(const or_options_t *, get_options, (void)); MOCK_DECL(or_options_t *, get_options_mutable, (void)); diff --git a/src/test/test_config.c b/src/test/test_config.c index d42335de3..d82f6d21d 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -585,6 +585,22 @@ test_config_parse_transport_options_line(void *arg) } }
+/* Mocks needed for the compute_max_mem_in_queues test */ +static int get_total_system_memory_mock(size_t *mem_out); + +static size_t total_system_memory_output = 0; +static int total_system_memory_return = 0; + +static int +get_total_system_memory_mock(size_t *mem_out) +{ + if (! mem_out) + return -1; + + *mem_out = total_system_memory_output; + return total_system_memory_return; +} + /* Mocks needed for the transport plugin line test */
static void pt_kickstart_proxy_mock(const smartlist_t *transport_list, @@ -5596,6 +5612,72 @@ test_config_include_opened_file_list(void *data) tor_free(dir); }
+static void +test_config_compute_max_mem_in_queues(void *data) +{ +#define GIGABYTE(x) (U64_LITERAL(x) << 30) +#define MEGABYTE(x) (U64_LITERAL(x) << 20) + (void)data; + MOCK(get_total_system_memory, get_total_system_memory_mock); + + /* We are unable to detect the amount of memory on the system. Tor will try + * to use some sensible default values for 64-bit and 32-bit systems. */ + total_system_memory_return = -1; + +#if SIZEOF_VOID_P >= 8 + /* We are on a 64-bit system. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(8)); +#else + /* We are on a 32-bit system. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1)); +#endif + + /* We are able to detect the amount of RAM on the system. */ + total_system_memory_return = 0; + + /* We are running on a system with one gigabyte of RAM. */ + total_system_memory_output = GIGABYTE(1); + + /* We have 0.75 * RAM available. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, + 3 * (GIGABYTE(1) / 4)); + + /* We are running on a tiny machine with 256 MB of RAM. */ + total_system_memory_output = MEGABYTE(256); + + /* We will now enforce a minimum of 256 MB of RAM available for the + * MaxMemInQueues here, even though we should only have had 0.75 * 256 = 192 + * MB available. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, MEGABYTE(256)); + + /* We are running on a machine with 8 GB of RAM. */ + total_system_memory_output = GIGABYTE(8); + + /* We will have 0.4 * RAM available. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, + 2 * (GIGABYTE(8) / 5)); + + /* We are running on a machine with 16 GB of RAM. */ + total_system_memory_output = GIGABYTE(16); + + /* We will have 0.4 * RAM available. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, + 2 * (GIGABYTE(16) / 5)); + + /* We are running on a machine with 32 GB of RAM. */ + total_system_memory_output = GIGABYTE(32); + + /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */ + tt_int_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, + MAX_DEFAULT_MEMORY_QUEUE_SIZE); + + done: + UNMOCK(get_total_system_memory); + +#undef GIGABYTE +#undef MEGABYTE +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL }
@@ -5644,6 +5726,7 @@ struct testcase_t config_tests[] = { CONFIG_TEST(check_bridge_distribution_setting_invalid, 0), CONFIG_TEST(check_bridge_distribution_setting_unrecognised, 0), CONFIG_TEST(include_opened_file_list, 0), + CONFIG_TEST(compute_max_mem_in_queues, 0), END_OF_TESTCASES };