March 2014 - tor-commits - lists.torproject.org

[tor/master] Document alternate bandwidth/memory unit spellings
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 27d231ca130b48e0963aaab7307109f7eafcc5c9 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 4 12:10:42 2014 -0500 Document alternate bandwidth/memory unit spellings --- changes/bug9222 | 5 +++++ doc/tor.1.txt | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/changes/bug9222 b/changes/bug9222 new file mode 100644 index 0000000..bd66277 --- /dev/null +++ b/changes/bug9222 @@ -0,0 +1,5 @@ + o Documentation: + + - Document in the manpage that "KBytes" may also be written as + "kilobytes" or "KB", that "Kbits" may also be written as + "kilobits", and so forth. Closes ticket #9222. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 14d477d..370d0cb 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -143,7 +143,17 @@ GENERAL OPTIONS the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be _at the very least_ 30 KBytes (that is, - 30720 bytes). (Default: 1 GByte) + 30720 bytes). (Default: 1 GByte) + + + + With this option, and in other options that take arguments in bytes, + KBytes, and so on, other formats are also supported. Notably, "KBytes" can + also be written as "kilobytes" or "kb"; "MBytes" can be written as + "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. + Tor also accepts "byte" and "bit" in the singular. + The prefixes "tera" and "T" are also recognized. + If no units are given, we default to bytes. + To avoid confusion, we recommend writing "bytes" or "bits" explicitly, + since it's easy to forget that "B" means bytes, not bits. [[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**:: Limit the maximum token bucket size (also known as the burst) to the given

1 0

[tor/master] Include v3 in documented 'protocols' in rend_service_descriptor_t
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 496fe685fddcf6f914819df053856621c3090c33 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 4 12:03:18 2014 -0500 Include v3 in documented 'protocols' in rend_service_descriptor_t Also make it unsigned and document that it's for INTRODUCE cell versions. Fixes 9099; bugfix on 0.2.1.5-alpha, which introduced the v3 protocol. --- changes/bug9099 | 5 +++++ src/or/or.h | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/changes/bug9099 b/changes/bug9099 new file mode 100644 index 0000000..5cd963e --- /dev/null +++ b/changes/bug9099 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix a comment about the rend_server_descriptor_t.protocols field + to more accurately describe its range. Also, make that + field unsigned, to more accurately reflect its usage. + Fixes bug 9099; bugfix on 0.2.1.5-alpha. diff --git a/src/or/or.h b/src/or/or.h index 478b478..e49a3b0 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4838,9 +4838,9 @@ typedef struct rend_service_descriptor_t { crypto_pk_t *pk; /**< This service's public key. */ int version; /**< Version of the descriptor format: 0 or 2. */ time_t timestamp; /**< Time when the descriptor was generated. */ - /** Bitmask: which rendezvous protocols are supported? - * (We allow bits '0', '1', and '2' to be set.) */ - int protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; + /** Bitmask: which introduce/rendezvous protocols are supported? + * (We allow bits '0', '1', '2' and '3' to be set.) */ + unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; /** List of the service's introduction points. Elements are removed if * introduction attempts fail. */ smartlist_t *intro_nodes;

1 0

[tor/master] doxygen comments for 10169 code
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit a406f6d0f05f812ced4bfc048f30bc8692be7e93 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Nov 20 12:01:21 2013 -0500 doxygen comments for 10169 code --- src/or/circuitlist.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c7dfe36..3237b1e 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1373,7 +1373,8 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_conn_cells); } -/** DOCDOC */ +/** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at stream. Return the number of bytes recovered. */ static size_t marked_circuit_streams_free_bytes(edge_connection_t *stream) { @@ -1392,7 +1393,8 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream) return result; } -/** DOCDOC */ +/** Aggressively free buffer contents on all the buffers of all streams on + * circuit c. Return the number of bytes recovered. */ static size_t marked_circuit_free_stream_bytes(circuit_t *c) { @@ -1439,7 +1441,9 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** DOCDOC*/ +/** Return the age in milliseconds of the oldest buffer chunk on any stream in + * the linked list stream, where age is taken in milliseconds before + * the time now (in truncated milliseconds since the epoch). */ static uint32_t circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) { @@ -1461,8 +1465,9 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) return age; } -/** DOCDOC - */ +/** Return the age in milliseconds of the oldest buffer chunk on any stream + * attached to the circuit c, where age is taken in milliseconds before + * the time now (in truncated milliseconds since the epoch). */ static uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now) { @@ -1475,7 +1480,9 @@ circuit_max_queued_data_age(const circuit_t *c, uint32_t now) } } -/** DATA */ +/** Return the age of the oldest cell or stream buffer chunk on the circuit + * c, where age is taken in milliseconds before the time now (in + * truncated milliseconds since the epoch). */ static uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now) {

1 0

[tor/master] Rename MaxMemInCellQueues to MaxMemInQueues
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit e572ec856dff263d7f43ec6d42fe3fc3b7557f73 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Nov 20 12:08:14 2013 -0500 Rename MaxMemInCellQueues to MaxMemInQueues --- doc/tor.1.txt | 10 +++++----- src/or/circuitlist.c | 10 +++++----- src/or/config.c | 9 +++++---- src/or/or.h | 5 ++--- src/or/relay.c | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 1f3afef..01f0be7 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1475,13 +1475,13 @@ is non-zero): localhost, RFC1918 addresses, and so on. This can create security issues; you should probably leave it off. (Default: 0) -**MaxMemInCellQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: +**MaxMemInQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: This option configures a threshold above which Tor will assume that it - needs to stop queueing cells because it's about to run out of memory. - If it hits this threshold, it will begin killing circuits until it - has recovered at least 10% of this memory. Do not set this option too + needs to stop queueing or buffering data because it's about to run out of + memory. If it hits this threshold, it will begin killing circuits until + it has recovered at least 10% of this memory. Do not set this option too low, or your relay may be unreliable under load. This option only - effects circuit queues, so the actual process size will be larger than + affects some queues, so the actual process size will be larger than this. (Default: 8GB) DIRECTORY SERVER OPTIONS diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 3237b1e..2e13541 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1512,11 +1512,11 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) return -1; } -#define FRACTION_OF_CELLS_TO_RETAIN_ON_OOM 0.90 +#define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated current_allocation * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_CIRCS_TO_RETAIN_ON_OOM of our maximum usage. */ + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ void circuits_handle_oom(size_t current_allocation) { @@ -1530,11 +1530,11 @@ circuits_handle_oom(size_t current_allocation) uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " - "MaxMemInCellQueues.)"); + "MaxMemInQueues.)"); { - size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * - FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); + size_t mem_target = (size_t)(get_options()->MaxMemInQueues * + FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; diff --git a/src/or/config.c b/src/or/config.c index d5c5689..0accb65 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -115,6 +115,7 @@ static config_abbrev_t _option_abbrevs[] = { { "BandwidthBurstBytes", "BandwidthBurst", 0, 0}, { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@ -343,7 +344,7 @@ static config_var_t _option_vars[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + V(MaxMemInQueues, MEMUNIT, "8 GB"), V(MaxOnionsPending, UINT, "100"), OBSOLETE("MonthlyAccountingStart"), V(MyFamily, STRING, NULL), @@ -3669,10 +3670,10 @@ options_validate(or_options_t *old_options, or_options_t *options, log_warn(LD_CONFIG, "EntryNodes is set, but UseEntryGuards is disabled. " "EntryNodes will be ignored."); - if (options->MaxMemInCellQueues < (500 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 500 MB for now. " + if (options->MaxMemInQueues < (500 << 20)) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 500 MB for now. " "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (500 << 20); + options->MaxMemInQueues = (500 << 20); } options->_AllowInvalid = 0; diff --git a/src/or/or.h b/src/or/or.h index 0dd47d0..c323783 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3075,9 +3075,8 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInCellQueues; /**< If we have more memory than this allocated - * for circuit cell queues, run the OOM handler - */ + uint64_t MaxMemInQueues; /**< If we have more memory than this allocated + * for queues and buffers, run the OOM handler */ /** @name port booleans * diff --git a/src/or/relay.c b/src/or/relay.c index 042fc18..e6d0f50 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1999,7 +1999,7 @@ cell_queues_check_size(void) { size_t alloc = total_cells_allocated * packed_cell_mem_cost(); alloc += buf_get_total_allocation(); - if (alloc >= get_options()->MaxMemInCellQueues) { + if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1; }

1 0

[tor/master] Count freed buffer bytes from buffers when oom-killing circuits.
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit eabcab2b7caab75ba8607c7dac225e4533998e80 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Nov 20 11:56:35 2013 -0500 Count freed buffer bytes from buffers when oom-killing circuits. Also, aggressively clear the buffers to try to make their bytes go away fast rather than waiting for the close-marked-connection code to get 'em. --- src/or/buffers.c | 2 +- src/or/circuitlist.c | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 4b176a8..71b61b1 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -547,7 +547,7 @@ buf_allocation(const buf_t *buf) size_t total = 0; const chunk_t *chunk; for (chunk = buf->head; chunk; chunk = chunk->next) { - total += chunk->memlen; + total += CHUNK_ALLOC_SIZE(chunk->memlen); } return total; } diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 27fe5ab..c7dfe36 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1373,6 +1373,36 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_conn_cells); } +/** DOCDOC */ +static size_t +marked_circuit_streams_free_bytes(edge_connection_t *stream) +{ + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + } + return result; +} + +/** DOCDOC */ +static size_t +marked_circuit_free_stream_bytes(circuit_t *c) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } +} + /** Return the number of cells used by the circuit c's cell queues. */ static size_t n_cells_in_circ_queues(const circuit_t *c) @@ -1486,7 +1516,8 @@ circuits_handle_oom(size_t current_allocation) /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); circuit_t *circ; - size_t n_cells_removed=0, n_cells_to_remove; + size_t mem_to_recover; + size_t mem_recovered=0; int n_circuits_killed=0; struct timeval now; uint32_t now_ms; @@ -1497,11 +1528,9 @@ circuits_handle_oom(size_t current_allocation) { size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); - size_t mem_to_recover; if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; - n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } tor_gettimeofday_cached(&now); @@ -1522,21 +1551,26 @@ circuits_handle_oom(size_t current_allocation) * them, and reclaim their storage aggressively. */ SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { size_t n = n_cells_in_circ_queues(circ); + size_t freed; if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } marked_circuit_free_cells(circ); + freed = marked_circuit_free_stream_bytes(circ); ++n_circuits_killed; - n_cells_removed += n; - if (n_cells_removed >= n_cells_to_remove) + + mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += freed; + + if (mem_recovered >= mem_to_recover) break; } SMARTLIST_FOREACH_END(circ); clean_cell_pool(); /* In case this helps. */ log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", - U64_PRINTF_ARG(n_cells_removed * packed_cell_mem_cost()), + U64_PRINTF_ARG(mem_recovered), n_circuits_killed); smartlist_free(circlist);

1 0

[tor/master] Changess file for #10169
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 03ff21b018c8d0b005d2a60c3ba2bf08d6cb00bb Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Nov 20 12:08:03 2013 -0500 Changess file for #10169 --- changes/bug10169 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changes/bug10169 b/changes/bug10169 new file mode 100644 index 0000000..979c4e4 --- /dev/null +++ b/changes/bug10169 @@ -0,0 +1,4 @@ + o Major features: + - Also consider stream buffer sizes when calculating OOM + conditions. Rename MaxMemInCellQueues to MaxMemInQueues. Fixes + bug 10169.

1 0

[tor/master] Drop the MaxMemInQueues lower limit down to 256 MB.
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 647248729fa65f0e51d062e2af8f4e8b38592bf5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Nov 20 12:12:47 2013 -0500 Drop the MaxMemInQueues lower limit down to 256 MB. on #9686, gmorehose reports that the 500 MB lower limit is too high for raspberry pi users. --- changes/bug9686 | 3 +++ src/or/config.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/changes/bug9686 b/changes/bug9686 new file mode 100644 index 0000000..82a8f44 --- /dev/null +++ b/changes/bug9686 @@ -0,0 +1,3 @@ + o Minor changes: + - Decrease the lower limit of MaxMemInQueues to 256 MBytes, to + appease raspberry pi users. Fixes bug 9686. \ No newline at end of file diff --git a/src/or/config.c b/src/or/config.c index 0accb65..01e62d6 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -3670,10 +3670,10 @@ options_validate(or_options_t *old_options, or_options_t *options, log_warn(LD_CONFIG, "EntryNodes is set, but UseEntryGuards is disabled. " "EntryNodes will be ignored."); - if (options->MaxMemInQueues < (500 << 20)) { - log_warn(LD_CONFIG, "MaxMemInQueues must be at least 500 MB for now. " + if (options->MaxMemInQueues < (256 << 20)) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " "Ideally, have it as large as you can afford."); - options->MaxMemInQueues = (500 << 20); + options->MaxMemInQueues = (256 << 20); } options->_AllowInvalid = 0;

1 0

[tor/master] Fix bugs in bug10169 bugfix memory tracking
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 79515917449c7e0d92f16db0d1e5af4a0370bbab Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 3 13:56:46 2014 -0500 Fix bugs in bug10169 bugfix memory tracking The chunk_grow() and chunk_copy() functions weren't adjusting the memory totals properly. Bugfix not on any released Tor version. --- src/or/buffers.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 71b61b1..856c9b6 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -245,12 +245,13 @@ static INLINE chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { off_t offset; + size_t memlen_orig = chunk->memlen; tor_assert(sz > chunk->memlen); offset = chunk->data - chunk->mem; chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); chunk->memlen = sz; chunk->data = chunk->mem + offset; - total_bytes_allocated_in_chunks += (sz - chunk->memlen); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(sz) - CHUNK_ALLOC_SIZE(memlen_orig); return chunk; } @@ -331,10 +332,11 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; + tor_assert(orig_n_to_skip == new_length); log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d.", + "length %d, kept %d, dropped %d. New length is %d", (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free); + orig_n_to_skip, orig_n_to_free, new_length); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); @@ -580,6 +582,7 @@ static chunk_t * chunk_copy(const chunk_t *in_chunk) { chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); newch->next = NULL; if (in_chunk->data) { off_t offset = in_chunk->data - in_chunk->mem;

1 0

[tor/master] Merge remote-tracking branch 'public/bug10169_023' into bug10169_024
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 5c45a333c3cdfc4c7a817425a1c3ae88085c389b Merge: 3511549 6472487 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 3 10:53:22 2014 -0500 Merge remote-tracking branch 'public/bug10169_023' into bug10169_024 Conflicts: doc/tor.1.txt src/or/config.c src/or/or.h The conflicts were all pretty trivial. changes/bug10169 | 4 ++ changes/bug9686 | 3 ++ doc/tor.1.txt | 10 ++-- src/or/buffers.c | 43 +++++++++++++++- src/or/buffers.h | 3 ++ src/or/circuitlist.c | 138 ++++++++++++++++++++++++++++++++++++++++---------- src/or/config.c | 9 ++-- src/or/or.h | 8 +-- src/or/relay.c | 3 +- 9 files changed, 181 insertions(+), 40 deletions(-) diff --cc doc/tor.1.txt index 3f8f6da,01f0be7..a600041 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@@ -1679,13 -1475,13 +1679,13 @@@ is non-zero) localhost, RFC1918 addresses, and so on. This can create security issues; you should probably leave it off. (Default: 0) - [[MaxMemInCellQueues]] **MaxMemInCellQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: -**MaxMemInQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: ++[[MaxMemInQueues]] **MaxMemInQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: This option configures a threshold above which Tor will assume that it - needs to stop queueing cells because it's about to run out of memory. - If it hits this threshold, it will begin killing circuits until it - has recovered at least 10% of this memory. Do not set this option too + needs to stop queueing or buffering data because it's about to run out of + memory. If it hits this threshold, it will begin killing circuits until + it has recovered at least 10% of this memory. Do not set this option too low, or your relay may be unreliable under load. This option only - affects circuit queues, so the actual process size will be larger than + affects some queues, so the actual process size will be larger than this. (Default: 8GB) DIRECTORY SERVER OPTIONS diff --cc src/or/circuitlist.c index b0e24a5,2e13541..8a581e6 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@@ -1508,11 -1368,43 +1508,43 @@@ marked_circuit_free_cells(circuit_t *ci log_warn(LD_BUG, "Called on non-marked circuit"); return; } - cell_queue_clear(&circ->n_conn_cells); + cell_queue_clear(&circ->n_chan_cells); if (! CIRCUIT_IS_ORIGIN(circ)) - cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_conn_cells); + cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } + /** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at stream. Return the number of bytes recovered. */ + static size_t + marked_circuit_streams_free_bytes(edge_connection_t *stream) + { + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + } + return result; + } + + /** Aggressively free buffer contents on all the buffers of all streams on + * circuit c. Return the number of bytes recovered. */ + static size_t + marked_circuit_free_stream_bytes(circuit_t *c) + { + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } + } + /** Return the number of cells used by the circuit c's cell queues. */ static size_t n_cells_in_circ_queues(const circuit_t *c) diff --cc src/or/config.c index 3984755,01e62d6..2805532 --- a/src/or/config.c +++ b/src/or/config.c @@@ -81,8 -114,8 +81,9 @@@ static config_abbrev_t option_abbrevs_[ { "BandwidthRateBytes", "BandwidthRate", 0, 0}, { "BandwidthBurstBytes", "BandwidthBurst", 0, 0}, { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, + { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@@ -299,10 -344,8 +300,10 @@@ static config_var_t option_vars_[] = V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + V(MaxMemInQueues, MEMUNIT, "8 GB"), - V(MaxOnionsPending, UINT, "100"), + OBSOLETE("MaxOnionsPending"), + V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), + V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), OBSOLETE("MonthlyAccountingStart"), V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), @@@ -2612,18 -3666,17 +2613,18 @@@ options_validate(or_options_t *old_opti if (options->UseBridges && options->EntryNodes) REJECT("You cannot set both UseBridges and EntryNodes."); - if (options->EntryNodes && !options->UseEntryGuards) - log_warn(LD_CONFIG, "EntryNodes is set, but UseEntryGuards is disabled. " - "EntryNodes will be ignored."); + if (options->EntryNodes && !options->UseEntryGuards) { + REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); + } - if (options->MaxMemInCellQueues < (500 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 500 MB for now. " + if (options->MaxMemInQueues < (256 << 20)) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (500 << 20); + options->MaxMemInQueues = (256 << 20); } - options->_AllowInvalid = 0; + options->AllowInvalid_ = 0; + if (options->AllowInvalidNodes) { SMARTLIST_FOREACH_BEGIN(options->AllowInvalidNodes, const char *, cp) { if (!strcasecmp(cp, "entry")) diff --cc src/or/or.h index 5318b0f,c323783..ec7e933 --- a/src/or/or.h +++ b/src/or/or.h @@@ -2817,24 -2564,18 +2817,27 @@@ typedef struct circuit_t * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + - /** For storage while n_conn is pending - * (state CIRCUIT_STATE_OR_WAIT). When defined, it is always - * length ONIONSKIN_CHALLENGE_LEN. */ - char *n_conn_onionskin; + /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */ + struct create_cell_t *n_chan_create_cell; - /** When was this circuit created? We keep this timestamp with a higher - * resolution than most so that the circuit-build-time tracking code can - * get millisecond resolution. */ + /** When did circuit construction actually begin (ie send the + * CREATE cell or begin cannibalization). + * + * Note: This timer will get reset if we decide to cannibalize + * a circuit. It may also get reset during certain phases of hidden + * service circuit use. + * + * We keep this timestamp with a higher resolution than most so that the + * circuit-build-time tracking code can get millisecond resolution. + */ + struct timeval timestamp_began; + + /** This timestamp marks when the init_circuit_base constructor ran. */ struct timeval timestamp_created; + /** When the circuit was first used, or 0 if the circuit is clean. * * XXXX023 Note that some code will artifically adjust this value backward

1 0

[tor/master] Actually release buffer freelists when handling OOM conditions.
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit fd28754dd3dce0e00304825d531348414c0a354b Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun Jan 12 12:26:45 2014 -0500 Actually release buffer freelists when handling OOM conditions. Otherwise freeing buffers won't help for a little while. --- src/or/buffers.c | 9 +++++++-- src/or/buffers.h | 2 +- src/or/circuitlist.c | 13 +++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 856c9b6..352e60a 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -276,12 +276,14 @@ preferred_chunk_size(size_t target) } /** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). */ -void + * last call to buf_shrink_freelists(). Return the amount of memory + * freed. */ +size_t buf_shrink_freelists(int free_all) { #ifdef ENABLE_BUF_FREELISTS int i; + size_t total_freed = 0; disable_control_logging(); for (i = 0; freelists[i].alloc_size; ++i) { int slack = freelists[i].slack; @@ -315,6 +317,7 @@ buf_shrink_freelists(int free_all) chunk_t *next = chunk->next; tor_assert(total_bytes_allocated_in_chunks >= CHUNK_ALLOC_SIZE(chunk->memlen)); total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + total_freed += CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); chunk = next; --n_to_free; @@ -343,8 +346,10 @@ buf_shrink_freelists(int free_all) } done: enable_control_logging(); + return total_freed; #else (void) free_all; + return 0; #endif } diff --git a/src/or/buffers.h b/src/or/buffers.h index 677d68d..b3b9003 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -18,7 +18,7 @@ void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); +size_t buf_shrink_freelists(int free_all); void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 2e13541..ffb5e0c 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1533,6 +1533,17 @@ circuits_handle_oom(size_t current_allocation) "MaxMemInQueues.)"); { + const size_t recovered = buf_shrink_freelists(1); + if (recovered >= current_allocation) { + log_warn(LD_BUG, "We somehow recovered more memory from freelists " + "than we thought we had allocated"); + current_allocation = 0; + } else { + current_allocation -= recovered; + } + } + + { size_t mem_target = (size_t)(get_options()->MaxMemInQueues * FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) @@ -1575,6 +1586,8 @@ circuits_handle_oom(size_t current_allocation) } SMARTLIST_FOREACH_END(circ); clean_cell_pool(); /* In case this helps. */ + buf_shrink_freelists(1); /* This is necessary to actually release buffer + chunks. */ log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", U64_PRINTF_ARG(mem_recovered),

1 0