August 2014 - tor-commits - lists.torproject.org

[tor/master] Whitespace fixes
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit 991545acf19039b2d8ca895bdcf53bb5f2770c31 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun Aug 24 13:32:39 2014 -0400 Whitespace fixes --- src/common/sandbox.c | 90 +------------------------------------------------- src/common/torgzip.c | 1 + src/or/circuitlist.c | 1 - 3 files changed, 2 insertions(+), 90 deletions(-) diff --git a/src/common/sandbox.c b/src/common/sandbox.c index e735feb..e215fe2 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -99,7 +99,7 @@ static sandbox_cfg_t *filter_dynamic = NULL; #undef SCMP_CMP #define SCMP_CMP(a,b,c) ((struct scmp_arg_cmp){(a),(b),(c),0}) #define SCMP_CMP_STR(a,b,c) \ - ((struct scmp_arg_cmp){(a),(b),(intptr_t)(void*)(c),0}) + ((struct scmp_arg_cmp) {(a),(b),(intptr_t)(void*)(c),0}) #define SCMP_CMP4(a,b,c,d) ((struct scmp_arg_cmp){(a),(b),(c),(d)}) /* We use a wrapper here because these masked comparisons seem to be pretty * verbose. Also, it's important to cast to scmp_datum_t before negating the @@ -1173,28 +1173,6 @@ sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) } int -sandbox_cfg_allow_stat_filename_array(sandbox_cfg_t **cfg, ...) -{ - int rc = 0; - char *fn = NULL; - - va_list ap; - va_start(ap, cfg); - - while ((fn = va_arg(ap, char*)) != NULL) { - rc = sandbox_cfg_allow_stat_filename(cfg, fn); - if (rc) { - log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_stat_filename_array fail"); - goto end; - } - } - - end: - va_end(ap); - return 0; -} - -int sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file) { sandbox_cfg_t *elem = NULL; @@ -1230,28 +1208,6 @@ sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2) } int -sandbox_cfg_allow_open_filename_array(sandbox_cfg_t **cfg, ...) -{ - int rc = 0; - char *fn = NULL; - - va_list ap; - va_start(ap, cfg); - - while ((fn = va_arg(ap, char*)) != NULL) { - rc = sandbox_cfg_allow_open_filename(cfg, fn); - if (rc) { - log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_open_filename_array fail"); - goto end; - } - } - - end: - va_end(ap); - return 0; -} - -int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) { sandbox_cfg_t *elem = NULL; @@ -1268,28 +1224,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) return 0; } -int -sandbox_cfg_allow_openat_filename_array(sandbox_cfg_t **cfg, ...) -{ - int rc = 0; - char *fn = NULL; - - va_list ap; - va_start(ap, cfg); - - while ((fn = va_arg(ap, char*)) != NULL) { - rc = sandbox_cfg_allow_openat_filename(cfg, fn); - if (rc) { - log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_openat_filename_array fail"); - goto end; - } - } - - end: - va_end(ap); - return 0; -} - #if 0 int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) @@ -1308,28 +1242,6 @@ sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) return 0; } -int -sandbox_cfg_allow_execve_array(sandbox_cfg_t **cfg, ...) -{ - int rc = 0; - char *fn = NULL; - - va_list ap; - va_start(ap, cfg); - - while ((fn = va_arg(ap, char*)) != NULL) { - - rc = sandbox_cfg_allow_execve(cfg, fn); - if (rc) { - log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_execve_array failed"); - goto end; - } - } - - end: - va_end(ap); - return 0; -} #endif /** Cache entry for getaddrinfo results; used when sandboxing is implemented diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 26ce218..b4688bf 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -586,3 +586,4 @@ tor_zlib_get_total_allocation(void) { return total_zlib_allocation; } + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index e04b4b9..35c5236 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -2083,7 +2083,6 @@ circuits_handle_oom(size_t current_allocation) circ->global_circuitlist_idx = circ_sl_idx; } SMARTLIST_FOREACH_END(circ); - /* Now sort the connection array ... */ now_ms_for_buf_cmp = now_ms; smartlist_sort(connection_array, conns_compare_by_buffer_age_);

1 0

[tor/master] Terser ways to sandbox-allow related filenames
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit 7c1143e11f531c3ff1126bdffdd69efd9121fa09 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun Aug 24 13:30:55 2014 -0400 Terser ways to sandbox-allow related filenames Using the *_array() functions here confused coverity, and was actually a bit longer than we needed. Now we just use macros for the repeated bits, so that we can mention a file and a suffix-appended version in one line. --- src/common/sandbox.h | 32 ---------- src/or/main.c | 163 ++++++++++++++++++++++++-------------------------- 2 files changed, 79 insertions(+), 116 deletions(-) diff --git a/src/common/sandbox.h b/src/common/sandbox.h index b4bb294..095d8d4 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -149,14 +149,6 @@ int sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file); /**DOCDOC*/ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2); -/** Function used to add a series of open allowed filenames to a supplied - * configuration. - * @param cfg sandbox configuration. - * @param ... a list of stealable pointers to permitted files. The last - * one must be NULL. -*/ -int sandbox_cfg_allow_open_filename_array(sandbox_cfg_t **cfg, ...); - /** * Function used to add a openat allowed filename to a supplied configuration. * The (char*) specifies the path to the allowed file; we steal the pointer to @@ -164,28 +156,12 @@ int sandbox_cfg_allow_open_filename_array(sandbox_cfg_t **cfg, ...); */ int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file); -/** Function used to add a series of openat allowed filenames to a supplied - * configuration. - * @param cfg sandbox configuration. - * @param ... a list of stealable pointers to permitted files. The last - * one must be NULL. - */ -int sandbox_cfg_allow_openat_filename_array(sandbox_cfg_t **cfg, ...); - #if 0 /** * Function used to add a execve allowed filename to a supplied configuration. * The (char*) specifies the path to the allowed file; that pointer is stolen. */ int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com); - -/** Function used to add a series of execve allowed filenames to a supplied - * configuration. - * @param cfg sandbox configuration. - * @param ... an array of stealable pointers to permitted files. The last - * one must be NULL. - */ -int sandbox_cfg_allow_execve_array(sandbox_cfg_t **cfg, ...); #endif /** @@ -194,14 +170,6 @@ int sandbox_cfg_allow_execve_array(sandbox_cfg_t **cfg, ...); */ int sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file); -/** Function used to add a series of stat64 allowed filenames to a supplied - * configuration. - * @param cfg sandbox configuration. - * @param ... an array of stealable pointers to permitted files. The last - * one must be NULL. - */ -int sandbox_cfg_allow_stat_filename_array(sandbox_cfg_t **cfg, ...); - /** Function used to initialise a sandbox configuration.*/ int sandbox_init(sandbox_cfg_t* cfg); diff --git a/src/or/main.c b/src/or/main.c index c7b532b..094120f 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2733,43 +2733,47 @@ sandbox_init_filter(void) sandbox_cfg_allow_openat_filename(&cfg, get_datadir_fname("cached-status")); - sandbox_cfg_allow_open_filename_array(&cfg, - get_datadir_fname("cached-certs"), - get_datadir_fname("cached-certs.tmp"), - get_datadir_fname("cached-consensus"), - get_datadir_fname("cached-consensus.tmp"), - get_datadir_fname("unverified-consensus"), - get_datadir_fname("unverified-consensus.tmp"), - get_datadir_fname("unverified-microdesc-consensus"), - get_datadir_fname("unverified-microdesc-consensus.tmp"), - get_datadir_fname("cached-microdesc-consensus"), - get_datadir_fname("cached-microdesc-consensus.tmp"), - get_datadir_fname("cached-microdescs"), - get_datadir_fname("cached-microdescs.tmp"), - get_datadir_fname("cached-microdescs.new"), - get_datadir_fname("cached-microdescs.new.tmp"), - get_datadir_fname("cached-descriptors"), - get_datadir_fname("cached-descriptors.new"), - get_datadir_fname("cached-descriptors.tmp"), - get_datadir_fname("cached-descriptors.new.tmp"), - get_datadir_fname("cached-descriptors.tmp.tmp"), - get_datadir_fname("cached-extrainfo"), - get_datadir_fname("cached-extrainfo.new"), - get_datadir_fname("cached-extrainfo.tmp"), - get_datadir_fname("cached-extrainfo.new.tmp"), - get_datadir_fname("cached-extrainfo.tmp.tmp"), - get_datadir_fname("state.tmp"), - get_datadir_fname("unparseable-desc.tmp"), - get_datadir_fname("unparseable-desc"), - get_datadir_fname("v3-status-votes"), - get_datadir_fname("v3-status-votes.tmp"), - tor_strdup("/dev/srandom"), - tor_strdup("/dev/urandom"), - tor_strdup("/dev/random"), - tor_strdup("/etc/hosts"), - tor_strdup("/proc/meminfo"), - NULL, 0 - ); +#define OPEN(name) \ + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name)) + +#define OPEN_DATADIR(name) \ + sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name)) + +#define OPEN_DATADIR2(name, name2) \ + sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname2((name), (name2))) + +#define OPEN_DATADIR_SUFFIX(name, suffix) do { \ + OPEN_DATADIR(name); \ + OPEN_DATADIR(name suffix); \ + } while (0) + +#define OPEN_DATADIR2_SUFFIX(name, name2, suffix) do { \ + OPEN_DATADIR2(name, name2); \ + OPEN_DATADIR2(name, name2 suffix); \ + } while (0) + + OPEN_DATADIR_SUFFIX("cached-certs", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("unverified-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("unverified-microdesc-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdesc-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdescs", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdescs.new", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-descriptors", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-descriptors.new", ".tmp"); + OPEN_DATADIR("cached-descriptors.tmp.tmp"); + OPEN_DATADIR_SUFFIX("cached-extrainfo", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-extrainfo.new", ".tmp"); + OPEN_DATADIR("cached-extrainfo.tmp.tmp"); + OPEN_DATADIR_SUFFIX("state", ".tmp"); + OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp"); + OPEN_DATADIR_SUFFIX("v3-status-votes", ".tmp"); + OPEN("/dev/srandom"); + OPEN("/dev/urandom"); + OPEN("/dev/random"); + OPEN("/etc/hosts"); + OPEN("/proc/meminfo"); + if (options->ServerDNSResolvConfFile) sandbox_cfg_allow_open_filename(&cfg, tor_strdup(options->ServerDNSResolvConfFile)); @@ -2810,14 +2814,17 @@ sandbox_init_filter(void) RENAME_SUFFIX("unparseable-desc", ".tmp"); RENAME_SUFFIX("v3-status-votes", ".tmp"); - sandbox_cfg_allow_stat_filename_array(&cfg, - get_datadir_fname(NULL), - get_datadir_fname("lock"), - get_datadir_fname("state"), - get_datadir_fname("router-stability"), - get_datadir_fname("cached-extrainfo.new"), - NULL, 0 - ); +#define STAT_DATADIR(name) \ + sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname(name)) + +#define STAT_DATADIR2(name, name2) \ + sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname2((name), (name2))) + + STAT_DATADIR(NULL); + STAT_DATADIR("lock"); + STAT_DATADIR("state"); + STAT_DATADIR("router-stability"); + STAT_DATADIR("cached-extrainfo.new"); { smartlist_t *files = smartlist_new(); @@ -2839,7 +2846,8 @@ sandbox_init_filter(void) sandbox_cfg_allow_rename(&cfg, tor_strdup(tmp_name), tor_strdup(file_name)); /* steals references */ - sandbox_cfg_allow_open_filename_array(&cfg, file_name, tmp_name, NULL); + sandbox_cfg_allow_open_filename(&cfg, file_name); + sandbox_cfg_allow_open_filename(&cfg, tmp_name); }); SMARTLIST_FOREACH(dirs, char *, dir, { /* steals reference */ @@ -2866,38 +2874,28 @@ sandbox_init_filter(void) // orport if (server_mode(get_options())) { - sandbox_cfg_allow_open_filename_array(&cfg, - get_datadir_fname2("keys", "secret_id_key"), - get_datadir_fname2("keys", "secret_onion_key"), - get_datadir_fname2("keys", "secret_onion_key_ntor"), - get_datadir_fname2("keys", "secret_onion_key_ntor.tmp"), - get_datadir_fname2("keys", "secret_id_key.old"), - get_datadir_fname2("keys", "secret_onion_key.old"), - get_datadir_fname2("keys", "secret_onion_key_ntor.old"), - get_datadir_fname2("keys", "secret_onion_key.tmp"), - get_datadir_fname2("keys", "secret_id_key.tmp"), - get_datadir_fname2("stats", "bridge-stats"), - get_datadir_fname2("stats", "bridge-stats.tmp"), - get_datadir_fname2("stats", "dirreq-stats"), - get_datadir_fname2("stats", "dirreq-stats.tmp"), - get_datadir_fname2("stats", "entry-stats"), - get_datadir_fname2("stats", "entry-stats.tmp"), - get_datadir_fname2("stats", "exit-stats"), - get_datadir_fname2("stats", "exit-stats.tmp"), - get_datadir_fname2("stats", "buffer-stats"), - get_datadir_fname2("stats", "buffer-stats.tmp"), - get_datadir_fname2("stats", "conn-stats"), - get_datadir_fname2("stats", "conn-stats.tmp"), - get_datadir_fname("approved-routers"), - get_datadir_fname("fingerprint"), - get_datadir_fname("fingerprint.tmp"), - get_datadir_fname("hashed-fingerprint"), - get_datadir_fname("hashed-fingerprint.tmp"), - get_datadir_fname("router-stability"), - get_datadir_fname("router-stability.tmp"), - tor_strdup("/etc/resolv.conf"), - NULL, 0 - ); + + OPEN_DATADIR2_SUFFIX("keys", "secret_id_key", "tmp"); + OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key_ntor", ".tmp"); + OPEN_DATADIR2("keys", "secret_id_key.old"); + OPEN_DATADIR2("keys", "secret_onion_key.old"); + OPEN_DATADIR2("keys", "secret_onion_key_ntor.old"); + + OPEN_DATADIR2_SUFFIX("stats", "bridge-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "dirreq-stats", ".tmp"); + + OPEN_DATADIR2_SUFFIX("stats", "entry-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "exit-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp"); + + OPEN_DATADIR("approved-routers"); + OPEN_DATADIR_SUFFIX("fingerprint", ".tmp"); + OPEN_DATADIR_SUFFIX("hashed-fingerprint", ".tmp"); + OPEN_DATADIR_SUFFIX("router-stability", ".tmp"); + + OPEN("/etc/resolv.conf"); RENAME_SUFFIX("fingerprint", ".tmp"); RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp"); @@ -2921,12 +2919,9 @@ sandbox_init_filter(void) get_datadir_fname2("keys", "secret_onion_key_ntor"), get_datadir_fname2("keys", "secret_onion_key_ntor.old")); - sandbox_cfg_allow_stat_filename_array(&cfg, - get_datadir_fname("keys"), - get_datadir_fname("stats"), - get_datadir_fname2("stats", "dirreq-stats"), - NULL, 0 - ); + STAT_DATADIR("keys"); + STAT_DATADIR("stats"); + STAT_DATADIR2("stats", "dirreq-stats"); } init_addrinfo();

1 0

[tor/master] Kill non-tunneled directory connections when handling OOM.
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit 68e430a6fb3056c1265da04fbb08004ddd449525 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 19 11:25:48 2014 -0400 Kill non-tunneled directory connections when handling OOM. Another part of 11792. --- changes/bug11792 | 4 ++ src/or/circuitlist.c | 118 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/changes/bug11792 b/changes/bug11792 index 7395b4a..66f7df8 100644 --- a/changes/bug11792 +++ b/changes/bug11792 @@ -9,3 +9,7 @@ Count that memory as reclaimed by our OOM handler. Part of ticket 11792. + - When handling out-of-memory conditions, also look at + non-tunnneled directory connections, and kill the ones that have + had data sitting on them for the longest. Part of ticket 11792. + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c738f93..bcdd858 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -21,6 +21,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -1800,7 +1801,7 @@ marked_circuit_free_cells(circuit_t *circ) } static size_t -marked_circuit_single_conn_free_bytes(connection_t *conn) +single_conn_free_bytes(connection_t *conn) { size_t result = 0; if (conn->inbuf) { @@ -1830,9 +1831,9 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream) size_t result = 0; for ( ; stream; stream = stream->next_stream) { connection_t *conn = TO_CONN(stream); - result += marked_circuit_single_conn_free_bytes(conn); + result += single_conn_free_bytes(conn); if (conn->linked_conn) { - result += marked_circuit_single_conn_free_bytes(conn->linked_conn); + result += single_conn_free_bytes(conn->linked_conn); } } return result; @@ -1890,6 +1891,28 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } +/** Return the age in milliseconds of the oldest buffer chunk on conn, + * where age is taken in milliseconds before the time now (in truncated + * milliseconds since the epoch). If the connection has no data, treat + * it as having age zero. + **/ +static uint32_t +conn_get_buffer_age(const connection_t *conn, uint32_t now) +{ + uint32_t age = 0, age2; + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + return age; +} + /** Return the age in milliseconds of the oldest buffer chunk on any stream in * the linked list stream, where age is taken in milliseconds before * the time now (in truncated milliseconds since the epoch). */ @@ -1899,18 +1922,10 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) uint32_t age = 0, age2; for (; stream; stream = stream->next_stream) { const connection_t *conn = TO_CONN(stream); - if (conn->outbuf) { - age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); - if (age2 > age) - age = age2; - } - if (conn->inbuf) { - age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); - if (age2 > age) - age = age2; - } + age2 = conn_get_buffer_age(conn, now); + if (age2 > age) + age = age2; } - return age; } @@ -1961,6 +1976,26 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) return -1; } +static uint32_t now_ms_for_buf_cmp; + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ +static int +conns_compare_by_buffer_age_(const void **a_, const void **b_) +{ + const connection_t *a = *a_; + const connection_t *b = *b_; + time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp); + time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp); + + if (age_a < age_b) + return 1; + else if (age_a == age_b) + return 0; + else + return -1; +} + #define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated current_allocation @@ -1971,10 +2006,13 @@ circuits_handle_oom(size_t current_allocation) { /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); + smartlist_t *connection_array = get_connection_array(); + int conn_idx; circuit_t *circ; size_t mem_to_recover; size_t mem_recovered=0; int n_circuits_killed=0; + int n_dirconns_killed=0; struct timeval now; uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " @@ -2013,12 +2051,46 @@ circuits_handle_oom(size_t current_allocation) /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); - - /* Okay, now the worst circuits are at the front of the list. Let's mark - * them, and reclaim their storage aggressively. */ + now_ms_for_buf_cmp = now_ms; + smartlist_sort(connection_array, conns_compare_by_buffer_age_); + now_ms_for_buf_cmp = 0; + + /* Fix up the connection array to its new order. */ + SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { + conn->conn_array_index = conn_sl_idx; + } SMARTLIST_FOREACH_END(conn); + + /* Okay, now the worst circuits and connections are at the front of their + * respective lists. Let's mark them, and reclaim their storage + * aggressively. */ + conn_idx = 0; SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { - size_t n = n_cells_in_circ_queues(circ); + size_t n; size_t freed; + + /* Free storage in any non-linked directory connections that have buffered + * data older than this circuit. */ + while (conn_idx < smartlist_len(connection_array)) { + connection_t *conn = smartlist_get(connection_array, conn_idx); + uint32_t conn_age = conn_get_buffer_age(conn, now_ms); + if (conn_age < circ->age_tmp) { + break; + } + if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + if (!conn->marked_for_close) + connection_mark_for_close(conn); + mem_recovered += single_conn_free_bytes(conn); + + ++n_dirconns_killed; + + if (mem_recovered >= mem_to_recover) + goto done_recovering_mem; + } + ++conn_idx; + } + + /* Now, kill the circuit. */ + n = n_cells_in_circ_queues(circ); if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } @@ -2031,9 +2103,11 @@ circuits_handle_oom(size_t current_allocation) mem_recovered += freed; if (mem_recovered >= mem_to_recover) - break; + goto done_recovering_mem; } SMARTLIST_FOREACH_END(circ); + done_recovering_mem: + #ifdef ENABLE_MEMPOOLS clean_cell_pool(); /* In case this helps. */ #endif /* ENABLE_MEMPOOLS */ @@ -2041,10 +2115,12 @@ circuits_handle_oom(size_t current_allocation) chunks. */ log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " - "%d circuits remain alive.", + "%d circuits remain alive. Also killed %d non-linked directory " + "connections.", U64_PRINTF_ARG(mem_recovered), n_circuits_killed, - smartlist_len(circlist) - n_circuits_killed); + smartlist_len(circlist) - n_circuits_killed, + n_dirconns_killed); smartlist_free(circlist); }

1 0

[tor/master] When looking for conns to close, count the age of linked queued data
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit d6033843a49ff029d9f8d070b3c1f446a6edc9fa Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 19 11:27:52 2014 -0400 When looking for conns to close, count the age of linked queued data Specifically, count the age of the data queued in a linked directory connection's buffers when counting a stream's age. --- src/or/circuitlist.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index bcdd858..6853420 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1925,6 +1925,11 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) age2 = conn_get_buffer_age(conn, now); if (age2 > age) age = age2; + if (conn->linked_conn) { + age2 = conn_get_buffer_age(conn->linked_conn, now); + if (age2 > age) + age = age2; + } } return age; }

1 0

[tor/master] When counting memory from closing a connection, count the dir conn too
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit ec59167cae1f5b3057ed722857d78ec78239e991 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 18 15:21:50 2014 -0400 When counting memory from closing a connection, count the dir conn too Fix part of bug 11972 --- changes/bug11792 | 5 +++++ src/or/circuitlist.c | 25 ++++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/changes/bug11792 b/changes/bug11792 new file mode 100644 index 0000000..1d38189 --- /dev/null +++ b/changes/bug11792 @@ -0,0 +1,5 @@ + o Minor features (security, OOM): + - When closing an edge connection because we've run out of memory, + also count the amount of memory that any tunnelled directory + connection attached to that connection had consumed. Part of + ticket 11792. \ No newline at end of file diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index f3a8350..9aeb3eb 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1799,6 +1799,21 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } +static size_t +marked_circuit_single_conn_free_bytes(connection_t *conn) +{ + size_t result = 0; + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + return result; +} + /** Aggressively free buffer contents on all the buffers of all streams in the * list starting at stream. Return the number of bytes recovered. */ static size_t @@ -1807,13 +1822,9 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream) size_t result = 0; for ( ; stream; stream = stream->next_stream) { connection_t *conn = TO_CONN(stream); - if (conn->inbuf) { - result += buf_allocation(conn->inbuf); - buf_clear(conn->inbuf); - } - if (conn->outbuf) { - result += buf_allocation(conn->outbuf); - buf_clear(conn->outbuf); + result += marked_circuit_single_conn_free_bytes(conn); + if (conn->linked_conn) { + result += marked_circuit_single_conn_free_bytes(conn->linked_conn); } } return result;

1 0

[tor/master] Count zlib buffer memory towards OOM totals.
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit 8e55cafd672353979e7628d5dad9e12429b401dd Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 19 10:59:15 2014 -0400 Count zlib buffer memory towards OOM totals. Part of 11792. (Uses the zlib-endorsed formula for memory needs for inflate/deflate from "zconf.h".) --- changes/bug11792 | 8 ++++++- src/common/torgzip.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/common/torgzip.h | 3 +++ src/or/circuitlist.c | 8 +++++++ src/or/relay.c | 1 + 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/changes/bug11792 b/changes/bug11792 index 1d38189..7395b4a 100644 --- a/changes/bug11792 +++ b/changes/bug11792 @@ -2,4 +2,10 @@ - When closing an edge connection because we've run out of memory, also count the amount of memory that any tunnelled directory connection attached to that connection had consumed. Part of - ticket 11792. \ No newline at end of file + ticket 11792. + + - When considering whether we're running low on memory, consider + memory that was allocated as part of zlib buffers as well. + Count that memory as reclaimed by our OOM handler. Part of + ticket 11792. + diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 15451ee..67fd742 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -46,6 +46,12 @@ #include <zlib.h> +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static size_t total_zlib_allocation = 0; + /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; * set to -1 if we haven't checked yet. */ static int gzip_is_supported = -1; @@ -411,6 +417,9 @@ struct tor_zlib_state_t { size_t input_so_far; /** Number of bytes written so far. Used to detect zlib bombs. */ size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; }; /** Construct and return a tor_zlib_state_t object using method. If @@ -420,6 +429,7 @@ tor_zlib_state_t * tor_zlib_new(int compress, compress_method_t method) { tor_zlib_state_t *out; + int bits; if (method == GZIP_METHOD && !is_gzip_supported()) { /* Old zlib version don't support gzip in inflateInit2 */ @@ -432,14 +442,19 @@ tor_zlib_new(int compress, compress_method_t method) out->stream.zfree = Z_NULL; out->stream.opaque = NULL; out->compress = compress; + bits = method_bits(method); if (compress) { if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK) + bits, 8, Z_DEFAULT_STRATEGY) != Z_OK) goto err; } else { - if (inflateInit2(&out->stream, method_bits(method)) != Z_OK) + if (inflateInit2(&out->stream, bits) != Z_OK) goto err; } + out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8); + + total_zlib_allocation += out->allocation; + return out; err: @@ -517,6 +532,8 @@ tor_zlib_free(tor_zlib_state_t *state) if (!state) return; + total_zlib_allocation -= state->allocation; + if (state->compress) deflateEnd(&state->stream); else @@ -525,3 +542,47 @@ tor_zlib_free(tor_zlib_state_t *state) tor_free(state); } +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits windowBits and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_zlib_state_size(const tor_zlib_state_t *state) +{ + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return total_zlib_allocation; +} diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 5db03fe..d407bf4 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -55,5 +55,8 @@ tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, int finish); void tor_zlib_free(tor_zlib_state_t *state); +size_t tor_zlib_state_size(const tor_zlib_state_t *state); +size_t tor_zlib_get_total_allocation(void); + #endif diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 9aeb3eb..c738f93 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1811,6 +1811,14 @@ marked_circuit_single_conn_free_bytes(connection_t *conn) result += buf_allocation(conn->outbuf); buf_clear(conn->outbuf); } + if (conn->type == CONN_TYPE_DIR) { + dir_connection_t *dir_conn = TO_DIR_CONN(conn); + if (dir_conn->zlib_state) { + result += tor_zlib_state_size(dir_conn->zlib_state); + tor_zlib_free(dir_conn->zlib_state); + dir_conn->zlib_state = NULL; + } + } return result; } diff --git a/src/or/relay.c b/src/or/relay.c index f42602d..7d7ef7a 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -2433,6 +2433,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); + alloc += tor_zlib_get_total_allocation(); if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1;

1 0

[tor/master] Merge branch 'bug11792_1_squashed'
by nickm＠torproject.org 24 Aug '14

24 Aug '14

commit 59e114832e7f67cb2fac1258c7423b6359160505 Merge: d31bcc4 d603384 Author: Nick Mathewson <nickm(a)torproject.org> Date: Sun Aug 24 13:09:08 2014 -0400 Merge branch 'bug11792_1_squashed' Conflicts: src/or/circuitlist.c changes/bug11792 | 15 ++++++ src/common/torgzip.c | 65 +++++++++++++++++++++- src/common/torgzip.h | 3 ++ src/or/circuitlist.c | 145 ++++++++++++++++++++++++++++++++++++++++++-------- src/or/relay.c | 1 + 5 files changed, 206 insertions(+), 23 deletions(-) diff --cc src/or/circuitlist.c index d6bc674,6853420..e04b4b9 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@@ -1975,7 -2009,11 +2034,9 @@@ conns_compare_by_buffer_age_(const voi void circuits_handle_oom(size_t current_allocation) { - /* Let's hope there's enough slack space for this allocation here... */ - smartlist_t *circlist = smartlist_new(); + smartlist_t *circlist; + smartlist_t *connection_array = get_connection_array(); + int conn_idx; - circuit_t *circ; size_t mem_to_recover; size_t mem_recovered=0; int n_circuits_killed=0; @@@ -2015,17 -2056,46 +2077,54 @@@ /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); + + /* Fix up the indices before we run into trouble */ + SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { + circ->global_circuitlist_idx = circ_sl_idx; + } SMARTLIST_FOREACH_END(circ); + - /* Okay, now the worst circuits are at the front of the list. Let's mark - * them, and reclaim their storage aggressively. */ ++ ++ /* Now sort the connection array ... */ + now_ms_for_buf_cmp = now_ms; + smartlist_sort(connection_array, conns_compare_by_buffer_age_); + now_ms_for_buf_cmp = 0; + + /* Fix up the connection array to its new order. */ + SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { + conn->conn_array_index = conn_sl_idx; + } SMARTLIST_FOREACH_END(conn); + + /* Okay, now the worst circuits and connections are at the front of their + * respective lists. Let's mark them, and reclaim their storage + * aggressively. */ + conn_idx = 0; SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { - size_t n = n_cells_in_circ_queues(circ); + size_t n; size_t freed; + + /* Free storage in any non-linked directory connections that have buffered + * data older than this circuit. */ + while (conn_idx < smartlist_len(connection_array)) { + connection_t *conn = smartlist_get(connection_array, conn_idx); + uint32_t conn_age = conn_get_buffer_age(conn, now_ms); + if (conn_age < circ->age_tmp) { + break; + } + if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + if (!conn->marked_for_close) + connection_mark_for_close(conn); + mem_recovered += single_conn_free_bytes(conn); + + ++n_dirconns_killed; + + if (mem_recovered >= mem_to_recover) + goto done_recovering_mem; + } + ++conn_idx; + } + + /* Now, kill the circuit. */ + n = n_cells_in_circ_queues(circ); if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } @@@ -2048,10 -2120,14 +2149,12 @@@ chunks. */ log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " - "%d circuits remain alive.", + "%d circuits remain alive. Also killed %d non-linked directory " + "connections.", U64_PRINTF_ARG(mem_recovered), n_circuits_killed, - smartlist_len(circlist) - n_circuits_killed); + smartlist_len(circlist) - n_circuits_killed, + n_dirconns_killed); - - smartlist_free(circlist); } /** Verify that cpath layer cp has all of its invariants

1 0

[stem/master] Correcting StatusEvent's docs and adding an 'arguments' attribute
by atagar＠torproject.org 24 Aug '14

24 Aug '14

commit 6d48a779c406643b8bdd2ceb4ad33299e092cbee Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 23 18:31:45 2014 -0700 Correcting StatusEvent's docs and adding an 'arguments' attribute Our StatusEvent class misdocumented its 'action' argument as being called 'message'. Also, it wasn't clear that it commonly had keyword arguments available. Technically they were available since the parent class supplied these in its keyword_args attribute, but adding an alias for that to the StatusEvent to make it clear that they're usually provided. Issues caught by sysrqb on... https://trac.torproject.org/12925 --- stem/response/events.py | 10 +++++++++- test/unit/response/events.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/stem/response/events.py b/stem/response/events.py index 0065d68..b640d2d 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -848,7 +848,8 @@ class StatusEvent(Event): :var stem.StatusType status_type: category of the status event :var stem.Runlevel runlevel: runlevel of the logged message - :var str message: logged message + :var str action: activity that caused this message + :var dict arguments: attributes about the event """ _POSITIONAL_ARGS = ('runlevel', 'action') @@ -864,6 +865,13 @@ class StatusEvent(Event): else: raise ValueError("BUG: Unrecognized status type (%s), likely an EVENT_TYPE_TO_CLASS addition without revising how 'status_type' is assigned." % self.type) + # Just an alias for our parent class' keyword_args since that already + # parses these for us. Unlike our other event types Tor commonly supplies + # arbitrary key/value pairs for these, so making an alias here to better + # draw attention that the StatusEvent will likely have them. + + self.arguments = self.keyword_args + self._log_if_unrecognized('runlevel', stem.Runlevel) diff --git a/test/unit/response/events.py b/test/unit/response/events.py index dac2609..a44d536 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -900,7 +900,7 @@ class TestEvents(unittest.TestCase): 'SUMMARY': 'Loading relay descriptors', } - self.assertEqual(expected_attr, event.keyword_args) + self.assertEqual(expected_attr, event.arguments) def test_status_event_bootstrap_stuck(self): event = _get_event(STATUS_CLIENT_BOOTSTRAP_STUCK)

1 0

[translation/tor-launcher-network-settings_completed] Update translations for tor-launcher-network-settings_completed
by translation＠torproject.org 23 Aug '14

23 Aug '14

commit 595274b4f3b2f6f33cdab803b3b14711a9c1729e Author: Translation commit bot <translation(a)torproject.org> Date: Sat Aug 23 23:45:41 2014 +0000 Update translations for tor-launcher-network-settings_completed --- nb/network-settings.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nb/network-settings.dtd b/nb/network-settings.dtd index fb5bd9b..527f6b6 100644 --- a/nb/network-settings.dtd +++ b/nb/network-settings.dtd @@ -59,8 +59,8 @@ <!ENTITY torsettings.bridgeHelp2Heading "Via nettet"> <!ENTITY torsettings.bridgeHelp2 "Bruk en nettleser for å besøke https://bridges.torproject.org"> <!ENTITY torsettings.bridgeHelp3Heading "Via e-post autosvar"> -<!ENTITY torsettings.bridgeHelp3 "Send e-post til bridges(a)torproject.org med tekstlinjen 'get bridges' som eneste innhold i e-posten.  -Men, for å gjøre det vanskeligere for en angriper å lære seg en masse bro-adresser, må du sende denne forespørselen fra en gmail.com eller yahoo.com adresse."> +<!ENTITY torsettings.bridgeHelp3.emailDesc "Send en e-post til bridges(a)torproject.org med 'get bridges' enten i emnefeltet eller i meldingen.  For å gjøre det vanskeligere for en angriper å ta rede på en mengde bro-adresser, må du sende denne forespørselen fra en av følgende i listen over e-posttilbydere (opplistet etter preferanse):"> +<!ENTITY torsettings.bridgeHelp3.emailList "https://www.riseup.net, https://mail.google.com, eller https://mail.yahoo.com"> <!ENTITY torsettings.bridgeHelp4Heading "Via supporten"> <!ENTITY torsettings.bridgeHelp4 "Som en siste utvei, kan du be om bro-adresser ved å sende en høflig e-post til help(a)rt.torproject.org.  Tenk over at en person må svare på hver slik forespørsel.">

1 0

[translation/tor-launcher-network-settings] Update translations for tor-launcher-network-settings
by translation＠torproject.org 23 Aug '14

23 Aug '14

commit ae6be3f5a5307be7b77b2dd182dc5bf720baba29 Author: Translation commit bot <translation(a)torproject.org> Date: Sat Aug 23 23:45:39 2014 +0000 Update translations for tor-launcher-network-settings --- nb/network-settings.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nb/network-settings.dtd b/nb/network-settings.dtd index b69c283..527f6b6 100644 --- a/nb/network-settings.dtd +++ b/nb/network-settings.dtd @@ -59,8 +59,8 @@ <!ENTITY torsettings.bridgeHelp2Heading "Via nettet"> <!ENTITY torsettings.bridgeHelp2 "Bruk en nettleser for å besøke https://bridges.torproject.org"> <!ENTITY torsettings.bridgeHelp3Heading "Via e-post autosvar"> -<!ENTITY torsettings.bridgeHelp3.emailDesc "Send email to bridges(a)torproject.org with the line 'get bridges' by itself in the body of the message.  However, to make it harder for an attacker to learn a lot of bridge addresses, you must send this request from one of the following email providers (listed in order of preference):"> -<!ENTITY torsettings.bridgeHelp3.emailList "https://www.riseup.net, https://mail.google.com, or https://mail.yahoo.com"> +<!ENTITY torsettings.bridgeHelp3.emailDesc "Send en e-post til bridges(a)torproject.org med 'get bridges' enten i emnefeltet eller i meldingen.  For å gjøre det vanskeligere for en angriper å ta rede på en mengde bro-adresser, må du sende denne forespørselen fra en av følgende i listen over e-posttilbydere (opplistet etter preferanse):"> +<!ENTITY torsettings.bridgeHelp3.emailList "https://www.riseup.net, https://mail.google.com, eller https://mail.yahoo.com"> <!ENTITY torsettings.bridgeHelp4Heading "Via supporten"> <!ENTITY torsettings.bridgeHelp4 "Som en siste utvei, kan du be om bro-adresser ved å sende en høflig e-post til help(a)rt.torproject.org.  Tenk over at en person må svare på hver slik forespørsel.">

1 0