March 2014 - tor-commits - lists.torproject.org

[tor/master] Merge remote-tracking branch 'public/bug10169_023' into bug10169_024
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 46118d7d7542aa960a26a08d9d5fbac33698765c Merge: 05d8111 833d027 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 4 10:54:54 2014 -0500 Merge remote-tracking branch 'public/bug10169_023' into bug10169_024 Conflicts: src/or/relay.c src/common/compat_libevent.c | 30 ++++++++++++++++++++++++++++++ src/common/compat_libevent.h | 1 + src/or/buffers.c | 2 +- src/or/circuitlist.c | 8 +++++--- src/or/relay.c | 3 ++- 5 files changed, 39 insertions(+), 5 deletions(-) diff --cc src/or/relay.c index 01143f4,4cbc8fa..857b7e9 --- a/src/or/relay.c +++ b/src/or/relay.c @@@ -2146,12 -1902,11 +2146,13 @@@ cell_queue_append(cell_queue_t *queue, /** Append a newly allocated copy of cell to the end of queue */ void -cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) +cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, + int wide_circ_ids) { struct timeval now; - packed_cell_t *copy = packed_cell_copy(cell); + packed_cell_t *copy = packed_cell_copy(cell, wide_circ_ids); - tor_gettimeofday_cached(&now); + tor_gettimeofday_cached_monotonic(&now); ++ copy->inserted_time = (uint32_t)tv_to_msec(&now); /* Remember the time when this cell was put in the queue. */

1 0

[tor/master] On OOM, also log N circuits remaining
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 79c234e0e3fa22d76029bd3b5e2c52072709cff3 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Feb 12 13:09:02 2014 -0500 On OOM, also log N circuits remaining --- src/or/circuitlist.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index ffb5e0c..45fc95b 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1589,9 +1589,11 @@ circuits_handle_oom(size_t current_allocation) buf_shrink_freelists(1); /* This is necessary to actually release buffer chunks. */ - log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", + log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " + "%d circuits remain alive.", U64_PRINTF_ARG(mem_recovered), - n_circuits_killed); + n_circuits_killed, + smartlist_len(circlist) - n_circuits_killed); smartlist_free(circlist); }

1 0

[tor/master] Merge remote-tracking branch 'public/bug10169_024' into bug10169_025_v2
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit bb375442141b4a5b301212394ce3e106cb34daf2 Merge: c8d41da 46118d7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 4 11:00:02 2014 -0500 Merge remote-tracking branch 'public/bug10169_024' into bug10169_025_v2 Conflicts: src/common/compat_libevent.h src/or/relay.c src/common/compat_libevent.c | 30 ++++++++++++++++++++++++++++++ src/common/compat_libevent.h | 1 + src/or/buffers.c | 2 +- src/or/circuitlist.c | 8 +++++--- src/or/relay.c | 3 ++- 5 files changed, 39 insertions(+), 5 deletions(-) diff --cc src/common/compat_libevent.c index b7987bc,835d3b8..61cbe91 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@@ -661,15 -651,35 +661,45 @@@ tor_gettimeofday_cache_clear(void { cached_time_hires.tv_sec = 0; } + +#ifdef TOR_UNIT_TESTS +/** For testing: force-update the cached time to a given value. */ +void +tor_gettimeofday_cache_set(const struct timeval *tv) +{ + tor_assert(tv); + memcpy(&cached_time_hires, tv, sizeof(*tv)); +} +#endif #endif + /** + * As tor_gettimeofday_cached, but can never move backwards in time. + * + * The returned value may diverge from wall-clock time, since wall-clock time + * can trivially be adjusted backwards, and this can't. Don't mix wall-clock + * time with these values in the same calculation. + * + * Depending on implementation, this function may or may not "smooth out" huge + * jumps forward in wall-clock time. It may or may not keep its results + * advancing forward (as opposed to stalling) if the wall-clock time goes + * backwards. The current implementation does neither of of these. + * + * This function is not thread-safe; do not call it outside the main thread. + * + * In future versions of Tor, this may return a time does not have its + * origin at the Unix epoch. + */ + void + tor_gettimeofday_cached_monotonic(struct timeval *tv) + { + struct timeval last_tv = { 0, 0 }; + + tor_gettimeofday_cached(tv); + if (timercmp(tv, &last_tv, <)) { + memcpy(tv, &last_tv, sizeof(struct timeval)); + } else { + memcpy(&last_tv, tv, sizeof(struct timeval)); + } + } + diff --cc src/common/compat_libevent.h index 17e0523,b5f6445..f0d1828 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@@ -91,9 -90,7 +91,10 @@@ int tor_add_bufferevent_to_rate_limit_g void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); +#ifdef TOR_UNIT_TESTS +void tor_gettimeofday_cache_set(const struct timeval *tv); +#endif + void tor_gettimeofday_cached_monotonic(struct timeval *tv); #endif diff --cc src/or/relay.c index dbc1710,857b7e9..93e1ddd --- a/src/or/relay.c +++ b/src/or/relay.c @@@ -2148,12 -2151,40 +2148,13 @@@ cell_queue_append_packed_copy(circuit_ { struct timeval now; packed_cell_t *copy = packed_cell_copy(cell, wide_circ_ids); + (void)circ; + (void)exitward; + (void)use_stats; - tor_gettimeofday_cached(&now); + tor_gettimeofday_cached_monotonic(&now); + copy->inserted_time = (uint32_t)tv_to_msec(&now); - /* Remember the time when this cell was put in the queue. */ - /*XXXX This may be obsoleted by inserted_time */ - if (get_options()->CellStatistics) { - uint32_t added; - insertion_time_queue_t *it_queue = queue->insertion_times; - if (!it_pool) - it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); - -#define SECONDS_IN_A_DAY 86400L - added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) - + ((uint32_t)now.tv_usec / (uint32_t)10000L)); - if (!it_queue) { - it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); - queue->insertion_times = it_queue; - } - if (it_queue->last && it_queue->last->insertion_time == added) { - it_queue->last->counter++; - } else { - insertion_time_elem_t *elem = mp_pool_get(it_pool); - elem->next = NULL; - elem->insertion_time = added; - elem->counter = 1; - if (it_queue->last) { - it_queue->last->next = elem; - it_queue->last = elem; - } else { - it_queue->first = it_queue->last = elem; - } - } - } cell_queue_append(queue, copy); }

1 0

[tor/master] More unit tests for OOM handling.
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit c8d41da52d6ae1edd1c4999e328b1e7eadc0ab5b Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 13 15:25:28 2014 -0500 More unit tests for OOM handling. This gets us up to no interesting untested new or changed lines for the 10169 code. --- src/or/circuitlist.c | 6 +- src/or/circuitlist.h | 3 + src/test/test_oom.c | 157 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 159 insertions(+), 7 deletions(-) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 618a42e..01d5aee 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1664,7 +1664,7 @@ n_cells_in_circ_queues(const circuit_t *c) * This function will return incorrect results if the oldest cell queued on * the circuit is older than 2**32 msec (about 49 days) old. */ -static uint32_t +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) { uint32_t age = 0; @@ -1711,7 +1711,7 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) /** Return the age in milliseconds of the oldest buffer chunk on any stream * attached to the circuit c, where age is taken in milliseconds before * the time now (in truncated milliseconds since the epoch). */ -static uint32_t +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now) { if (CIRCUIT_IS_ORIGIN(c)) { @@ -1726,7 +1726,7 @@ circuit_max_queued_data_age(const circuit_t *c, uint32_t now) /** Return the age of the oldest cell or stream buffer chunk on the circuit * c, where age is taken in milliseconds before the time now (in * truncated milliseconds since the epoch). */ -static uint32_t +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now) { uint32_t cell_age = circuit_max_queued_cell_age(c, now); diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index a6340c1..a29c29a 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -76,6 +76,9 @@ void channel_note_destroy_not_pending(channel_t *chan, circid_t id); #ifdef CIRCUITLIST_PRIVATE STATIC void circuit_free(circuit_t *circ); STATIC size_t n_cells_in_circ_queues(const circuit_t *c); +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now); #endif #endif diff --git a/src/test/test_oom.c b/src/test/test_oom.c index 3b636e6..cc6e532 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -76,7 +76,6 @@ dummy_origin_circuit_new(int n_cells) return TO_CIRCUIT(circ); } -#if 0 static void add_bytes_to_buf(generic_buffer_t *buf, size_t n_bytes) { @@ -91,18 +90,34 @@ add_bytes_to_buf(generic_buffer_t *buf, size_t n_bytes) } static edge_connection_t * -dummy_edge_conn_new(int type, size_t in_bytes, size_t out_bytes) +dummy_edge_conn_new(circuit_t *circ, + int type, size_t in_bytes, size_t out_bytes) { - edge_connection_t *conn = edge_connection_new(type, AF_INET); + edge_connection_t *conn; + + if (type == CONN_TYPE_EXIT) + conn = edge_connection_new(type, AF_INET); + else + conn = ENTRY_TO_EDGE_CONN(entry_connection_new(type, AF_INET)); /* We add these bytes directly to the buffers, to avoid all the * edge connection read/write machinery. */ add_bytes_to_buf(TO_CONN(conn)->inbuf, in_bytes); add_bytes_to_buf(TO_CONN(conn)->outbuf, out_bytes); + conn->on_circuit = circ; + if (type == CONN_TYPE_EXIT) { + or_circuit_t *oc = TO_OR_CIRCUIT(circ); + conn->next_stream = oc->n_streams; + oc->n_streams = conn; + } else { + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + conn->next_stream = oc->p_streams; + oc->p_streams = conn; + } + return conn; } -#endif /** Run unit tests for buffers.c */ static void @@ -192,8 +207,142 @@ test_oom_circbuf(void *arg) UNMOCK(circuit_mark_for_close_); } +/** Run unit tests for buffers.c */ +static void +test_oom_streambuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL, *c5 = NULL; + struct timeval tv = { 1389641159, 0 }; + uint32_t tvms; + int i; + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + init_cell_pool(); + + /* Far too low for real life. */ + options->MaxMemInQueues = 81*packed_cell_mem_cost() + 4096 * 34; + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Start all circuits with a bit of data queued in cells */ + tv.tv_usec = 500*1000; /* go halfway into the second. */ + tor_gettimeofday_cache_set(&tv); + c1 = dummy_or_circuit_new(10,10); + tv.tv_usec = 510*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_origin_circuit_new(20); + tv.tv_usec = 520*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(20,20); + tv.tv_usec = 530*1000; + tor_gettimeofday_cache_set(&tv); + c4 = dummy_or_circuit_new(0,0); + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + + tv.tv_usec = 600*1000; + tor_gettimeofday_cache_set(&tv); + + /* Add some connections to c1...c4. */ + for (i = 0; i < 4; ++i) { + edge_connection_t *ec; + /* link it to a circuit */ + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c1, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c2, CONN_TYPE_AP, 1000, 1000); + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); /* Yes, 4 twice*/ + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + } + + tv.tv_sec += 1; + tv.tv_usec = 0; + tvms = (uint32_t) tv_to_msec(&tv); + + tt_int_op(circuit_max_queued_cell_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_cell_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_cell_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_cell_age(c4, tvms), ==, 0); + + tt_int_op(circuit_max_queued_data_age(c1, tvms), ==, 390); + tt_int_op(circuit_max_queued_data_age(c2, tvms), ==, 380); + tt_int_op(circuit_max_queued_data_age(c3, tvms), ==, 0); + tt_int_op(circuit_max_queued_data_age(c4, tvms), ==, 370); + + tt_int_op(circuit_max_queued_item_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_item_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_item_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 370); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + tt_int_op(buf_get_total_allocation(), ==, 4096*16*2); + + /* Now give c4 a very old buffer of modest size */ + { + edge_connection_t *ec; + tv.tv_sec -= 1; + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + } + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 1000); + + tt_int_op(cell_queues_check_size(), ==, 0); + + /* And run over the limit. */ + tv.tv_usec = 800*1000; + tor_gettimeofday_cache_set(&tv); + c5 = dummy_or_circuit_new(0,5); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + /* C4 should have died. */ + tt_assert(! c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(c4->marked_for_close); + tt_assert(! c5->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*8*2); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + circuit_free(c5); + + UNMOCK(circuit_mark_for_close_); +} + struct testcase_t oom_tests[] = { { "circbuf", test_oom_circbuf, TT_FORK, NULL, NULL }, + { "streambuf", test_oom_streambuf, TT_FORK, NULL, NULL }, END_OF_TESTCASES };

1 0

[tor/master] Have the OOM handler also count the age the data in a stream buffer
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 91ec6f7269bd7a5b73629f38e9779e84a0fb84f2 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Nov 15 18:38:52 2013 -0500 Have the OOM handler also count the age the data in a stream buffer --- src/or/buffers.c | 41 +++++++++++++++++++++++++++ src/or/buffers.h | 3 ++ src/or/circuitlist.c | 75 ++++++++++++++++++++++++++++++++++++++++---------- src/or/or.h | 3 ++ src/or/relay.c | 1 + 5 files changed, 108 insertions(+), 15 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 9be0476..4b176a8 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -68,6 +68,8 @@ typedef struct chunk_t { size_t datalen; /**< The number of bytes stored in this chunk */ size_t memlen; /**< The number of usable bytes of storage in mem. */ char *data; /**< A pointer to the first byte of data stored in mem. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in * this chunk. */ } chunk_t; @@ -139,6 +141,9 @@ static chunk_freelist_t freelists[] = { * could help with? */ static uint64_t n_freelist_miss = 0; +/** DOCDOC */ +static size_t total_bytes_allocated_in_chunks = 0; + static void assert_freelist_ok(chunk_freelist_t *fl); /** Return the freelist to hold chunks of size alloc, or NULL if @@ -172,6 +177,8 @@ chunk_free_unchecked(chunk_t *chunk) } else { if (freelist) ++freelist->n_free; + tor_assert(total_bytes_allocated_in_chunks >= alloc); + total_bytes_allocated_in_chunks -= alloc; tor_free(chunk); } } @@ -200,6 +207,7 @@ chunk_new_with_alloc_size(size_t alloc) else ++n_freelist_miss; ch = tor_malloc(alloc); + total_bytes_allocated_in_chunks += alloc; } ch->next = NULL; ch->datalen = 0; @@ -211,6 +219,10 @@ chunk_new_with_alloc_size(size_t alloc) static void chunk_free_unchecked(chunk_t *chunk) { + if (!chunk) + return; + tor_assert(total_bytes_allocated_in_chunks >= CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); } static INLINE chunk_t * @@ -221,6 +233,7 @@ chunk_new_with_alloc_size(size_t alloc) ch->next = NULL; ch->datalen = 0; ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; ch->data = &ch->mem[0]; return ch; } @@ -237,6 +250,7 @@ chunk_grow(chunk_t *chunk, size_t sz) chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); chunk->memlen = sz; chunk->data = chunk->mem + offset; + total_bytes_allocated_in_chunks += (sz - chunk->memlen); return chunk; } @@ -298,6 +312,8 @@ buf_shrink_freelists(int free_all) *chp = NULL; while (chunk) { chunk_t *next = chunk->next; + tor_assert(total_bytes_allocated_in_chunks >= CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); chunk = next; --n_to_free; @@ -599,6 +615,7 @@ static chunk_t * buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) { chunk_t *chunk; + struct timeval now; if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { chunk = chunk_new_with_alloc_size(buf->default_chunk_size); } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { @@ -606,6 +623,10 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) } else { chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); } + + tor_gettimeofday_cached(&now); + chunk->inserted_time = (uint32_t)tv_to_msec(&now); + if (buf->tail) { tor_assert(buf->head); buf->tail->next = chunk; @@ -618,6 +639,26 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } +/** Return the age of the oldest chunk in the buffer buf, in + * milliseconds. Requires the current time, in truncated milliseconds since + * the epoch, as its input now. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + /** Read up to at_most bytes from the socket fd into * chunk (which must be on buf). If we get an EOF, set * *reached_eof to 1. Return -1 on error, 0 on eof or blocking, diff --git a/src/or/buffers.h b/src/or/buffers.h index a5886ad..677d68d 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -25,6 +25,9 @@ size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 6250c11..27fe5ab 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1409,20 +1409,63 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** Temporary variable for circuits_compare_by_oldest_queued_cell_ This is a - * kludge to work around the fact that qsort doesn't provide a way for - * comparison functions to take an extra argument. */ -static uint32_t circcomp_now_tmp; +/** DOCDOC*/ +static uint32_t +circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) +{ + uint32_t age = 0, age2; + for (; stream; stream = stream->next_stream) { + const connection_t *conn = TO_CONN(stream); + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + } + + return age; +} + +/** DOCDOC + */ +static uint32_t +circuit_max_queued_data_age(const circuit_t *c, uint32_t now) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return circuit_get_streams_max_data_age( + TO_ORIGIN_CIRCUIT((circuit_t*)c)->p_streams, now); + } else { + return circuit_get_streams_max_data_age( + TO_OR_CIRCUIT((circuit_t*)c)->n_streams, now); + } +} + +/** DATA */ +static uint32_t +circuit_max_queued_item_age(const circuit_t *c, uint32_t now) +{ + uint32_t cell_age = circuit_max_queued_cell_age(c, now); + uint32_t data_age = circuit_max_queued_data_age(c, now); + if (cell_age > data_age) + return cell_age; + else + return data_age; +} -/** Helper to sort a list of circuit_t by age of oldest cell, in descending - * order. Requires that circcomp_now_tmp is set correctly. */ +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ static int -circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) +circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) { const circuit_t *a = *a_; const circuit_t *b = *b_; - uint32_t age_a = circuit_max_queued_cell_age(a, circcomp_now_tmp); - uint32_t age_b = circuit_max_queued_cell_age(b, circcomp_now_tmp); + uint32_t age_a = a->age_tmp; + uint32_t age_b = b->age_tmp; if (age_a < age_b) return 1; @@ -1446,6 +1489,7 @@ circuits_handle_oom(size_t current_allocation) size_t n_cells_removed=0, n_cells_to_remove; int n_circuits_killed=0; struct timeval now; + uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " "MaxMemInCellQueues.)"); @@ -1460,18 +1504,19 @@ circuits_handle_oom(size_t current_allocation) n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } + tor_gettimeofday_cached(&now); + now_ms = (uint32_t)tv_to_msec(&now); + /* This algorithm itself assumes that you've got enough memory slack * to actually run it. */ - for (circ = global_circuitlist; circ; circ = circ->next) + for (circ = global_circuitlist; circ; circ = circ->next) { + circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); smartlist_add(circlist, circ); - - /* Set circcomp_now_tmp so that the sort can work. */ - tor_gettimeofday_cached(&now); - circcomp_now_tmp = (uint32_t)tv_to_msec(&now); + } /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ - smartlist_sort(circlist, circuits_compare_by_oldest_queued_cell_); + smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); /* Okay, now the worst circuits are at the front of the list. Let's mark * them, and reclaim their storage aggressively. */ diff --git a/src/or/or.h b/src/or/or.h index 4622391..0dd47d0 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2564,6 +2564,9 @@ typedef struct circuit_t { * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + /** For storage while n_conn is pending * (state CIRCUIT_STATE_OR_WAIT). When defined, it is always * length ONIONSKIN_CHALLENGE_LEN. */ diff --git a/src/or/relay.c b/src/or/relay.c index a193ad8..042fc18 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1998,6 +1998,7 @@ static int cell_queues_check_size(void) { size_t alloc = total_cells_allocated * packed_cell_mem_cost(); + alloc += buf_get_total_allocation(); if (alloc >= get_options()->MaxMemInCellQueues) { circuits_handle_oom(alloc); return 1;

1 0

[tor/master] Mention doc/HACKING in the readme
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit bfa0e022bc6ec629473d3dfb598cad698ceee256 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 4 10:34:25 2014 -0500 Mention doc/HACKING in the readme --- README | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README b/README index 812680c..b2a391a 100644 --- a/README +++ b/README @@ -22,3 +22,6 @@ Frequently Asked Questions: https://www.torproject.org/faq.html https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ + +To get started working on Tor development: + See the doc/HACKING file.

1 0

[torspec/master] Defang TLS UKS attacks in tor-spec.txt
by nickm＠torproject.org 04 Mar '14

04 Mar '14

commit 426e9ac1069ee843000aaeed9260ba4c9733af00 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 4 10:23:40 2014 -0500 Defang TLS UKS attacks in tor-spec.txt Our implementations disable session resumption, and don't send AUTHENTICATE before checking CERTS. It turns out that these are pretty important: see discussion of "triple handshake" and related UKS-enabled attacks at https://lists.torproject.org/pipermail/tor-dev/2014-March/006372.html Basically, session resumption plus renegotiation enables the triple- handshake attack. And sending an AUTHENTICATE cell before checking the CERTS cell would lead to sending the wrong SID field in the AUTHENTICATE cell, which would whittle our defense there even further. --- tor-spec.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tor-spec.txt b/tor-spec.txt index c3b96a3..604a1e7 100644 --- a/tor-spec.txt +++ b/tor-spec.txt @@ -355,6 +355,12 @@ see tor-design.pdf. less than HASH_LEN bits. Responders SHOULD NOT select any SSLv3 ciphersuite other than the DHE+3DES suites listed above. +2.2. TLS security considerations + + Implementations MUST NOT allow TLS session resumption -- it can + exacerbate some attacks (e.g. the "Triple Handshake" attack from + Feb 2013), and it plays havoc with forward secrecy guarantees. + 3. Cell Packet format The basic unit of communication for onion routers and onion @@ -645,6 +651,10 @@ see tor-design.pdf. signature. The server MUST ignore any extra bytes in the signed data after the SHA256 hash. + Initiators MUST NOT send an AUTHENTICATE cell before they have + verified the certificates presented in the responder's CERTS + cell, and authenticated the responder. + 4.5. NETINFO cells If version 2 or higher is negotiated, each party sends the other a

1 0

[bridgedb/develop] Skip coverage branch testing for those which include ``pass``.
by isis＠torproject.org 04 Mar '14

04 Mar '14

commit 63a41751b7a05f340196af8c2871770ad805a480 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Mar 4 04:30:33 2014 +0000 Skip coverage branch testing for those which include ``pass``. --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 46c0c56..569fd88 100644 --- a/.coveragerc +++ b/.coveragerc @@ -18,6 +18,7 @@ exclude_lines = pragma: no cover # don't complain if the code doesn't hit unimplemented sections: raise NotImplementedError + pass # don't complain if non-runnable or debuging code isn't run: if 0: if False:

1 0

[bridgedb/develop] Change a test_crypto class name and add a docstring.
by isis＠torproject.org 04 Mar '14

04 Mar '14

commit e89059d9d4741c63d73d7a49eff1856d87a4a7e9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Mar 4 04:34:37 2014 +0000 Change a test_crypto class name and add a docstring. --- lib/bridgedb/test/test_crypto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_crypto.py b/lib/bridgedb/test/test_crypto.py index 2e6894e..cc1ee9a 100644 --- a/lib/bridgedb/test/test_crypto.py +++ b/lib/bridgedb/test/test_crypto.py @@ -24,7 +24,8 @@ SEKRIT_KEY = b'v\x16Xm\xfc\x1b}\x063\x85\xaa\xa5\xf9\xad\x18\xb2P\x93\xc6k\xf9' SEKRIT_KEY += b'\x8bI\xd9\xb8xw\xf5\xec\x1b\x7f\xa8' -class CryptoTest(unittest.TestCase): +class GetKeyTests(unittest.TestCase): + """Tests for :func:`bridgedb.crypto.getKey`.""" def test_getKey_nokey(self): """Test retrieving the secret_key from an empty file."""

1 0

[bridgedb/develop] Remove unused HTTPServer.WebResourceBridges.domains attribute.
by isis＠torproject.org 04 Mar '14

04 Mar '14

commit 7a439e59c712c0dd1b425044a7493315c0a53476 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Mar 4 04:38:08 2014 +0000 Remove unused HTTPServer.WebResourceBridges.domains attribute. This was left over from commit 120c8e6d4b9d714e79722d83c19e110120977e79, wherein the HTTPS and email distributors were split into different files. The domains are taken from config.EMAIL_DOMAINS and are only applicable to the email server. --- lib/bridgedb/HTTPServer.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 724e8cd..b679ff5 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -249,7 +249,7 @@ class WebResourceBridges(twisted.web.resource.Resource): isLeaf = True def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, - includeFingerprints=True, domains=None): + includeFingerprints=True): """Create a new WebResource. :type distributor: :class:`IPBasedDistributor` @@ -273,10 +273,6 @@ class WebResourceBridges(twisted.web.resource.Resource): self.useForwardedHeader = useForwardedHeader self.includeFingerprints = includeFingerprints - # do not use mutable types as __init__ defaults! - if not domains: domains = [] - self.domains = domains - def render(self, request): """Render a response for a client HTTP request. @@ -481,8 +477,7 @@ def addWebServer(cfg, dist, sched): resource = WebResourceBridges(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER, cfg.HTTP_USE_IP_FROM_FORWARDED_HEADER, - includeFingerprints=cfg.HTTPS_INCLUDE_FINGERPRINTS, - domains=cfg.EMAIL_DOMAINS) + includeFingerprints=cfg.HTTPS_INCLUDE_FINGERPRINTS) resource_options = WebResourceOptions()

1 0