May 2017 - tor-commits - lists.torproject.org

[tor/master] Use compression_method_get_by_name() instead of explicit checks.
by nickm＠torproject.org 15 May '17

15 May '17

commit f8218b5adaa54c61c3b79ad7f055e1a4a433daa0 Author: Alexander Færøy <ahf(a)torproject.org> Date: Fri May 12 12:21:49 2017 +0200 Use compression_method_get_by_name() instead of explicit checks. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 78e0fb3..bcd2eba 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2057,20 +2057,15 @@ parse_http_response(const char *headers, int *code, time_t *date, if (!strcmpstart(s, "Content-Encoding: ")) { enc = s+18; break; }); - if (!enc || !strcmp(enc, "identity")) { + + if (enc == NULL) *compression = NO_METHOD; - } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) { - *compression = ZLIB_METHOD; - } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) { - *compression = GZIP_METHOD; - } else if (!strcmp(enc, "x-zstd")) { - *compression = ZSTD_METHOD; - } else if (!strcmp(enc, "x-tor-lzma")) { - *compression = LZMA_METHOD; - } else { - log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", - escaped(enc)); - *compression = UNKNOWN_METHOD; + else { + *compression = compression_method_get_by_name(enc); + + if (*compression == UNKNOWN_METHOD) + log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.", + escaped(enc)); } } SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));

1 0

[tor/master] Add API for getting human readable descriptions of a compress_method_t
by nickm＠torproject.org 15 May '17

15 May '17

commit 3a05687c6d4c1ed11707e53b5dbec1b81e2a9c0a Author: Alexander Færøy <ahf(a)torproject.org> Date: Fri May 12 12:45:00 2017 +0200 Add API for getting human readable descriptions of a compress_method_t See: https://bugs.torproject.org/21667 --- src/common/compress.c | 26 ++++++++++++++++++++++++++ src/common/compress.h | 1 + 2 files changed, 27 insertions(+) diff --git a/src/common/compress.c b/src/common/compress.c index 6fe4569..6513029 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -343,6 +343,32 @@ compression_method_get_name(compress_method_t method) return NULL; } +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * method, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + /** Return the compression method represented by the string name, or * UNKNOWN_METHOD if the string isn't recognized. */ compress_method_t diff --git a/src/common/compress.h b/src/common/compress.h index 5b47c5d..7c0dc14 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -50,6 +50,7 @@ int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); int tor_compress_supports_method(compress_method_t method); unsigned tor_compress_get_supported_method_bitmask(void); const char * compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); compress_method_t compression_method_get_by_name(const char *name); const char *tor_compress_version_str(compress_method_t method);

1 0

[tor/master] Decide compression method in the various handle_* functions().
by nickm＠torproject.org 15 May '17

15 May '17

commit cf2f7a1bea3beb03368fc309e7bf7d9ba598b6dd Author: Alexander Færøy <ahf(a)torproject.org> Date: Thu May 11 03:32:51 2017 +0200 Decide compression method in the various handle_* functions(). See: https://bugs.torproject.org/21667 --- src/or/directory.c | 89 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index ca49388..78e0fb3 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3706,6 +3706,22 @@ find_best_diff(const smartlist_t *digests, int flav, return NULL; } +/** Try to find the best supported compression method possible from a given + * compression_methods. Return NO_METHOD if no mutually supported + * compression method could be found. */ +static compress_method_t +find_best_compression_method(unsigned compression_methods) +{ + unsigned u; + for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { + compress_method_t method = srv_meth_pref_precompressed[u]; + if (compression_methods & (1u<<method)) + return method; + } + + return NO_METHOD; +} + /** Helper function for GET /tor/status-vote/current/consensus */ static int @@ -3713,7 +3729,8 @@ handle_get_current_consensus(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); const time_t if_modified_since = args->if_modified_since; int clear_spool = 0; @@ -3786,11 +3803,11 @@ handle_get_current_consensus(dir_connection_t *conn, } else if (flavor) { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, (uint8_t*)flavor, strlen(flavor)); - compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + compression_used = compress_method; } else { spooled = spooled_resource_new(DIR_SPOOL_NETWORKSTATUS, NULL, 0); - compression_used = compressed ? ZLIB_METHOD : NO_METHOD; + compression_used = compress_method; } tor_free(flavor); smartlist_add(conn->spool, spooled); @@ -3806,7 +3823,7 @@ handle_get_current_consensus(dir_connection_t *conn, size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since, - compressed, + compress_method != NO_METHOD, &size_guess, &n_expired); @@ -3848,8 +3865,8 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compression_used, smartlist_len(conn->spool) == 1 ? lifetime : 0); - if (! compressed) - conn->compress_state = tor_compress_new(0, ZLIB_METHOD, + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(0, compress_method, HIGH_COMPRESSION); /* Prime the connection with some data. */ @@ -3870,7 +3887,8 @@ static int handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); { int current; ssize_t body_len = 0; @@ -3927,13 +3945,13 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed + body_len += compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); - if (compressed) { + if (compress_method != NO_METHOD) { estimated_len += ln/2; } else { body_len += ln; estimated_len += ln; @@ -3945,12 +3963,12 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } write_http_response_header(conn, body_len ? body_len : -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compress_method, lifetime); if (smartlist_len(items)) { - if (compressed) { - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_compress(c, strlen(c), conn, 0)); @@ -3961,9 +3979,9 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed + connection_write_to_buf(compress_method != NO_METHOD ? d->dir_compressed : d->dir, - compressed + compress_method != NO_METHOD ? d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } @@ -3982,7 +4000,8 @@ static int handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); int clear_spool = 1; { conn->spool = smartlist_new(); @@ -3993,7 +4012,8 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); size_t size_guess = 0; - dirserv_spool_remove_missing_and_guess_size(conn, 0, compressed, + dirserv_spool_remove_missing_and_guess_size(conn, 0, + compress_method != NO_METHOD, &size_guess, NULL); if (smartlist_len(conn->spool) == 0) { write_http_status_line(conn, 404, "Not found"); @@ -4009,11 +4029,11 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) clear_spool = 0; write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, + compress_method, MICRODESC_CACHE_LIFETIME); - if (compressed) - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -4034,7 +4054,8 @@ static int handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); const or_options_t *options = get_options(); int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || @@ -4073,8 +4094,8 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) size_t size_guess = 0; int n_expired = 0; dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff, - compressed, &size_guess, - &n_expired); + compress_method != NO_METHOD, + &size_guess, &n_expired); /* If we are the bridge authority and the descriptor is a bridge * descriptor, remember that we served this descriptor for desc stats. */ @@ -4104,11 +4125,9 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) dir_conn_clear_spool(conn); goto done; } - write_http_response_header(conn, -1, - compressed ? ZLIB_METHOD : NO_METHOD, - cache_lifetime); - if (compressed) - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + write_http_response_header(conn, -1, compress_method, cache_lifetime); + if (compress_method != NO_METHOD) + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -4129,7 +4148,8 @@ static int handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; - const int compressed = args->compression_supported & (1u << ZLIB_METHOD); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new(); @@ -4192,16 +4212,19 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) SMARTLIST_FOREACH(certs, authority_cert_t *, c, len += c->cache_info.signed_descriptor_len); - if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { + if (global_write_bucket_low(TO_CONN(conn), + compress_method != NO_METHOD ? len/2 : len, + 2)) { write_http_status_line(conn, 503, "Directory busy, try again later"); goto keys_done; } - write_http_response_header(conn, compressed?-1:len, - compressed ? ZLIB_METHOD : NO_METHOD, + write_http_response_header(conn, + compress_method != NO_METHOD ? -1 : len, + compress_method, 60*60); - if (compressed) { - conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_compress(

1 0

[tor/master] Handle Zstandard and LZMA in our check for correct guessed compression.
by nickm＠torproject.org 15 May '17

15 May '17

commit 61b6de5906b92fa1354efe60aa1506ab215d60b3 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon May 8 14:00:44 2017 +0200 Handle Zstandard and LZMA in our check for correct guessed compression. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/or/directory.c b/src/or/directory.c index c7c3d38..3df5be9 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2309,6 +2309,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) description1 = "as deflated"; else if (compression == GZIP_METHOD) description1 = "as gzipped"; + else if (compression == ZSTD_METHOD) + description1 = "as Zstandard compressed"; + else if (compression == LZMA_METHOD) + description1 = "as LZMA compressed"; else if (compression == NO_METHOD) description1 = "as uncompressed"; else @@ -2317,6 +2321,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) description2 = "deflated"; else if (guessed == GZIP_METHOD) description2 = "gzipped"; + else if (guessed == ZSTD_METHOD) + description2 = "Zstandard compressed"; + else if (guessed == LZMA_METHOD) + description2 = "LZMA compressed"; else if (!plausible) description2 = "confusing binary junk"; else

1 0

[tor/master] Use tor_compress_supports_method() instead of constants.
by nickm＠torproject.org 15 May '17

15 May '17

commit 630563719728e039364b1e4cc2ada9017405a383 Author: Alexander Færøy <ahf(a)torproject.org> Date: Mon May 8 16:25:20 2017 +0200 Use tor_compress_supports_method() instead of constants. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 3df5be9..56d1cca 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2336,14 +2336,14 @@ connection_dir_client_reached_eof(dir_connection_t *conn) description2, (compression>0 && guessed>0)?" Trying both.":""); } - /* Try declared compression first if we can. */ - if (compression == GZIP_METHOD || compression == ZLIB_METHOD) + /* Try declared compression first if we can. + * tor_compress_supports_method() also returns true for NO_METHOD. */ + if (tor_compress_supports_method(compression)) tor_uncompress(&new_body, &new_len, body, body_len, compression, !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ - if (!new_body && - (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && + if (!new_body && tor_compress_supports_method(guessed) && compression != guessed) tor_uncompress(&new_body, &new_len, body, body_len, guessed, !allow_partial, LOG_PROTOCOL_WARN);

1 0

[tor/master] Use dir_compressed(_len) instead of dir_z(_len).
by nickm＠torproject.org 15 May '17

15 May '17

commit ca632144e59304ce2e41a32ddd1cf302d651cc68 Author: Alexander Færøy <ahf(a)torproject.org> Date: Thu May 4 16:21:05 2017 +0200 Use dir_compressed(_len) instead of dir_z(_len). This patch renames `dir_z` to `dir_compressed` and `dir_z_len` to `dir_compressed_len`. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 10 +++++++--- src/or/dirserv.c | 12 ++++++------ src/or/or.h | 4 ++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index da92e0f..83a2e6c 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3885,7 +3885,9 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) goto vote_done; } SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - body_len += compressed ? d->dir_z_len : d->dir_len); + body_len += compressed + ? d->dir_compressed_len + : d->dir_len); estimated_len += body_len; SMARTLIST_FOREACH(items, const char *, item, { size_t ln = strlen(item); @@ -3917,8 +3919,10 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) } } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, - connection_write_to_buf(compressed ? d->dir_z : d->dir, - compressed ? d->dir_z_len : d->dir_len, + connection_write_to_buf(compressed + ? d->dir_compressed : d->dir, + compressed + ? d->dir_compressed_len : d->dir_len, TO_CONN(conn))); } vote_done: diff --git a/src/or/dirserv.c b/src/or/dirserv.c index c25dbf8..bbff13f 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1179,8 +1179,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len), + d->dir, d->dir_len, ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; @@ -1191,7 +1191,7 @@ static void clear_cached_dir(cached_dir_t *d) { tor_free(d->dir); - tor_free(d->dir_z); + tor_free(d->dir_compressed); memset(d, 0, sizeof(cached_dir_t)); } @@ -3508,7 +3508,7 @@ spooled_resource_estimate_size(const spooled_resource_t *spooled, if (cached == NULL) { return 0; } - size_t result = compressed ? cached->dir_z_len : cached->dir_len; + size_t result = compressed ? cached->dir_compressed_len : cached->dir_len; return result; } } @@ -3567,8 +3567,8 @@ spooled_resource_flush_some(spooled_resource_t *spooled, int64_t total_len; const char *ptr; if (cached) { - total_len = cached->dir_z_len; - ptr = cached->dir_z; + total_len = cached->dir_compressed_len; + ptr = cached->dir_compressed; } else { total_len = spooled->cce_len; ptr = (const char *)spooled->cce_body; diff --git a/src/or/or.h b/src/or/or.h index 297ec47..acbf8ce 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1934,9 +1934,9 @@ typedef struct addr_policy_t { * compressed form. */ typedef struct cached_dir_t { char *dir; /**< Contents of this object, NUL-terminated. */ - char *dir_z; /**< Compressed contents of this object. */ + char *dir_compressed; /**< Compressed contents of this object. */ size_t dir_len; /**< Length of dir (not counting its NUL). */ - size_t dir_z_len; /**< Length of dir_z. */ + size_t dir_compressed_len; /**< Length of dir_compressed. */ time_t published; /**< When was this object published. */ common_digests_t digests; /**< Digests of this object (networkstatus only) */ /** Sha3 digest (also ns only) */

1 0

[tor/master] consdiffmgr: Extract the code for compressing and storing
by nickm＠torproject.org 15 May '17

15 May '17

commit 6da31ec4848b23db4b923ff5141c24b82f466c8d Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri May 5 10:48:02 2017 -0400 consdiffmgr: Extract the code for compressing and storing We're going to use this for consensuses too. --- src/or/consdiffmgr.c | 142 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 43 deletions(-) diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index d478101..b108c80 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -1072,6 +1072,91 @@ typedef struct compressed_result_t { } compressed_result_t; /** + * Compress the bytestring input of length len using the + * <n>n_methods compression methods listed in the array methods. + * + * For each successful compression, set the fields in the results_out + * array in the position corresponding to the compression method. Use + * labels_in as a basis for the labels of the result. + * + * Return 0 if all compression succeeded; -1 if any failed. + */ +static int +compress_multiple(compressed_result_t *results_out, int n_methods, + const compress_method_t *methods, + const uint8_t *input, size_t len, + const config_line_t *labels_in) +{ + int rv = 0; + int i; + for (i = 0; i < n_methods; ++i) { + compress_method_t method = methods[i]; + const char *methodname = compression_method_get_name(method); + char *result; + size_t sz; + if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) { + results_out[i].body = (uint8_t*)result; + results_out[i].bodylen = sz; + results_out[i].labels = config_lines_dup(labels_in); + cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST, + results_out[i].body, + results_out[i].bodylen); + config_line_prepend(&results_out[i].labels, + LABEL_COMPRESSION_TYPE, + methodname); + } else { + rv = -1; + } + } + return rv; +} + +/** + * Given an array of n compressed_result_t in results, + * as produced by compress_multiple, store them all into the + * consdiffmgr, and store handles to them in the handles_out + * array. + * + * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none + * was stored. + */ +static cdm_diff_status_t +store_multiple(consensus_cache_entry_handle_t **handles_out, + int n, + const compress_method_t *methods, + const compressed_result_t *results, + const char *description) +{ + cdm_diff_status_t status = CDM_DIFF_ERROR; + consdiffmgr_ensure_space_for_files(n); + + int i; + for (i = 0; i < n; ++i) { + compress_method_t method = methods[i]; + uint8_t *body_out = results[i].body; + size_t bodylen_out = results[i].bodylen; + config_line_t *labels = results[i].labels; + const char *methodname = compression_method_get_name(method); + if (body_out && bodylen_out && labels) { + /* Success! Store the results */ + log_info(LD_DIRSERV, "Adding %s, compressed with %s", + description, methodname); + + consensus_cache_entry_t *ent = + consensus_cache_add(cdm_cache_get(), + labels, + body_out, + bodylen_out); + + status = CDM_DIFF_PRESENT; + handles_out[i] = consensus_cache_entry_handle_new(ent); + consensus_cache_entry_decref(ent); + } + } + return status; +} + +/** * An object passed to a worker thread that will try to produce a consensus * diff. */ @@ -1235,24 +1320,10 @@ consensus_diff_worker_threadfn(void *state_, void *work_) job->out[0].body, job->out[0].bodylen); - unsigned u; - for (u = 1; u < n_diff_compression_methods(); ++u) { - compress_method_t method = compress_diffs_with[u]; - const char *methodname = compression_method_get_name(method); - char *result; - size_t sz; - if (0 == tor_compress(&result, &sz, consensus_diff, difflen, method)) { - job->out[u].body = (uint8_t*)result; - job->out[u].bodylen = sz; - job->out[u].labels = config_lines_dup(common_labels); - cdm_labels_prepend_sha3(&job->out[u].labels, LABEL_SHA3_DIGEST, - job->out[u].body, - job->out[u].bodylen); - config_line_prepend(&job->out[u].labels, - LABEL_COMPRESSION_TYPE, - methodname); - } - } + compress_multiple(job->out+1, + n_diff_compression_methods()-1, + compress_diffs_with+1, + (const uint8_t*)consensus_diff, difflen, common_labels); config_free_lines(common_labels); return WQ_RPL_REPLY; @@ -1318,36 +1389,20 @@ consensus_diff_worker_replyfn(void *work_) cache = 0; } - int status = CDM_DIFF_ERROR; consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)]; memset(handles, 0, sizeof(handles)); - consdiffmgr_ensure_space_for_files(n_diff_compression_methods()); + char description[128]; + tor_snprintf(description, sizeof(description), + "consensus diff from %s to %s", + lv_from_digest, lv_to_digest); - unsigned u; - for (u = 0; u < n_diff_compression_methods(); ++u) { - compress_method_t method = compress_diffs_with[u]; - uint8_t *body_out = job->out[u].body; - size_t bodylen_out = job->out[u].bodylen; - config_line_t *labels = job->out[u].labels; - const char *methodname = compression_method_get_name(method); - if (body_out && bodylen_out && labels) { - /* Success! Store the results */ - log_info(LD_DIRSERV, "Adding consensus diff from %s to %s, " - "compressed with %s", - lv_from_digest, lv_to_digest, methodname); + int status = store_multiple(handles, + n_diff_compression_methods(), + compress_diffs_with, + job->out, + description); - consensus_cache_entry_t *ent = - consensus_cache_add(cdm_cache_get(), - labels, - body_out, - bodylen_out); - - status = CDM_DIFF_PRESENT; - handles[u] = consensus_cache_entry_handle_new(ent); - consensus_cache_entry_decref(ent); - } - } if (status != CDM_DIFF_PRESENT) { /* Failure! Nothing to do but complain */ log_warn(LD_DIRSERV, @@ -1357,6 +1412,7 @@ consensus_diff_worker_replyfn(void *work_) status = CDM_DIFF_ERROR; } + unsigned u; for (u = 0; u < ARRAY_LENGTH(handles); ++u) { compress_method_t method = compress_diffs_with[u]; if (cache) {

1 0

[tor/master] Add client_meth_pref array to define client compression preference.
by nickm＠torproject.org 15 May '17

15 May '17

commit 141f6e321190b7c8f928aba88ad0be8f553ee282 Author: Alexander Færøy <ahf(a)torproject.org> Date: Fri May 12 13:37:29 2017 +0200 Add client_meth_pref array to define client compression preference. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 7965ffd..d9d402a 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3321,6 +3321,16 @@ parse_accept_encoding_header(const char *h) return result; } +/** Array of compression methods to use (if supported) for requesting + * compressed data, ordered from best to worst. */ +static compress_method_t client_meth_pref[] = { + LZMA_METHOD, + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + /** Return a newly allocated string containing a comma separated list of * supported encodings. */ STATIC char * @@ -3329,11 +3339,10 @@ accept_encoding_header(void) smartlist_t *methods = smartlist_new(); char *header = NULL; compress_method_t method; + unsigned i; - // FIXME(ahf): Should we rename `srv_meth_pref_precompressed` and use this - // instead to define the order in the directory module rather than the order - // defined in the compression module? - for (method = UNKNOWN_METHOD; method > NO_METHOD; --method) { + for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) { + method = client_meth_pref[i]; if (tor_compress_supports_method(method)) smartlist_add(methods, (char *)compression_method_get_name(method)); }

1 0

[tor/master] Use compression_method_get_human_name() in connection_dir_client_reached_eof()
by nickm＠torproject.org 15 May '17

15 May '17

commit ef187bc280f1501bd7918b9c76a692027b320a67 Author: Alexander Færøy <ahf(a)torproject.org> Date: Fri May 12 12:54:16 2017 +0200 Use compression_method_get_human_name() in connection_dir_client_reached_eof() This patch refactors connection_dir_client_reached_eof() to use compression_method_get_human_name() to set description1 and description2 variables. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index bcd2eba..ed4f961 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2307,32 +2307,18 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (compression == UNKNOWN_METHOD || guessed != compression) { /* Tell the user if we don't believe what we're told about compression.*/ const char *description1, *description2; - if (compression == ZLIB_METHOD) - description1 = "as deflated"; - else if (compression == GZIP_METHOD) - description1 = "as gzipped"; - else if (compression == ZSTD_METHOD) - description1 = "as Zstandard compressed"; - else if (compression == LZMA_METHOD) - description1 = "as LZMA compressed"; - else if (compression == NO_METHOD) - description1 = "as uncompressed"; - else - description1 = "with an unknown Content-Encoding"; - if (guessed == ZLIB_METHOD) - description2 = "deflated"; - else if (guessed == GZIP_METHOD) - description2 = "gzipped"; - else if (guessed == ZSTD_METHOD) - description2 = "Zstandard compressed"; - else if (guessed == LZMA_METHOD) - description2 = "LZMA compressed"; - else if (!plausible) + + description1 = compression_method_get_human_name(compression); + + if (BUG(description1 == NULL)) + description1 = compression_method_get_human_name(UNKNOWN_METHOD); + + if (guessed == UNKNOWN_METHOD && !plausible) description2 = "confusing binary junk"; else - description2 = "uncompressed"; + description2 = compression_method_get_human_name(guessed); - log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, " + log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled as %s, " "but it seems to be %s.%s", conn->base_.address, conn->base_.port, description1, description2,

1 0

[tor/master] Use different preferences for compression methods when streaming.
by nickm＠torproject.org 15 May '17

15 May '17

commit 7a3efe25d9625a566d08eae7ab9ffd862c1d4a6d Author: Alexander Færøy <ahf(a)torproject.org> Date: Fri May 12 13:55:18 2017 +0200 Use different preferences for compression methods when streaming. See: https://bugs.torproject.org/21667 --- src/or/directory.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index d9d402a..4a528f2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3296,6 +3296,15 @@ static compress_method_t srv_meth_pref_precompressed[] = { NO_METHOD }; +/** Array of compression methods to use (if supported) for serving + * streamed data, ordered from best to worst. */ +static compress_method_t srv_meth_pref_streaming_compression[] = { + ZSTD_METHOD, + ZLIB_METHOD, + GZIP_METHOD, + NO_METHOD +}; + /** Parse the compression methods listed in an Accept-Encoding header h, * and convert them to a bitfield where compression method x is supported if * and only if 1 << x is set in the bitfield. */ @@ -3700,11 +3709,22 @@ find_best_diff(const smartlist_t *digests, int flav, * compression_methods. Return NO_METHOD if no mutually supported * compression method could be found. */ static compress_method_t -find_best_compression_method(unsigned compression_methods) +find_best_compression_method(unsigned compression_methods, int stream) { unsigned u; - for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) { - compress_method_t method = srv_meth_pref_precompressed[u]; + compress_method_t *methods; + size_t length; + + if (stream) { + methods = srv_meth_pref_streaming_compression; + length = ARRAY_LENGTH(srv_meth_pref_streaming_compression); + } else { + methods = srv_meth_pref_precompressed; + length = ARRAY_LENGTH(srv_meth_pref_precompressed); + } + + for (u = 0; u < length; ++u) { + compress_method_t method = methods[u]; if (compression_methods & (1u<<method)) return method; } @@ -3720,7 +3740,7 @@ handle_get_current_consensus(dir_connection_t *conn, { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 0); const time_t if_modified_since = args->if_modified_since; int clear_spool = 0; @@ -3878,7 +3898,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); { int current; ssize_t body_len = 0; @@ -3990,7 +4010,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); int clear_spool = 1; { conn->spool = smartlist_new(); @@ -4044,7 +4064,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); const or_options_t *options = get_options(); int clear_spool = 1; if (!strcmpstart(url,"/tor/server/") || @@ -4138,7 +4158,7 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; const compress_method_t compress_method = - find_best_compression_method(args->compression_supported); + find_best_compression_method(args->compression_supported, 1); const time_t if_modified_since = args->if_modified_since; { smartlist_t *certs = smartlist_new();

1 0