September 2016 - tor-commits - lists.torproject.org

[tor/master] Remove a needless level of indirection from crypto_cipher_t
by nickm＠torproject.org 19 Sep '16

19 Sep '16

commit 83129031b1a1a3c719810d30df0e3ec6fa320661 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Sep 16 10:18:02 2016 -0400 Remove a needless level of indirection from crypto_cipher_t Now that crypto_cipher_t only contains a pointer, it no longer has any reason for an independent existence. --- src/common/crypto.c | 23 ++++++----------------- src/common/crypto.h | 2 +- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index f6ee6b0..fb77340 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -120,13 +120,6 @@ struct crypto_pk_t RSA *key; /**< The key itself */ }; -/** Key and stream information for a stream cipher. */ -struct crypto_cipher_t -{ - aes_cnt_cipher_t *cipher; /**< The key in format usable for counter-mode AES - * encryption */ -}; - /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake * while we're waiting for the second.*/ struct crypto_dh_t { @@ -559,8 +552,7 @@ crypto_cipher_new_with_iv(const char *key, const char *iv) tor_assert(key); tor_assert(iv); - env = tor_malloc(sizeof(crypto_cipher_t)); - env->cipher = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128); + env = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128); return env; } @@ -583,10 +575,7 @@ crypto_cipher_free(crypto_cipher_t *env) if (!env) return; - tor_assert(env->cipher); - aes_cipher_free(env->cipher); - memwipe(env, 0, sizeof(crypto_cipher_t)); - tor_free(env); + aes_cipher_free(env); } /* public key crypto */ @@ -1586,14 +1575,14 @@ crypto_cipher_encrypt(crypto_cipher_t *env, char *to, const char *from, size_t fromlen) { tor_assert(env); - tor_assert(env->cipher); + tor_assert(env); tor_assert(from); tor_assert(fromlen); tor_assert(to); tor_assert(fromlen < SIZE_T_CEILING); memcpy(to, from, fromlen); - aes_crypt_inplace(env->cipher, to, fromlen); + aes_crypt_inplace(env, to, fromlen); return 0; } @@ -1611,7 +1600,7 @@ crypto_cipher_decrypt(crypto_cipher_t *env, char *to, tor_assert(fromlen < SIZE_T_CEILING); memcpy(to, from, fromlen); - aes_crypt_inplace(env->cipher, to, fromlen); + aes_crypt_inplace(env, to, fromlen); return 0; } @@ -1622,7 +1611,7 @@ void crypto_cipher_crypt_inplace(crypto_cipher_t *env, char *buf, size_t len) { tor_assert(len < SIZE_T_CEILING); - aes_crypt_inplace(env->cipher, buf, len); + aes_crypt_inplace(env, buf, len); } /** Encrypt fromlen bytes (at least 1) from from with the key in diff --git a/src/common/crypto.h b/src/common/crypto.h index 2d1155e..e60cf34 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -117,7 +117,7 @@ typedef struct { } common_digests_t; typedef struct crypto_pk_t crypto_pk_t; -typedef struct crypto_cipher_t crypto_cipher_t; +typedef struct aes_cnt_cipher crypto_cipher_t; typedef struct crypto_digest_t crypto_digest_t; typedef struct crypto_xof_t crypto_xof_t; typedef struct crypto_dh_t crypto_dh_t;

1 0

[tor/master] Add support for AES256 and AES192
by nickm＠torproject.org 19 Sep '16

19 Sep '16

commit 6cb9c2cf77cc8375f89cc8d625d0b60e292d8160 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Sep 16 11:21:33 2016 -0400 Add support for AES256 and AES192 (This will be used by prop224) --- src/common/aes.c | 2 +- src/common/crypto.c | 39 +++++++++++++++++++++++-------- src/common/crypto.h | 4 ++++ src/test/test_crypto.c | 62 ++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 82 insertions(+), 25 deletions(-) diff --git a/src/common/aes.c b/src/common/aes.c index 7131ce1..cb8fb68 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -48,7 +48,7 @@ ENABLE_GCC_WARNING(redundant-decls) /* We have five strategies for implementing AES counter mode. * - * Best with x86 and x86_64: Use EVP_aes_ctr128() and EVP_EncryptUpdate(). + * Best with x86 and x86_64: Use EVP_aes_*_ctr() and EVP_EncryptUpdate(). * This is possible with OpenSSL 1.0.1, where the counter-mode implementation * can use bit-sliced or vectorized AES or AESNI as appropriate. * diff --git a/src/common/crypto.c b/src/common/crypto.c index fb77340..72c1c45 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -542,29 +542,48 @@ crypto_pk_free(crypto_pk_t *env) } /** Allocate and return a new symmetric cipher using the provided key and iv. - * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes. Both - * must be provided. - */ + * The key is bits bits long; the IV is CIPHER_IV_LEN bytes. Both + * must be provided. Key length must be 128, 192, or 256 */ crypto_cipher_t * -crypto_cipher_new_with_iv(const char *key, const char *iv) +crypto_cipher_new_with_iv_and_bits(const uint8_t *key, + const uint8_t *iv, + int bits) { - crypto_cipher_t *env; tor_assert(key); tor_assert(iv); - env = aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, 128); + return aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, bits); +} - return env; +/** Allocate and return a new symmetric cipher using the provided key and iv. + * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes. Both + * must be provided. + */ +crypto_cipher_t * +crypto_cipher_new_with_iv(const char *key, const char *iv) +{ + return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)iv, + 128); } /** Return a new crypto_cipher_t with the provided key and an IV of all - * zero bytes. */ + * zero bytes and key length bits. Key length must be 128, 192, or + * 256. */ crypto_cipher_t * -crypto_cipher_new(const char *key) +crypto_cipher_new_with_bits(const char *key, int bits) { char zeroiv[CIPHER_IV_LEN]; memset(zeroiv, 0, sizeof(zeroiv)); - return crypto_cipher_new_with_iv(key, zeroiv); + return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)zeroiv, + bits); +} + +/** Return a new crypto_cipher_t with the provided key (of + * CIPHER_KEY_LEN bytes) and an IV of all zero bytes. */ +crypto_cipher_t * +crypto_cipher_new(const char *key) +{ + return crypto_cipher_new_with_bits(key, 128); } /** Free a symmetric cipher. diff --git a/src/common/crypto.h b/src/common/crypto.h index e60cf34..116e0a6 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -138,7 +138,11 @@ void crypto_pk_free(crypto_pk_t *env); void crypto_set_tls_dh_prime(void); crypto_cipher_t *crypto_cipher_new(const char *key); +crypto_cipher_t *crypto_cipher_new_with_bits(const char *key, int bits); crypto_cipher_t *crypto_cipher_new_with_iv(const char *key, const char *iv); +crypto_cipher_t *crypto_cipher_new_with_iv_and_bits(const uint8_t *key, + const uint8_t *iv, + int bits); void crypto_cipher_free(crypto_cipher_t *env); /* public key crypto */ diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 6586c06..64a46f7 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -363,9 +363,9 @@ test_crypto_rng_engine(void *arg) ; } -/** Run unit tests for our AES functionality */ +/** Run unit tests for our AES128 functionality */ static void -test_crypto_aes(void *arg) +test_crypto_aes128(void *arg) { char *data1 = NULL, *data2 = NULL, *data3 = NULL; crypto_cipher_t *env1 = NULL, *env2 = NULL; @@ -514,32 +514,61 @@ test_crypto_aes(void *arg) static void test_crypto_aes_ctr_testvec(void *arg) { - (void)arg; + const char *bitstr = arg; char *mem_op_hex_tmp=NULL; + crypto_cipher_t *c=NULL; /* from NIST SP800-38a, section F.5 */ - const char key16[] = "2b7e151628aed2a6abf7158809cf4f3c"; const char ctr16[] = "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; const char plaintext16[] = "6bc1bee22e409f96e93d7e117393172a" "ae2d8a571e03ac9c9eb76fac45af8e51" "30c81c46a35ce411e5fbc1191a0a52ef" "f69f2445df4f9b17ad2b417be66c3710"; - const char ciphertext16[] = - "874d6191b620e3261bef6864990db6ce" - "9806f66b7970fdff8617187bb9fffdff" - "5ae4df3edbd5d35e5b4f09020db03eab" - "1e031dda2fbe03d1792170a0f3009cee"; + const char *ciphertext16; + const char *key16; + int bits; + + if (!strcmp(bitstr, "128")) { + ciphertext16 = /* section F.5.1 */ + "874d6191b620e3261bef6864990db6ce" + "9806f66b7970fdff8617187bb9fffdff" + "5ae4df3edbd5d35e5b4f09020db03eab" + "1e031dda2fbe03d1792170a0f3009cee"; + key16 = "2b7e151628aed2a6abf7158809cf4f3c"; + bits = 128; + } else if (!strcmp(bitstr, "192")) { + ciphertext16 = /* section F.5.3 */ + "1abc932417521ca24f2b0459fe7e6e0b" + "090339ec0aa6faefd5ccc2c6f4ce8e94" + "1e36b26bd1ebc670d1bd1d665620abf7" + "4f78a7f6d29809585a97daec58c6b050"; + key16 = "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b"; + bits = 192; + } else if (!strcmp(bitstr, "256")) { + ciphertext16 = /* section F.5.5 */ + "601ec313775789a5b7a7f504bbf3d228" + "f443e3ca4d62b59aca84e990cacaf5c5" + "2b0930daa23de94ce87017ba2d84988d" + "dfc9c58db67aada613c2dd08457941a6"; + key16 = + "603deb1015ca71be2b73aef0857d7781" + "1f352c073b6108d72d9810a30914dff4"; + bits = 256; + } else { + tt_abort_msg("AES doesn't support this number of bits."); + } - char key[16]; + char key[32]; char iv[16]; char plaintext[16*4]; + memset(key, 0xf9, sizeof(key)); /* poison extra bytes */ base16_decode(key, sizeof(key), key16, strlen(key16)); base16_decode(iv, sizeof(iv), ctr16, strlen(ctr16)); base16_decode(plaintext, sizeof(plaintext), plaintext16, strlen(plaintext16)); - crypto_cipher_t *c = crypto_cipher_new_with_iv(key, iv); + c = crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)iv, bits); crypto_cipher_crypt_inplace(c, plaintext, sizeof(plaintext)); test_memeq_hex(plaintext, ciphertext16); @@ -2873,9 +2902,14 @@ struct testcase_t crypto_tests[] = { { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, &passthrough_setup, (void*)"broken" }, { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL }, - { "aes_AES", test_crypto_aes, TT_FORK, &passthrough_setup, (void*)"aes" }, - { "aes_EVP", test_crypto_aes, TT_FORK, &passthrough_setup, (void*)"evp" }, - { "aes_ctr_testvec", test_crypto_aes_ctr_testvec, 0, NULL, NULL }, + { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" }, + { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" }, + { "aes128_ctr_testvec", test_crypto_aes_ctr_testvec, 0, + &passthrough_setup, (void*)"128" }, + { "aes192_ctr_testvec", test_crypto_aes_ctr_testvec, 0, + &passthrough_setup, (void*)"192" }, + { "aes256_ctr_testvec", test_crypto_aes_ctr_testvec, 0, + &passthrough_setup, (void*)"256" }, CRYPTO_LEGACY(sha), CRYPTO_LEGACY(pk), { "pk_fingerprints", test_crypto_pk_fingerprints, TT_FORK, NULL, NULL },

1 0

[tor/master] Merge remote-tracking branch 'teor/bug20117'
by nickm＠torproject.org 19 Sep '16

19 Sep '16

commit 144bd86570ee2afc4efd8ecd7c59a65973452064 Merge: b08ddb6 42a74f7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Sep 19 14:21:12 2016 -0400 Merge remote-tracking branch 'teor/bug20117' changes/bug20117 | 6 ++++++ doc/tor.1.txt | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-)

1 0

[tor/master] Document the default PathsNeededToBuildCircuits value
by nickm＠torproject.org 19 Sep '16

19 Sep '16

commit 42a74f707c3031dab2aa21983040f58ba7b190e4 Author: teor <teor2345(a)gmail.com> Date: Fri Sep 9 11:20:20 2016 +1000 Document the default PathsNeededToBuildCircuits value ... when the directory authorities don't set min_paths_for_circs_pct. Fixes bug 20117; bugfix on 02c320916e02 in tor-0.2.4.10-alpha. Reported by Jesse V. --- changes/bug20117 | 6 ++++++ doc/tor.1.txt | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/changes/bug20117 b/changes/bug20117 new file mode 100644 index 0000000..6532260 --- /dev/null +++ b/changes/bug20117 @@ -0,0 +1,6 @@ + o Minor bugfixes (documentation): + - Document the default PathsNeededToBuildCircuits value that's + used by clients when the directory authorities don't set + min_paths_for_circs_pct. + Fixes bug 20117; bugfix on 02c320916e02 in tor-0.2.4.10-alpha. + Patch by teor, reported by Jesse V. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 1856592..f53ff94 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1558,8 +1558,9 @@ The following options are useful only for clients (that is, if that fraction of possible paths. Note that setting this option too low can make your Tor client less anonymous, and setting it too high can prevent your Tor client from bootstrapping. If this option is negative, - Tor will use a default value chosen by the directory - authorities. (Default: -1.) + Tor will use a default value chosen by the directory authorities. If the + directory authorities do not choose a value, Tor will default to 0.6. + (Default: -1.) [[ClientBootstrapConsensusAuthorityDownloadSchedule]] **ClientBootstrapConsensusAuthorityDownloadSchedule** __N__,__N__,__...__:: Schedule for when clients should download consensuses from authorities

1 0

[collector/master] Split up bridge descriptor tarballs into one per type.
by karsten＠torproject.org 19 Sep '16

19 Sep '16

commit f608c94c7f731241bf7ee8e627ca1da98c23d858 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Sep 18 16:09:52 2016 +0200 Split up bridge descriptor tarballs into one per type. --- CHANGELOG.md | 4 ++++ src/main/resources/create-tarballs.sh | 26 ++++++++++++++++++++------ src/main/webapp/index.html | 19 +++++++------------ 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e17abad..99b2918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ minutes to finish properly, rather than killing them immediately. - Replace TCP ports with hashes in @type bridge-network-status 1.1 and @type bridge-server-descriptor 1.2. + - Split up bridge descriptor tarballs into one tarball per month + and descriptor type: bridge-statuses-YYYY-MM.tar.xz, + bridge-server-descriptors-YYYY-MM.tar.xz and + bridge-extra-infos-YYYY-MM.tar.xz. * Minor changes - Remove quotes around base URL in index.json. diff --git a/src/main/resources/create-tarballs.sh b/src/main/resources/create-tarballs.sh index cc75219..4f2c6eb 100755 --- a/src/main/resources/create-tarballs.sh +++ b/src/main/resources/create-tarballs.sh @@ -56,8 +56,12 @@ TARBALLS=( server-descriptors-$YEARTWO-$MONTHTWO extra-infos-$YEARONE-$MONTHONE extra-infos-$YEARTWO-$MONTHTWO - bridge-descriptors-$YEARONE-$MONTHONE - bridge-descriptors-$YEARTWO-$MONTHTWO + bridge-statuses-$YEARONE-$MONTHONE + bridge-statuses-$YEARTWO-$MONTHTWO + bridge-server-descriptors-$YEARONE-$MONTHONE + bridge-server-descriptors-$YEARTWO-$MONTHTWO + bridge-extra-infos-$YEARONE-$MONTHONE + bridge-extra-infos-$YEARTWO-$MONTHTWO ) TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq)) @@ -77,8 +81,12 @@ DIRECTORIES=( $OUTDIR/relay-descriptors/server-descriptor/$YEARTWO/$MONTHTWO/ $OUTDIR/relay-descriptors/extra-info/$YEARONE/$MONTHONE/ $OUTDIR/relay-descriptors/extra-info/$YEARTWO/$MONTHTWO/ - $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/ - $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/ + $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/statuses/ + $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/statuses/ + $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/server-descriptors/ + $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/server-descriptors/ + $OUTDIR/bridge-descriptors/$YEARONE/$MONTHONE/extra-infos/ + $OUTDIR/bridge-descriptors/$YEARTWO/$MONTHTWO/extra-infos/ ) DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq)) @@ -109,8 +117,14 @@ cd $CURRENTPATH echo `date` "Finished tarball creation. Starting symlink-update ..." -mkdir -p $ARCHIVEDIR/bridge-descriptors/ -ln -f -s -t $ARCHIVEDIR/bridge-descriptors/ $TARBALLTARGETDIR/bridge-descriptors-20??-??.tar.xz +mkdir -p $ARCHIVEDIR/bridge-descriptors/statuses/ +ln -f -s -t $ARCHIVEDIR/bridge-descriptors/statuses/ $TARBALLTARGETDIR/bridge-statuses-20??-??.tar.xz + +mkdir -p $ARCHIVEDIR/bridge-descriptors/server-descriptors/ +ln -f -s -t $ARCHIVEDIR/bridge-descriptors/server-descriptors/ $TARBALLTARGETDIR/bridge-server-descriptors-20??-??.tar.xz + +mkdir -p $ARCHIVEDIR/bridge-descriptors/extra-infos/ +ln -f -s -t $ARCHIVEDIR/bridge-descriptors/extra-infos/ $TARBALLTARGETDIR/bridge-extra-infos-20??-??.tar.xz mkdir -p $ARCHIVEDIR/bridge-pool-assignments/ ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/ $TARBALLTARGETDIR/bridge-pool-assignments-20??-??.tar.xz diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index bbc70f2..7ed9145 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -168,21 +168,21 @@ <td>Bridge Network Statuses</td> <td><tt>@type bridge-network-status 1.1</tt></td> <td><a href="recent/bridge-descriptors/statuses/" class="btn btn-primary btn-xs">recent</a></td> - <td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td> + <td><a href="archive/bridge-descriptors/statuses/" class="btn btn-primary btn-xs">archive</a></td> <td><a href="#type-bridge-network-status" class="btn btn-default btn-xs">format</a></td> </tr> <tr> <td>Bridge Server Descriptors</td> <td><tt>@type bridge-server-descriptor 1.2</tt></td> <td><a href="recent/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs">recent</a></td> - <td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td> + <td><a href="archive/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs">archive</a></td> <td><a href="#type-bridge-server-descriptor" class="btn btn-default btn-xs">format</a></td> </tr> <tr> <td>Bridge Extra-info Descriptors</td> <td><tt>@type bridge-extra-info 1.3</tt></td> <td><a href="recent/bridge-descriptors/extra-infos/" class="btn btn-primary btn-xs">recent</a></td> - <td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td> + <td><a href="archive/bridge-descriptors/extra-infos/" class="btn btn-primary btn-xs">archive</a></td> <td><a href="#type-bridge-extra-info" class="btn btn-default btn-xs">format</a></td> </tr> <tr> @@ -500,8 +500,6 @@ descriptor including all signatures.</li> Sanitized bridge network statuses are similar to version 2 relay network statuses, but with only a <tt>published</tt> line in the header and without any lines in the footer. -The bridge descriptor archive tarballs contain all bridge -descriptors of a given month, not just network statuses. The format has changed over time to accomodate changes to the sanitizing process, with earlier versions being: @@ -523,10 +521,8 @@ ports.</li> Bridge server descriptors follow the same format as relay server descriptors, except for the sanitizing steps described above. -The bridge descriptor archive tarballs contain all bridge -descriptors of a given month, not just server descriptors. -These tarballs contain one descriptor per file, whereas recently published bridge descriptor -files +The bridge server descriptor archive tarballs contain one descriptor per +file, whereas recently published bridge server descriptor files contain all descriptors collected in an hour concatenated into a single file to reduce the number of files. The format has changed over time to accomodate changes to the sanitizing @@ -579,9 +575,8 @@ published by bridges using an Ed25519 master key.</li> </ul> -The bridge descriptor archive tarballs contain all bridge -descriptors of a given month, not just extra-info descriptors. -These tarballs contain one descriptor per file, whereas recently published bridge descriptor +The bridge extra-info descriptor archive tarballs contain one descriptor +per file, whereas recently published bridge extra-info descriptor files contain all descriptors collected in an hour concatenated into a single file to reduce the number of files.

1 0

[collector/master] Sanitize TCP ports in bridge descriptors.
by karsten＠torproject.org 19 Sep '16

19 Sep '16

commit ecb053899eb965c2778cf05479c26549d67f7956 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 10 13:28:42 2016 +0200 Sanitize TCP ports in bridge descriptors. Implements #19317. --- CHANGELOG.md | 2 + .../bridgedescs/SanitizedBridgesWriter.java | 81 +++++++++++++++++----- src/main/webapp/index.html | 46 ++++++++++-- 3 files changed, 107 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24206ee..e17abad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Add support for Bifroest's bridge descriptor tarballs. - Use a shutdown hook that gives currently running modules up to 10 minutes to finish properly, rather than killing them immediately. + - Replace TCP ports with hashes in @type bridge-network-status 1.1 + and @type bridge-server-descriptor 1.2. * Minor changes - Remove quotes around base URL in index.json. diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index b787f78..b61cd30 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -138,7 +138,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { while ((line = br.readLine()) != null) { String[] parts = line.split(","); if ((line.length() != ("yyyy-MM,".length() + 31 * 2) - && line.length() != ("yyyy-MM,".length() + 50 * 2)) + && line.length() != ("yyyy-MM,".length() + 50 * 2) + && line.length() != ("yyyy-MM,".length() + 83 * 2)) || parts.length != 2) { logger.warn("Invalid line in bridge-ip-secrets file " + "starting with '" + line.substring(0, 7) + "'! " @@ -218,8 +219,10 @@ public class SanitizedBridgesWriter extends CollecTorMain { scrubbedAddressPart = this.scrubIpv4Address(addressPart, fingerprintBytes, published); } + String scrubbedPort = this.scrubTcpPort(portPart, fingerprintBytes, + published); return (scrubbedAddressPart == null ? null : - scrubbedAddressPart + ":" + portPart); + scrubbedAddressPart + ":" + scrubbedPort); } private String scrubIpv4Address(String address, byte[] fingerprintBytes, @@ -334,14 +337,42 @@ public class SanitizedBridgesWriter extends CollecTorMain { return sb.toString(); } + private String scrubTcpPort(String portString, byte[] fingerprintBytes, + String published) throws IOException { + if (portString.equals("0")) { + return "0"; + } else if (this.replaceIpAddressesWithHashes) { + if (this.persistenceProblemWithSecrets) { + /* There's a persistence problem, so we shouldn't scrub more TCP + * ports in this execution. */ + return null; + } + byte[] hashInput = new byte[2 + 20 + 33]; + int portNumber = Integer.parseInt(portString); + hashInput[0] = (byte) (portNumber >> 8); + hashInput[1] = (byte) portNumber; + System.arraycopy(fingerprintBytes, 0, hashInput, 2, 20); + String month = published.substring(0, "yyyy-MM".length()); + byte[] secret = this.getSecretForMonth(month); + System.arraycopy(secret, 50, hashInput, 22, 33); + byte[] hashOutput = DigestUtils.sha256(hashInput); + int hashedPort = ((((hashOutput[0] & 0xFF) << 8) + | (hashOutput[1] & 0xFF)) >> 2) | 0xC000; + return String.valueOf(hashedPort); + } else { + return "1"; + } + } + private byte[] getSecretForMonth(String month) throws IOException { if (!this.secretsForHashingIpAddresses.containsKey(month) - || this.secretsForHashingIpAddresses.get(month).length == 31) { - byte[] secret = new byte[50]; + || this.secretsForHashingIpAddresses.get(month).length < 83) { + byte[] secret = new byte[83]; this.secureRandom.nextBytes(secret); if (this.secretsForHashingIpAddresses.containsKey(month)) { System.arraycopy(this.secretsForHashingIpAddresses.get(month), 0, - secret, 0, 31); + secret, 0, + this.secretsForHashingIpAddresses.get(month).length); } if (month.compareTo( this.bridgeSanitizingCutOffTimestamp) < 0) { @@ -362,8 +393,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { bw.close(); } catch (IOException e) { logger.warn("Could not store new secret " - + "to disk! Not calculating any IP address hashes in " - + "this execution!", e); + + "to disk! Not calculating any IP address or TCP port " + + "hashes in this execution!", e); this.persistenceProblemWithSecrets = true; throw new IOException(e); } @@ -471,11 +502,15 @@ public class SanitizedBridgesWriter extends CollecTorMain { fingerprintBytes, descPublicationTime); String nickname = parts[1]; + String scrubbedOrPort = this.scrubTcpPort(orPort, + fingerprintBytes, descPublicationTime); + String scrubbedDirPort = this.scrubTcpPort(dirPort, + fingerprintBytes, descPublicationTime); scrubbed.append("r " + nickname + " " + hashedBridgeIdentityBase64 + " " + hashedDescriptorIdentifier + " " + descPublicationTime - + " " + scrubbedAddress + " " + orPort + " " + dirPort - + "\n"); + + " " + scrubbedAddress + " " + scrubbedOrPort + " " + + scrubbedDirPort + "\n"); /* Sanitize any addresses in a lines using the fingerprint and * descriptor publication time from the previous r line. */ @@ -555,7 +590,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { outputFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( outputFile)); - bw.write("@type bridge-network-status 1.0\n"); + bw.write("@type bridge-network-status 1.1\n"); bw.write("published " + publicationTime + "\n"); bw.write(header.toString()); for (String scrubbed : scrubbedLines.values()) { @@ -595,6 +630,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { String hashedBridgeIdentity = null; String address = null; String routerLine = null; + String scrubbedRouterLine = null; String scrubbedAddress = null; String masterKeyEd25519 = null; List<String> orAddresses = null; @@ -611,7 +647,12 @@ public class SanitizedBridgesWriter extends CollecTorMain { * the bridge identity fingerprint for replacing the IP address in * the scrubbed version. */ } else if (line.startsWith("router ")) { - address = line.split(" ")[2]; + String[] parts = line.split(" "); + if (parts.length != 6) { + logger.warn("Invalid router line: '" + line + "'. Skipping."); + return; + } + address = parts[2]; routerLine = line; /* Store or-address parts in a list and sanitize them when we have @@ -671,6 +712,17 @@ public class SanitizedBridgesWriter extends CollecTorMain { } } } + String[] routerLineParts = routerLine.split(" "); + String nickname = routerLineParts[1]; + String scrubbedOrPort = this.scrubTcpPort(routerLineParts[3], + fingerprintBytes, published); + String scrubbedDirPort = this.scrubTcpPort(routerLineParts[4], + fingerprintBytes, published); + String scrubbedSocksPort = this.scrubTcpPort( + routerLineParts[5], fingerprintBytes, published); + scrubbedRouterLine = String.format("router %s %s %s %s %s%n", + nickname, scrubbedAddress, scrubbedOrPort, + scrubbedDirPort, scrubbedSocksPort); } catch (IOException e) { /* There's a persistence problem, so we shouldn't scrub more * IP addresses in this execution. */ @@ -692,10 +744,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { /* When we reach the signature, we're done. Write the sanitized * descriptor to disk below. */ } else if (line.startsWith("router-signature")) { - String[] routerLineParts = routerLine.split(" "); - scrubbedDesc = "router " + routerLineParts[1] + " " - + scrubbedAddress + " " + routerLineParts[3] + " " - + routerLineParts[4] + " " + routerLineParts[5] + "\n"; + scrubbedDesc = scrubbedRouterLine; if (scrubbedOrAddresses != null) { for (String scrubbedOrAddress : scrubbedOrAddresses) { scrubbedDesc = scrubbedDesc += "or-address " @@ -915,7 +964,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { outputFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( outputFile, appendToFile)); - bw.write("@type bridge-server-descriptor 1.1\n"); + bw.write("@type bridge-server-descriptor 1.2\n"); bw.write(scrubbedDesc); if (descriptorDigestSha256Base64 != null) { bw.write("router-digest-sha256 " + descriptorDigestSha256Base64 diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index ccdeea2..bbc70f2 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -166,14 +166,14 @@ </tr> <tr> <td>Bridge Network Statuses</td> - <td><tt>@type bridge-network-status 1.0</tt></td> + <td><tt>@type bridge-network-status 1.1</tt></td> <td><a href="recent/bridge-descriptors/statuses/" class="btn btn-primary btn-xs">recent</a></td> <td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td> <td><a href="#type-bridge-network-status" class="btn btn-default btn-xs">format</a></td> </tr> <tr> <td>Bridge Server Descriptors</td> - <td><tt>@type bridge-server-descriptor 1.1</tt></td> + <td><tt>@type bridge-server-descriptor 1.2</tt></td> <td><a href="recent/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs">recent</a></td> <td><a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a></td> <td><a href="#type-bridge-server-descriptor" class="btn btn-default btn-xs">format</a></td> @@ -447,6 +447,28 @@ with <tt>xx:xxxx</tt> being the hex-formatted 3 byte output of a similar hash function as described for IPv4 addresses. The only differences are that the input <tt>IP address</tt> is 16 bytes long and the <tt>secret</tt> is only 19 bytes long.</li></ul> +<li>Replace TCP port with TCP port hash: It may be less obvious +that TCP ports need to be sanitized, but an unusual TCP port used by a +high-value bridge might still stand out and provide yet another way to +locate and block the bridge. +Therefore, each non-zero TCP port is replaced with a number in the range +from 49152 to 65535 that is the result of +<tt>H(port | bridge identity | secret)[:2] / 2^2 + 2^15 + 2^14</tt> +written as decimal number. +The input <tt>port</tt> is the 2-byte long binary representation of the +TCP port. +The <tt>bridge identity</tt> is the 20-byte long binary representation of +the bridge's long-term identity fingerprint. +The <tt>secret</tt> is a 33-byte long secure random string that changes +once per month for all descriptors and statuses published in that month. +<tt>H()</tt> is SHA-256. +The <tt>[:2]</tt> operator means that we pick the 2 most significant bytes +of the result. +The subsequent integer division and additions make sure that sanitized +ports are in the range from 49152 to 65535 which is reserved for private +services. +All operations assume inputs to be in network byte order. +TCP ports that are 0 in the original descriptor are left unchanged.</li> <li>Replace contact information: If there is contact information in a descriptor, the contact line is changed to <tt>somebody</tt>.</li> @@ -468,7 +490,7 @@ descriptor including all signatures.</li> </ol> <h3 id="type-bridge-network-status" class="hover">Bridge Network Statuses -<tt>@type bridge-network-status 1.0</tt> +<tt>@type bridge-network-status 1.1</tt> <a href="recent/bridge-descriptors/statuses/" class="btn btn-primary btn-xs">recent</a> <a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a> <a href="#type-bridge-network-status" class="hover">#</a> @@ -480,10 +502,19 @@ statuses, but with only a <tt>published</tt> line in the header and without any lines in the footer. The bridge descriptor archive tarballs contain all bridge descriptors of a given month, not just network statuses. +The format has changed over time to accomodate changes to the sanitizing +process, with earlier versions being: +<ul> +<li><tt>@type bridge-network-status 1.0</tt> was the +first version.</li> +<li><tt>@type bridge-network-status 1.1</tt> introduced sanitized TCP +ports.</li> +</ul> + <h3 id="type-bridge-server-descriptor" class="hover">Bridge Server descriptors -<tt>@type bridge-server-descriptor 1.1</tt> +<tt>@type bridge-server-descriptor 1.2</tt> <a href="recent/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs">recent</a> <a href="archive/bridge-descriptors/" class="btn btn-primary btn-xs">archive</a> <a href="#type-bridge-server-descriptor" class="hover">#</a> @@ -511,9 +542,12 @@ number of sanitized bridge extra-info descriptors was raised. As a result, there may be sanitized bridge server descriptors with version <tt>@type bridge-server-descriptor 1.0</tt> with and without those lines.</li> -<li><tt>@type bridge-server-descriptor 1.1</tt> added +<li><tt>@type bridge-server-descriptor 1.1</tt> added <tt>master-key-ed25519</tt> lines and <tt>router-digest-sha256</tt> to -server descriptors published by bridges using an Ed25519 master key.</li> +server descriptors published by bridges using an Ed25519 master +key.</li> +<li><tt>@type bridge-server-descriptor 1.2</tt> introduced sanitized TCP +ports.</li> </ul> <h3 id="type-bridge-extra-info" class="hover">Bridge Extra-info Descriptors

1 0

[ooni-probe/master] Generate the filename for the report late on to avoid conflicts in measurement IDs
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 40b83a84265a24bd6777195b43a36edc8dc42640 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 22:05:15 2016 +0200 Generate the filename for the report late on to avoid conflicts in measurement IDs --- ooni/deck/deck.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index e4c35c4..3fc10ab 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -1,4 +1,6 @@ import os +import uuid +import errno from copy import deepcopy from string import Template @@ -90,7 +92,6 @@ class NGDeck(object): self._measurement_path = FilePath(config.measurements_directory) self._tasks = [] - self.task_ids = [] if deck_path is not None: self.open(deck_path) @@ -145,7 +146,6 @@ class NGDeck(object): self.bouncer.backend_type == "onion"): self.requires_tor = True self._tasks.append(deck_task) - self.task_ids.append(deck_task.id) if self.metadata.get('no_collector', False): self.no_collector = True @@ -217,7 +217,10 @@ class NGDeck(object): def _run_ooni_task(self, task, director): net_test_loader = task.ooni["net_test_loader"] - test_details = task.ooni["test_details"] + # XXX-REFACTOR we do this so late to avoid the collision between the + # same id and hence generating the same filename. + test_details = net_test_loader.getTestDetails() + task.id = generate_filename(test_details) measurement_id = None report_filename = task.output_path @@ -228,9 +231,9 @@ class NGDeck(object): try: measurement_dir.createDirectory() except OSError as ose: - # Ignore 'File Exists' - if ose.errno != 17: - raise + if ose.errno == errno.EEXIST: + raise Exception("Directory already exists, there is a " + "collision") report_filename = measurement_dir.child("measurements.njson.progress").path pid_file = measurement_dir.child("running.pid") @@ -388,10 +391,9 @@ class DeckTask(object): ) file_path = resolve_file_path(filename, self.cwd) input_file['test_options'][input_file['key']] = file_path - self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails() - self.id = generate_filename(self.ooni['test_details']) def setup(self): + self.id = str(uuid.uuid4()) return getattr(self, "_setup_"+self.type)() def _load(self, data):

1 0

[ooni-probe/master] Add simple unittest for api_status
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 417e64893fa21a9ca1b9e66efa812d784a128731 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 16:08:27 2016 +0200 Add simple unittest for api_status --- .coveragerc | 2 +- ooni/tests/test_wui_server.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 34ed07c..4f5cfba 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,4 @@ [run] source = ooni [report] -omit = ooni/nettests/*, ooni/api/*, ooni/kit/* +omit = ooni/nettests/*, ooni/api/*, ooni/kit/*, ooni/contrib/* diff --git a/ooni/tests/test_wui_server.py b/ooni/tests/test_wui_server.py new file mode 100644 index 0000000..ebaaacd --- /dev/null +++ b/ooni/tests/test_wui_server.py @@ -0,0 +1,32 @@ +from __future__ import print_function + +import json + +from .bases import ConfigTestCase +from ooni.ui.web.server import WebUIAPI + +from mock import patch, Mock + +class TestWebUIServer(ConfigTestCase): + def setUp(self): + super(TestWebUIServer, self).setUp() + + director = Mock() + scheduler = Mock() + + self.wui_api = WebUIAPI(self.config, director, scheduler) + + def tearDown(self): + super(TestWebUIServer, self).tearDown() + self.wui_api.status_poller.stop() + self.wui_api.director_event_poller.stop() + + def test_api_status(self): + request = Mock() + resp = self.wui_api.api_status(request) + j = json.loads(resp) + expected_keys = ["software_version", "software_name", "asn", + "country_code", "director_started", "initialized", + "quota_warning"] + for key in expected_keys: + self.assertTrue(key in j)

1 0

[ooni-probe/master] Move lepidopter update install process to the install script
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 1165aeee44567e604f24291a716ef2fb6202e38f Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 5 16:48:56 2016 +0200 Move lepidopter update install process to the install script --- setup.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index c465789..60d08a5 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,7 @@ def is_updater_installed(): return os.path.exists('/etc/lepidopter-update/version') -def install_updater(): +def install_lepidopter_update(): check_call(["data/updater.py", "install"]) @@ -199,17 +199,14 @@ class OoniInstall(InstallCommand): def run(self): self.pre_install() self.do_egg_install() - + if is_lepidopter() and not is_updater_installed(): + print("Lepidopter now requires that ooniprobe is installed via the " + "updater") + print("Let me install the auto-updater for you and we shall use that " + "for updates in the future.") + install_lepidopter_update() def setup_package(): - if is_lepidopter() and not is_updater_installed(): - print("Lepidopter now requires that ooniprobe is installed via the " - "updater") - print("Let me install the auto-updater for you and we shall use that " - "for updates in the future.") - install_updater() - return - setup_requires = [] install_requires = [] dependency_links = []

1 0

[ooni-probe/master] Bump version to rc.1
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 7fe1bc1be0f657b8c2492408288067f2d4fd1d62 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 16:19:02 2016 +0200 Bump version to rc.1 --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index d4b6f95..de2157c 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0-alpha.3" +__version__ = "2.0.0-rc.1" __all__ = [ 'agent',

1 0