April 2014 - tor-commits - lists.torproject.org

[bridgedb/develop] Use `email.message.Message` instead of `MimeWriter.MimeWriter`
by isis＠torproject.org 02 Apr '14

02 Apr '14

commit 5092364f7e3ce42b61b0336a95519639d0e59308 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Mar 30 12:42:06 2014 +0000 Use `email.message.Message` instead of `MimeWriter.MimeWriter` `MimeWriter.MimeWriter` was deprecated in Python 2.3, now is a great time for us to switch to the replacement. Fixes #11370 --- CHANGELOG | 4 ++++ lib/bridgedb/EmailServer.py | 39 ++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4f36267..7f160df 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,10 @@ Changes in version 0.1.7 - descriptor reparsing into another thread and significantly increased the availability of bridgedb, as a result. + * FIXES #11370 We were using an old (and deprecated) module when + we created our email responses. Now we use the newer + version. This should only affect internal functionality + and should not result in any noticable user-visible changes. And includes the following general changes: * BUMPS leekspin version to 0.1.3 diff --git a/lib/bridgedb/EmailServer.py b/lib/bridgedb/EmailServer.py index 8b7783f..90b22af 100644 --- a/lib/bridgedb/EmailServer.py +++ b/lib/bridgedb/EmailServer.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_EmailServer -*- # BridgeDB by Nick Mathewson. # Copyright (c) 2007-2013, The Tor Project, Inc. # See LICENSE for licensing information @@ -6,8 +7,10 @@ This module implements the email interface to the bridge database. """ -from StringIO import StringIO -import MimeWriter +from __future__ import unicode_literals + +from email import message +from io import StringIO import gettext import gpgme import logging @@ -430,17 +433,19 @@ def addSMTPServer(cfg, dist, sched): def composeEmail(fromAddr, clientAddr, subject, body, msgID=False, gpgContext=None): - f = StringIO() - w = MimeWriter.MimeWriter(f) - w.addheader("From", fromAddr) - w.addheader("To", clientAddr) - w.addheader("Message-ID", twisted.mail.smtp.messageid()) + msg = message.Message() + msg.add_header("From", fromAddr) + msg.add_header("To", clientAddr) + msg.add_header("Message-ID", twisted.mail.smtp.messageid()) if not subject.startswith("Re:"): subject = "Re: %s"%subject - w.addheader("Subject", subject) + msg.add_header("Subject", subject) if msgID: - w.addheader("In-Reply-To", msgID) - w.addheader("Date", twisted.mail.smtp.rfc822date()) - mailbody = w.startbody("text/plain") + msg.add_header("In-Reply-To", msgID) + msg.add_header("Date", twisted.mail.smtp.rfc822date()) + msg.set_default_type("text/plain") + headers = [': '.join(m) for m in msg.items()] + mail = StringIO("\r\n".join(headers)) + mail.writelines(unicode(msg.as_string())) # gpg-clearsign messages if gpgContext: @@ -450,20 +455,20 @@ def composeEmail(fromAddr, clientAddr, subject, body, msgID=False, if (len(sigs) != 1): logging.warn('Failed to sign message!') signature.seek(0) - [mailbody.write(l) for l in signature] + [mail.write(l) for l in signature] else: - mailbody.write(body) + mail.write(body) # Only log the email text (including all headers) if SAFE_LOGGING is # disabled: if not Util.safe_logging: - f.seek(0) - logging.debug("Email contents:\n%s" % f.read()) + mail.seek(0) + logging.debug("Email contents:\n%s" % mail.read()) else: logging.debug("Email text for %r created." % Util.logSafely(clientAddr)) - f.seek(0) + mail.seek(0) - return clientAddr, f + return clientAddr, mail def getGPGContext(cfg): """Import a key from a file and initialise a context for GnuPG operations.

1 0

[bridgedb/develop] Make sure leekspin created our descriptors
by isis＠torproject.org 02 Apr '14

02 Apr '14

commit 286a33737d44b530feec987c09bbf65b421913cd Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Thu Mar 27 00:36:46 2014 +0000 Make sure leekspin created our descriptors --- lib/bridgedb/test/test_bridgedb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/bridgedb/test/test_bridgedb.py b/lib/bridgedb/test/test_bridgedb.py index e73e2bc..3015094 100644 --- a/lib/bridgedb/test/test_bridgedb.py +++ b/lib/bridgedb/test/test_bridgedb.py @@ -87,6 +87,9 @@ class BridgeDBCliTest(unittest.TestCase): eidesc = pjoin(runDir, 'cached-extrainfo') eindesc = pjoin(runDir, 'cached-extrainfo.new') self.doCopyFile(eindesc, eidesc, 'duplicated cached-extrainfo(.new)') + self.assertTrue(os.path.isfile(eidesc)) + self.assertTrue(os.path.isfile(eindesc)) + print("Running `bridgedb' to test server startups...") # Sorry Windows users

1 0

[bridgedb/develop] Merge remote-tracking branch 'tpo-sysrqb/bug5232_adding_concurrent_processing_squashed_r1' into develop
by isis＠torproject.org 02 Apr '14

02 Apr '14

commit 13231f18ff2dc8a80d6d5561833b503f690f3872 Merge: 8745167 8dfcd5d Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Apr 1 21:54:44 2014 +0000 Merge remote-tracking branch 'tpo-sysrqb/bug5232_adding_concurrent_processing_squashed_r1' into develop .travis.yml | 2 +- CHANGELOG | 14 +++ lib/bridgedb/Bridges.py | 66 +++++------ lib/bridgedb/Bucket.py | 104 ++++++++-------- lib/bridgedb/Dist.py | 115 +++++++++--------- lib/bridgedb/EmailServer.py | 56 ++++++--- lib/bridgedb/Main.py | 210 +++++++++++++++++++++------------ lib/bridgedb/Stability.py | 204 ++++++++++++++++---------------- lib/bridgedb/Storage.py | 158 +++++++++++++++++++++++-- lib/bridgedb/Tests.py | 4 +- lib/bridgedb/test/test_EmailServer.py | 160 +++++++++++++++++++++++++ lib/bridgedb/test/test_Storage.py | 56 +++++++++ lib/bridgedb/test/test_bridgedb.py | 39 +++++- setup.py | 2 +- 14 files changed, 840 insertions(+), 350 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'public/bug11232'
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit 6bef082d0a6047fd3625f548d60ad257aaed2fab Merge: c0441cc 61090bb Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Apr 1 09:39:48 2014 -0400 Merge remote-tracking branch 'public/bug11232' changes/bug11232 | 4 ++++ src/common/address.c | 2 +- src/common/compat.h | 2 +- src/common/memarea.c | 43 +++++++++++++++++++++++++++---------------- src/ext/csiphash.c | 5 +++++ src/test/test_circuitlist.c | 6 +++--- 6 files changed, 41 insertions(+), 21 deletions(-)

1 0

[tor/master] changes file for bug11232
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit 61090bb64f03d5fd944535311191486d22b3134d Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 18 10:59:51 2014 -0400 changes file for bug11232 --- changes/bug11232 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changes/bug11232 b/changes/bug11232 new file mode 100644 index 0000000..c1a8780 --- /dev/null +++ b/changes/bug11232 @@ -0,0 +1,4 @@ + o Minor bugfixes: + - Use AddressSanitizer and Ubsan sanitizers (in clang-3.4) to fix some + miscellaneous errors in our tests and codebase. Fix for bug 11232. + Bugfixes on versions back as far as 0.2.1.11-alpha.

1 0

[tor/master] Fix a ubsan warning in our ctypes replacements
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit 2aea6ca3260cee82a60168c047f4d0cc71f7c652 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 18 10:47:26 2014 -0400 Fix a ubsan warning in our ctypes replacements ubsan doesn't like 1<<31, since that's an undefined integer overflow. Instead, we should do 1u<<31. --- src/common/compat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/compat.h b/src/common/compat.h index 32effa5..30a3303 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -321,7 +321,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle) extern const uint32_t TOR_##name##_TABLE[]; \ static INLINE int TOR_##name(char c) { \ uint8_t u = c; \ - return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1 << (u & 31))); \ + return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1u << (u & 31))); \ } DECLARE_CTYPE_FN(ISALPHA) DECLARE_CTYPE_FN(ISALNUM)

1 0

[tor/master] memarea.c: use flexible array member for mem
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit 204f7255f4c06fe12d99e785b91ea2fb18047018 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 18 10:02:27 2014 -0400 memarea.c: use flexible array member for mem This make clang's memory sanitizer happier that we aren't reading off the end of a char[1]. We hadn't replaced the char[1] with a char[FLEXIBLE_ARRAY_MEMBER] before because we were doing a union trick to force alignment. Now we use __attribute__(aligned) where available, and we do the union trick elsewhere. Most of this patch is just replacing accesses to (x)->u.mem with (x)->U_MEM, where U_MEM is defined as "u.mem" or "mem" depending on our implementation. --- src/common/memarea.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/common/memarea.c b/src/common/memarea.c index 0ae0ccc..e2d07fc 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -29,6 +29,13 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) +#define USE_ALIGNED_ATTRIBUTE +#define U_MEM mem +#else +#define U_MEM u.mem +#endif + #ifdef USE_SENTINELS /** Magic value that we stick at the end of a memarea so we can make sure * there are no run-off-the-end bugs. */ @@ -39,12 +46,12 @@ * end, set those bytes. */ #define SET_SENTINEL(chunk) \ STMT_BEGIN \ - set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + set_uint32( &(chunk)->U_MEM[chunk->mem_size], SENTINEL_VAL ); \ STMT_END /** Assert that the sentinel on a memarea is set correctly. */ #define CHECK_SENTINEL(chunk) \ STMT_BEGIN \ - uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END #else @@ -71,19 +78,23 @@ realign_pointer(void *ptr) typedef struct memarea_chunk_t { /** Next chunk in this area. Only kept around so we can free it. */ struct memarea_chunk_t *next_chunk; - size_t mem_size; /**< How much RAM is available in u.mem, total? */ - char *next_mem; /**< Next position in u.mem to allocate data at. If it's + size_t mem_size; /**< How much RAM is available in mem, total? */ + char *next_mem; /**< Next position in mem to allocate data at. If it's * greater than or equal to mem+mem_size, this chunk is * full. */ +#ifdef USE_ALIGNED_ATTRIBUTE + char mem[FLEXIBLE_ARRAY_MEMBER] __attribute__((aligned(MEMAREA_ALIGN))); +#else union { char mem[1]; /**< Memory space in this chunk. */ void *void_for_alignment_; /**< Dummy; used to make sure mem is aligned. */ } u; +#endif } memarea_chunk_t; /** How many bytes are needed for overhead before we get to the memory part * of a chunk? */ -#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, u) +#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM) /** What's the smallest that we'll allocate a chunk? */ #define CHUNK_SIZE 4096 @@ -121,7 +132,7 @@ alloc_chunk(size_t sz, int freelist_ok) res = tor_malloc(chunk_size); res->next_chunk = NULL; res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; - res->next_mem = res->u.mem; + res->next_mem = res->U_MEM; tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); @@ -140,7 +151,7 @@ chunk_free_unchecked(memarea_chunk_t *chunk) ++freelist_len; chunk->next_chunk = freelist; freelist = chunk; - chunk->next_mem = chunk->u.mem; + chunk->next_mem = chunk->U_MEM; } else { tor_free(chunk); } @@ -183,7 +194,7 @@ memarea_clear(memarea_t *area) } area->first->next_chunk = NULL; } - area->first->next_mem = area->first->u.mem; + area->first->next_mem = area->first->U_MEM; } /** Remove all unused memarea chunks from the internal freelist. */ @@ -207,7 +218,7 @@ memarea_owns_ptr(const memarea_t *area, const void *p) memarea_chunk_t *chunk; const char *ptr = p; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { - if (ptr >= chunk->u.mem && ptr < chunk->next_mem) + if (ptr >= chunk->U_MEM && ptr < chunk->next_mem) return 1; } return 0; @@ -226,7 +237,7 @@ memarea_alloc(memarea_t *area, size_t sz) tor_assert(sz < SIZE_T_CEILING); if (sz == 0) sz = 1; - if (chunk->next_mem+sz > chunk->u.mem+chunk->mem_size) { + if (chunk->next_mem+sz > chunk->U_MEM+chunk->mem_size) { if (sz+CHUNK_HEADER_SIZE >= CHUNK_SIZE) { /* This allocation is too big. Stick it in a special chunk, and put * that chunk second in the list. */ @@ -244,8 +255,8 @@ memarea_alloc(memarea_t *area, size_t sz) result = chunk->next_mem; chunk->next_mem = chunk->next_mem + sz; /* Reinstate these if bug 930 ever comes back - tor_assert(chunk->next_mem >= chunk->u.mem); - tor_assert(chunk->next_mem <= chunk->u.mem+chunk->mem_size); + tor_assert(chunk->next_mem >= chunk->U_MEM); + tor_assert(chunk->next_mem <= chunk->U_MEM+chunk->mem_size); */ chunk->next_mem = realign_pointer(chunk->next_mem); return result; @@ -304,8 +315,8 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) for (chunk = area->first; chunk; chunk = chunk->next_chunk) { CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; - tor_assert(chunk->next_mem >= chunk->u.mem); - u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); + tor_assert(chunk->next_mem >= chunk->U_MEM); + u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->U_MEM); } *allocated_out = a; *used_out = u; @@ -320,9 +331,9 @@ memarea_assert_ok(memarea_t *area) for (chunk = area->first; chunk; chunk = chunk->next_chunk) { CHECK_SENTINEL(chunk); - tor_assert(chunk->next_mem >= chunk->u.mem); + tor_assert(chunk->next_mem >= chunk->U_MEM); tor_assert(chunk->next_mem <= - (char*) realign_pointer(chunk->u.mem+chunk->mem_size)); + (char*) realign_pointer(chunk->U_MEM+chunk->mem_size)); } }

1 0

[tor/master] Fix a ubsan warning in addr_mask_get_bits
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit dfdeb6418d3d99e6e9099a3ad8ca83c206b52bdc Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 18 10:49:39 2014 -0400 Fix a ubsan warning in addr_mask_get_bits ubsan doesn't like us to do (1u<<32) when 32 is wider than unsigned. Fortunately, we already special-case addr_mask_get_bits(0), so we can just change the loop bounds. --- src/common/address.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/address.c b/src/common/address.c index 69049fa..bb1a244 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1585,7 +1585,7 @@ addr_mask_get_bits(uint32_t mask) return 0; if (mask == 0xFFFFFFFFu) return 32; - for (i=0; i<=32; ++i) { + for (i=1; i<=32; ++i) { if (mask == (uint32_t) ~((1u<<(32-i))-1)) { return i; }

1 0

[tor/master] Fix a use-after-free in test_circuitlist.c
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit 98b1aad2018b572e19d2404ea7369afa2d0c4b05 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 18 09:46:39 2014 -0400 Fix a use-after-free in test_circuitlist.c Found by clang-3.4 analyzers. --- src/test/test_circuitlist.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c index 720b407..54aa51d 100644 --- a/src/test/test_circuitlist.c +++ b/src/test/test_circuitlist.c @@ -150,13 +150,13 @@ test_clist_maps(void *arg) tt_assert(! circuit_id_in_use_on_channel(100, ch1)); done: - tor_free(ch1); - tor_free(ch2); - tor_free(ch3); if (or_c1) circuit_free(TO_CIRCUIT(or_c1)); if (or_c2) circuit_free(TO_CIRCUIT(or_c2)); + tor_free(ch1); + tor_free(ch2); + tor_free(ch3); UNMOCK(circuitmux_attach_circuit); UNMOCK(circuitmux_detach_circuit); }

1 0

[tor/master] csiphash: don't attempt unaligned access
by nickm＠torproject.org 01 Apr '14

01 Apr '14

commit aaa33f144c4ad0db0c067df2f6703790be400acb Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 18 10:43:46 2014 -0400 csiphash: don't attempt unaligned access In digestmap_set/get benchmarks, doing unaligned access on x86 doesn't save more than a percent or so in the fast case. In the slow case (where we cross a cache line), it could be pretty expensive. It also makes ubsan unhappy. --- src/ext/csiphash.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index 4f58c4a..2a96f28 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -81,11 +81,16 @@ HALF_ROUND(v0,v1,v2,v3,13,16); \ HALF_ROUND(v2,v1,v0,v3,17,21); +#if 0 +/* This does not seem to save very much runtime in the fast case, and it's + * potentially a big loss in the slow case where we're misaligned and we cross + * a cache line. */ #if (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || \ defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__)) # define UNALIGNED_OK 1 #endif +#endif uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *key) { uint64_t k0 = key->k0;

1 0