June 2013 - tor-commits - lists.torproject.org

[tor/maint-0.2.3] Merge branch 'circuit_queue_cap-0.2.3-squashed' into maint-0.2.3
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit d780cd5fd428176ebf4a72610ec4066c18346c09 Merge: 4835fae 9e8c104 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jun 14 01:44:46 2013 -0400 Merge branch 'circuit_queue_cap-0.2.3-squashed' into maint-0.2.3 changes/bug9063 | 3 +++ src/or/or.h | 9 +++++++++ src/or/relay.c | 23 ++++++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-)

1 0

[tor/maint-0.2.3] Don't queue more cells as a middle relay than the spec allows to be in flight
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit 4cce58d3c22c65350fffb3c4bc4dccd73e0193d7 Author: Andrea Shepard <andrea(a)torproject.org> Date: Thu Jun 13 20:41:00 2013 -0700 Don't queue more cells as a middle relay than the spec allows to be in flight --- changes/bug9063 | 3 +++ src/or/or.h | 8 ++++++++ src/or/relay.c | 23 ++++++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/changes/bug9063 b/changes/bug9063 new file mode 100644 index 0000000..af3b1a8 --- /dev/null +++ b/changes/bug9063 @@ -0,0 +1,3 @@ + o Normal bugfixes: + - Close any circuit that has more cells queued than the spec permits. + Fixes bug #9063; bugfix on 0.2.3.25. diff --git a/src/or/or.h b/src/or/or.h index 6728b86..2565777 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -539,6 +539,8 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ESTABLISHED_REND(p) \ ((p) == CIRCUIT_PURPOSE_C_REND_JOINED || \ (p) == CIRCUIT_PURPOSE_S_REND_JOINED) +/** True iff the circuit_t c is actually an or_circuit_t */ +#define CIRCUIT_IS_ORCIRC(c) (((circuit_t *)(c))->magic == OR_CIRCUIT_MAGIC) /** How many circuits do we want simultaneously in-progress to handle * a given stream? */ @@ -814,6 +816,12 @@ typedef enum { /** Amount to increment a stream window when we get a stream SENDME. */ #define STREAMWINDOW_INCREMENT 50 +/** Maximum number of queued cells on a circuit for which we are the + * midpoint before we give up and kill it. This must be >= circwindow + * to avoid killing innocent circuits. + */ +#define ORCIRC_MAX_MIDDLE_CELLS (11*(CIRCWINDOW_START_MAX)/10) + /* Cell commands. These values are defined in tor-spec.txt. */ #define CELL_PADDING 0 #define CELL_CREATE 1 diff --git a/src/or/relay.c b/src/or/relay.c index a17c333..087459c 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -2532,8 +2532,10 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, cell_t *cell, cell_direction_t direction, streamid_t fromstream) { + or_circuit_t *orcirc = NULL; cell_queue_t *queue; int streams_blocked; + if (circ->marked_for_close) return; @@ -2541,11 +2543,30 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; } else { - or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + orcirc = TO_OR_CIRCUIT(circ); queue = &orcirc->p_conn_cells; streams_blocked = circ->streams_blocked_on_p_conn; } + /* Are we a middle circuit about to exceed ORCIRC_MAX_MIDDLE_CELLS? */ + if ((circ->n_conn != NULL) && CIRCUIT_IS_ORCIRC(circ)) { + orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_conn) { + if (queue->n + 1 >= ORCIRC_MAX_MIDDLE_CELLS) { + /* Queueing this cell would put queue over the cap */ + log_warn(LD_CIRC, + "Got a cell exceeding the cap of %u in the %s direction " + "on middle circ ID %u; killing the circuit.", + ORCIRC_MAX_MIDDLE_CELLS, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", + (direction == CELL_DIRECTION_OUT) ? + circ->n_circ_id : orcirc->p_circ_id); + circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); + return; + } + } + } + cell_queue_append_packed_copy(queue, cell); /* If we have too many cells on the circuit, we should stop reading from

1 0

[tor/master] Define SEEK_SET for platforms that lack it.
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit 2338681efbdc8070c0cdfd0b9226fcdeb37f0538 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 21:56:35 2013 -0400 Define SEEK_SET for platforms that lack it. --- src/common/compat.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/compat.c b/src/common/compat.c index 24b44fb..69eb064 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -870,6 +870,9 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) /** @{ */ /** Some old versions of Unix didn't define constants for these values, * and instead expect you to say 0, 1, or 2. */ +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif #ifndef SEEK_CUR #define SEEK_CUR 1 #endif

1 0

[tor/master] If we write the annotation but not the microdescriptor, rewind.
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit caa0d15c4902a8d376a2a79bfbfdb21bf047366a Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 12:29:01 2013 -0400 If we write the annotation but not the microdescriptor, rewind. This fixes bug 9047 (and some parts of 9031, 8922, 8883 that weren't fixed in 8822). Bugfix on 0.2.2.6-alpha. --- changes/bug9047 | 6 ++++++ src/common/compat.c | 12 ++++++++++++ src/common/compat.h | 1 + src/or/microdesc.c | 19 +++++++++++++------ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/changes/bug9047 b/changes/bug9047 new file mode 100644 index 0000000..497f0d3 --- /dev/null +++ b/changes/bug9047 @@ -0,0 +1,6 @@ + o Minor bugfixes: + - If for some reason we fail to write a microdescriptor while + rebuilding the cache, do not let the annotations from that + microdescriptor linger in the cache file, and do not let the + microdescriptor stay recorded as present in its old location. + Fixes bug 9047; bugfix on 0.2.2.6-alpha. diff --git a/src/common/compat.c b/src/common/compat.c index c97a454..24b44fb 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -900,6 +900,18 @@ tor_fd_seekend(int fd) #endif } +/** Move fd to position pos in the file. Return -1 on error, 0 + * on success. */ +int +tor_fd_setpos(int fd, off_t pos) +{ +#ifdef _WIN32 + return _lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0; +#else + return lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0; +#endif +} + #undef DEBUG_SOCKET_COUNTING #ifdef DEBUG_SOCKET_COUNTING /** A bitarray of all fds that should be passed to tor_socket_close(). Only diff --git a/src/common/compat.h b/src/common/compat.h index f0a34aa..8ab7190 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -411,6 +411,7 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking, void tor_lockfile_unlock(tor_lockfile_t *lockfile); off_t tor_fd_getpos(int fd); +int tor_fd_setpos(int fd, off_t pos); int tor_fd_seekend(int fd); #ifdef _WIN32 diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 05e3b41..b93bd83 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -74,7 +74,7 @@ static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) { ssize_t r = 0; - size_t written; + ssize_t written; if (md->body == NULL) { *annotation_len_out = 0; return 0; @@ -99,10 +99,10 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) md->off = tor_fd_getpos(fd); written = write_all(fd, md->body, md->bodylen, 0); - if (written != md->bodylen) { + if (written != (ssize_t)md->bodylen) { log_warn(LD_DIR, - "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", - (unsigned long)written, (unsigned long)md->bodylen, + "Couldn't dump microdescriptor (wrote %ld out of %lu): %s", + (long)written, (unsigned long)md->bodylen, strerror(errno)); return -1; } @@ -456,8 +456,15 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) size = dump_microdescriptor(fd, md, &annotation_len); if (size < 0) { - /* XXX handle errors from dump_microdescriptor() */ - /* log? return -1? die? coredump the universe? */ + if (md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + md->saved_location = SAVED_NOWHERE; + md->off = 0; + md->bodylen = 0; + md->no_save = 1; + + /* rewind, in case it was a partial write. */ + tor_fd_setpos(fd, off); continue; } tor_assert(((size_t)size) == annotation_len + md->bodylen);

1 0

[tor/master] Merge branch 'bug9047' into maint-0.2.4
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit 72f775e18416300bd230f5784dc779db6138537b Merge: 25dddf7 2338681 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 21:58:36 2013 -0400 Merge branch 'bug9047' into maint-0.2.4 changes/bug9047 | 6 ++++++ src/common/compat.c | 15 +++++++++++++++ src/common/compat.h | 1 + src/or/microdesc.c | 19 +++++++++++++------ 4 files changed, 35 insertions(+), 6 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.4'
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit 483385d2bdd391db2ecbb775935554558dba7b56 Merge: 73ca1cf 72f775e Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 21:59:27 2013 -0400 Merge remote-tracking branch 'origin/maint-0.2.4' changes/bug9047 | 6 ++++++ src/common/compat.c | 15 +++++++++++++++ src/common/compat.h | 1 + src/or/microdesc.c | 19 +++++++++++++------ 4 files changed, 35 insertions(+), 6 deletions(-)

1 0

[tor/maint-0.2.4] Merge branch 'bug9047' into maint-0.2.4
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit 72f775e18416300bd230f5784dc779db6138537b Merge: 25dddf7 2338681 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 21:58:36 2013 -0400 Merge branch 'bug9047' into maint-0.2.4 changes/bug9047 | 6 ++++++ src/common/compat.c | 15 +++++++++++++++ src/common/compat.h | 1 + src/or/microdesc.c | 19 +++++++++++++------ 4 files changed, 35 insertions(+), 6 deletions(-)

1 0

[tor/maint-0.2.4] Define SEEK_SET for platforms that lack it.
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit 2338681efbdc8070c0cdfd0b9226fcdeb37f0538 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 21:56:35 2013 -0400 Define SEEK_SET for platforms that lack it. --- src/common/compat.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/compat.c b/src/common/compat.c index 24b44fb..69eb064 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -870,6 +870,9 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) /** @{ */ /** Some old versions of Unix didn't define constants for these values, * and instead expect you to say 0, 1, or 2. */ +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif #ifndef SEEK_CUR #define SEEK_CUR 1 #endif

1 0

[tor/maint-0.2.4] If we write the annotation but not the microdescriptor, rewind.
by nickm＠torproject.org 14 Jun '13

14 Jun '13

commit caa0d15c4902a8d376a2a79bfbfdb21bf047366a Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jun 13 12:29:01 2013 -0400 If we write the annotation but not the microdescriptor, rewind. This fixes bug 9047 (and some parts of 9031, 8922, 8883 that weren't fixed in 8822). Bugfix on 0.2.2.6-alpha. --- changes/bug9047 | 6 ++++++ src/common/compat.c | 12 ++++++++++++ src/common/compat.h | 1 + src/or/microdesc.c | 19 +++++++++++++------ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/changes/bug9047 b/changes/bug9047 new file mode 100644 index 0000000..497f0d3 --- /dev/null +++ b/changes/bug9047 @@ -0,0 +1,6 @@ + o Minor bugfixes: + - If for some reason we fail to write a microdescriptor while + rebuilding the cache, do not let the annotations from that + microdescriptor linger in the cache file, and do not let the + microdescriptor stay recorded as present in its old location. + Fixes bug 9047; bugfix on 0.2.2.6-alpha. diff --git a/src/common/compat.c b/src/common/compat.c index c97a454..24b44fb 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -900,6 +900,18 @@ tor_fd_seekend(int fd) #endif } +/** Move fd to position pos in the file. Return -1 on error, 0 + * on success. */ +int +tor_fd_setpos(int fd, off_t pos) +{ +#ifdef _WIN32 + return _lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0; +#else + return lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0; +#endif +} + #undef DEBUG_SOCKET_COUNTING #ifdef DEBUG_SOCKET_COUNTING /** A bitarray of all fds that should be passed to tor_socket_close(). Only diff --git a/src/common/compat.h b/src/common/compat.h index f0a34aa..8ab7190 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -411,6 +411,7 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking, void tor_lockfile_unlock(tor_lockfile_t *lockfile); off_t tor_fd_getpos(int fd); +int tor_fd_setpos(int fd, off_t pos); int tor_fd_seekend(int fd); #ifdef _WIN32 diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 05e3b41..b93bd83 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -74,7 +74,7 @@ static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) { ssize_t r = 0; - size_t written; + ssize_t written; if (md->body == NULL) { *annotation_len_out = 0; return 0; @@ -99,10 +99,10 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) md->off = tor_fd_getpos(fd); written = write_all(fd, md->body, md->bodylen, 0); - if (written != md->bodylen) { + if (written != (ssize_t)md->bodylen) { log_warn(LD_DIR, - "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", - (unsigned long)written, (unsigned long)md->bodylen, + "Couldn't dump microdescriptor (wrote %ld out of %lu): %s", + (long)written, (unsigned long)md->bodylen, strerror(errno)); return -1; } @@ -456,8 +456,15 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) size = dump_microdescriptor(fd, md, &annotation_len); if (size < 0) { - /* XXX handle errors from dump_microdescriptor() */ - /* log? return -1? die? coredump the universe? */ + if (md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + md->saved_location = SAVED_NOWHERE; + md->off = 0; + md->bodylen = 0; + md->no_save = 1; + + /* rewind, in case it was a partial write. */ + tor_fd_setpos(fd, off); continue; } tor_assert(((size_t)size) == annotation_len + md->bodylen);

1 0

[stem/master] Adding the mock module as a test dependency
by atagar＠torproject.org 13 Jun '13

13 Jun '13

commit 606ccfbb5001d5a13bce8b7a51cea283c074f3c1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jun 9 12:14:56 2013 -0700 Adding the mock module as a test dependency Mocking is hard. We've gotten a lot of mileage by writing our own mocking helpers, but they're clunky and complicated. It's about time for us to use a more standard, mature option. The pypi mock module is not only well documented, but looks to be the direction the python world is going. As of python 3.3 it's now part of the standard library. As the first step of swapping us over I'm adding it as a dependency for run_tests.py. This warns the user if it's unavailable... atagar@morrigan:~/Desktop/stem$ ./run_tests.py --unit To run stem's tests you'll need mock... https://pypi.python.org/pypi/mock/ You can get it by running 'sudo pip install mock'. --- run_tests.py | 17 +++++++++++++++++ stem/prereq.py | 20 ++++++++++++++++++++ test/settings.cfg | 1 + test/util.py | 8 ++++++++ 4 files changed, 46 insertions(+) diff --git a/run_tests.py b/run_tests.py index 6f77b0f..7bcbe5e 100755 --- a/run_tests.py +++ b/run_tests.py @@ -75,6 +75,12 @@ LOG_TYPE_ERROR = """\ TRACE, DEBUG, INFO, NOTICE, WARN, ERROR """ +MOCK_UNAVAILABLE_MSG = """\ +To run stem's tests you'll need mock... + +https://pypi.python.org/pypi/mock/ +""" + def main(): start_time = time.time() @@ -104,11 +110,22 @@ def main(): println("Nothing to run (for usage provide --help)\n") sys.exit() + if not stem.prereq.is_mock_available(): + println(MOCK_UNAVAILABLE_MSG) + + if stem.util.system.is_available('pip'): + println("You can get it by running 'sudo pip install mock'.") + elif stem.util.system.is_available('apt-get'): + println("You can get it by running 'sudo apt-get install python-mock'.") + + sys.exit(1) + test.util.run_tasks( "INITIALISING", Task("checking stem version", test.util.check_stem_version), Task("checking python version", test.util.check_python_version), Task("checking pycrypto version", test.util.check_pycrypto_version), + Task("checking mock version", test.util.check_mock_version), Task("checking pyflakes version", test.util.check_pyflakes_version), Task("checking pep8 version", test.util.check_pep8_version), Task("checking for orphaned .pyc files", test.util.clean_orphaned_pyc, (SRC_PATHS,)), diff --git a/stem/prereq.py b/stem/prereq.py index 2104cd5..148b6f8 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -22,6 +22,7 @@ Checks for stem dependencies. We require python 2.6 or greater (including the import sys IS_CRYPTO_AVAILABLE = None +IS_MOCK_AVAILABLE = None def check_requirements(): @@ -85,3 +86,22 @@ def is_crypto_available(): log.log_once("stem.prereq.is_crypto_available", log.INFO, msg) return IS_CRYPTO_AVAILABLE + + +def is_mock_available(): + """ + Checks if the mock module is available. + + :returns: **True** if the mock module is available and **False** otherwise + """ + + global IS_MOCK_AVAILABLE + + if IS_MOCK_AVAILABLE is None: + try: + import mock + IS_MOCK_AVAILABLE = True + except ImportError: + IS_MOCK_AVAILABLE = False + + return IS_MOCK_AVAILABLE diff --git a/test/settings.cfg b/test/settings.cfg index db318e4..96fc407 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -131,6 +131,7 @@ pep8.ignore E127 pyflakes.ignore stem/prereq.py => 'RSA' imported but unused pyflakes.ignore stem/prereq.py => 'asn1' imported but unused +pyflakes.ignore stem/prereq.py => 'mock' imported but unused pyflakes.ignore stem/prereq.py => 'long_to_bytes' imported but unused pyflakes.ignore stem/descriptor/__init__.py => redefinition of unused 'OrderedDict' from line 61 pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_bytes_impl' from line 51 diff --git a/test/util.py b/test/util.py index 4cdbdc2..4ab7460 100644 --- a/test/util.py +++ b/test/util.py @@ -378,6 +378,14 @@ def check_pycrypto_version(): return "missing" +def check_mock_version(): + if stem.prereq.is_mock_available(): + import mock + return mock.__version__ + else: + return "missing" + + def check_pyflakes_version(): try: import pyflakes

1 0