October 2018 - tor-commits - lists.torproject.org

[stem/master] Avoid downloading descriptors from Serge
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 98e1db3a2cc57c1a73adcd28001906f6c271bdfd Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 24 14:05:24 2018 -0700 Avoid downloading descriptors from Serge Like Bifroest before it, Serge is a bridge authority and frequently times out when asked to serve descriptors. Avoiding it, both so descriptor downloading works better and so I stop getting so many timeout notifications from DocTor. --- stem/descriptor/remote.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index dbae3cdf..f1171697 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -297,6 +297,10 @@ class Query(object): Avoid downloading from Bifroest. This is the bridge authority so it doesn't vote in the consensus, and apparently times out frequently. + .. versionchanged:: 1.8.0 + Serge has replaced Bifroest as our bridge authority. Avoiding descriptor + downloads from it instead. + :var str resource: resource being fetched, such as '/tor/server/all' :var str descriptor_type: type of descriptors being fetched (for options see :func:`~stem.descriptor.__init__.parse_file`), this is guessed from the @@ -497,7 +501,7 @@ class Query(object): """ if use_authority or not self.endpoints: - picked = random.choice([auth for auth in stem.directory.Authority.from_cache().values() if auth.nickname not in ('tor26', 'Bifroest')]) + picked = random.choice([auth for auth in stem.directory.Authority.from_cache().values() if auth.nickname not in ('tor26', 'Serge')]) return stem.DirPort(picked.address, picked.dir_port) else: return random.choice(self.endpoints)

1 0

[stem/master] Integration test race when multiple tor processes are running
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 165c75ba21184b4fe865e7a30ce2d69012836a26 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 24 12:40:38 2018 -0700 Integration test race when multiple tor processes are running Our flakiest integration tests on Jenkins tend to be our system tests... ====================================================================== FAIL: test_pid_by_name_ps_linux ---------------------------------------------------------------------- Traceback (most recent call last): File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/require.py", line 58, in wrapped return func(self, *args, **kwargs) File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/require.py", line 58, in wrapped return func(self, *args, **kwargs) File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/require.py", line 58, in wrapped return func(self, *args, **kwargs) File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/integ/util/system.py", line 211, in test_pid_by_name_ps_linux self.assertEqual(tor_pid, stem.util.system.pid_by_name(tor_cmd)) AssertionError: 31963 != None ---------------------------------------------------------------------- These tests skip themselves if multiple tor processes are running, *but* they fail in this fashion if a tor process spawns *during* the test. To reduce the chances of this the tests now check for additional tor processes both before *and* after the test. In theory an error could still occure if a tor process both starts *and* stops in the middle of a test but I'm unsure if this will really happen in practice. Lets see. If these assertion errors continue to show up I might simply move them under a special target. --- test/integ/util/system.py | 21 ++++++++++++++++++++- test/require.py | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/test/integ/util/system.py b/test/integ/util/system.py index cc6f9eda..06202bfa 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -65,13 +65,32 @@ def _has_port(): return test.runner.Torrc.PORT in test.runner.get_runner().get_options() -require_single_tor_instance = test.require.needs(_is_single_tor_running, 'multiple tor instances') require_control_port = test.require.needs(_has_port, 'test instance has no port') require_linux = test.require.needs(_is_linux, 'linux only') require_bsd = test.require.needs(stem.util.system.is_bsd, 'bsd only') require_path = test.require.needs(lambda: 'PATH' in os.environ, 'requires PATH') +def require_single_tor_instance(func): + # Checking both before and after the test to see if we're running only a + # single tor instance. We do both to narrow the possability of the test + # failing due to a race. + + def wrapped(self, *args, **kwargs): + if _is_single_tor_running(): + try: + return func(self, *args, **kwargs) + except: + if _is_single_tor_running(): + raise + else: + self.skipTest('(multiple tor instances)') + else: + self.skipTest('(multiple tor instances)') + + return wrapped + + class TestSystem(unittest.TestCase): def test_daemon_task_when_successful(self): """ diff --git a/test/require.py b/test/require.py index 598576bf..25a7ae12 100644 --- a/test/require.py +++ b/test/require.py @@ -49,7 +49,7 @@ def only_run_once(func): def needs(condition, message): """ - Skips teh test unless the conditional evaluates to 'true'. + Skips the test unless the conditional evaluates to 'true'. """ def decorator(func):

1 0

[stem/master] Note that set_process_name can raise IOErrors
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 4f993e79ff1acc9e565b190be97f9a30a066fcc3 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Oct 19 12:05:12 2018 -0700 Note that set_process_name can raise IOErrors Oops, the code explicitly raises these when unable to do so but our docs didn't mention it. --- stem/util/system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stem/util/system.py b/stem/util/system.py index 17eeb61d..2d52b022 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -1409,6 +1409,8 @@ def set_process_name(process_name): best-effort, not necessarily working on all platforms. :param str process_name: new name for our process + + :raises: **IOError** if the process cannot be renamed """ # This is mostly based on...

1 0

[stem/master] Ensure _emit_event() helper mocks time.time()
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 94837b90d058a16e56b091a25889c03c8746c18d Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 24 13:15:28 2018 -0700 Ensure _emit_event() helper mocks time.time() Nothing actually changed. Just moving our time.time() mocks into the _emit_event() helper since that mock's always needed for tests that send events. --- test/unit/control/controller.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/test/unit/control/controller.py b/test/unit/control/controller.py index ed00385d..ed72fbe8 100644 --- a/test/unit/control/controller.py +++ b/test/unit/control/controller.py @@ -669,7 +669,6 @@ class TestControl(unittest.TestCase): self.assertRaises(InvalidRequest, self.controller.add_event_listener, Mock(), EventType.SIGNAL) - @patch('time.time', Mock(return_value = TEST_TIMESTAMP)) def test_events_get_received(self): """ Trigger an event, checking that our listeners get notified. @@ -683,7 +682,6 @@ class TestControl(unittest.TestCase): self._emit_event(BW_EVENT) self.bw_listener.assert_called_once_with(BW_EVENT) - @patch('time.time', Mock(return_value = TEST_TIMESTAMP)) def test_event_listing_with_error(self): """ Raise an exception in an event listener to confirm it doesn't break our @@ -700,7 +698,6 @@ class TestControl(unittest.TestCase): self._emit_event(BW_EVENT) self.bw_listener.assert_called_once_with(BW_EVENT) - @patch('time.time', Mock(return_value = TEST_TIMESTAMP)) def test_event_listing_with_malformed_event(self): """ Attempt to parse a malformed event emitted from Tor. It's important this @@ -857,17 +854,18 @@ class TestControl(unittest.TestCase): # processed asynchronously, so the only way to endsure it's done # with its work is to join on the thread. - with patch('stem.control.Controller.is_alive') as is_alive_mock: - is_alive_mock.return_value = True - self.controller._launch_threads() - - try: - # Converting an event back into an uncast ControlMessage, then feeding it - # into our controller's event queue. - - uncast_event = ControlMessage.from_str(event.raw_content()) - self.controller._event_queue.put(uncast_event) - self.controller._event_notice.set() - finally: - is_alive_mock.return_value = False - self.controller._close() + with patch('time.time', Mock(return_value = TEST_TIMESTAMP)): + with patch('stem.control.Controller.is_alive') as is_alive_mock: + is_alive_mock.return_value = True + self.controller._launch_threads() + + try: + # Converting an event back into an uncast ControlMessage, then feeding it + # into our controller's event queue. + + uncast_event = ControlMessage.from_str(event.raw_content()) + self.controller._event_queue.put(uncast_event) + self.controller._event_notice.set() + finally: + is_alive_mock.return_value = False + self.controller._close()

1 0

[stem/master] Inconsistent unit test failures in control.controller
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 5bd8a10632e3602e03b4ad2ac8829c57585c4fd6 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Oct 24 13:31:55 2018 -0700 Inconsistent unit test failures in control.controller On occasion our unit tests that emit then listen for controller events fail with... ====================================================================== FAIL: test_events_get_received ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/lib/python2.7/dist-packages/mock/mock.py", line 1305, in patched return func(*args, **keywargs) File "/srv/jenkins-workspace/workspace/stem-tor-ci/test/unit/control/controller.py", line 684, in test_events_get_received self.bw_listener.assert_called_once_with(BW_EVENT) File "/usr/lib/python2.7/dist-packages/mock/mock.py", line 947, in assert_called_once_with raise AssertionError(msg) AssertionError: Expected 'mock' to be called once. Called 0 times. ---------------------------------------------------------------------- The specific test varies, but it should always be one of our emit_event() callers. This has caused nine jenkins test failures in the last three months (so pretty noisy). Dave ran into this too... https://trac.torproject.org/projects/tor/ticket/27173 I can't reliably (or even frequently) reproduce this locally but I think I know what's going on... * emit_event() adds an event to the queue via put(). This is a blocking call so we can be assured the event's in there. * emit_event() then calls set() on the event_notice condition so the Controller's event thread will process the event. * The controller's event thread calls get_nowait() on its event queue. This *ususally* gets the event we just inserted, but since this is get_nowait() rather than get() it's non-blocking, and may raise a queue.Empty instead. * When queue.Empty gets thrown we once again wait on the event_notice condition. * emit_event() then calls close() on its Controller which joins and terminates its threads. The close() method calls set() on event_noice to give it a *second* opportunity to get the event. * Once again, because we use get_nowait() there's a small chance we'll raise queue.Empty without processing the event. * The event thread then terminates with the enqueued event never being processed. * Finally, we get the assertion failure above. This is actually a real bug that can cause Controller users to be highly delayed or never receive an event. For instance... * Tor emits an event. * Our reader thread enqueues the event and calls set() on the condition to notify the event thread. * For the reasons mentioned above it fails to do so. * Because we use wait() without a timeout our event thread then resumes blocking, causing the event to not get another chance to be delivered until a second event comes along. This sucks, and is pretty easy to remediate. Rather than waiting indefinitely we should instead have our event thread periodically check re-check its own queue for undelivered messages (picked 50 ms as a decent sounding balance between responsiveness and load). Our test's emit_event() in turn now blocks until the event is delivered. Needless to say this is a long song and dance so quite possible I'm in the wrong on something but this seems, both in theory and practice thus far, to be a good step forward. --- docs/change_log.rst | 4 ++++ stem/control.py | 3 ++- test/unit/control/controller.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 9230350c..f47ec984 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -45,6 +45,10 @@ Unreleased The following are only available within Stem's `git repository <download.html>`_. + * **Controller** + + * Controller events could fail to be delivered in a timely fashion (:trac:`27173`) + .. _version_1.7: Version 1.7 (October 7th, 2018) diff --git a/stem/control.py b/stem/control.py index 2e893788..68873d76 100644 --- a/stem/control.py +++ b/stem/control.py @@ -966,6 +966,7 @@ class BaseController(object): try: event_message = self._event_queue.get_nowait() self._handle_event(event_message) + self._event_queue.task_done() # Attempt to finish processing enqueued events when our controller closes @@ -978,7 +979,7 @@ class BaseController(object): if not self.is_alive(): break - self._event_notice.wait() + self._event_notice.wait(0.05) self._event_notice.clear() diff --git a/test/unit/control/controller.py b/test/unit/control/controller.py index ed72fbe8..13d26c4b 100644 --- a/test/unit/control/controller.py +++ b/test/unit/control/controller.py @@ -866,6 +866,7 @@ class TestControl(unittest.TestCase): uncast_event = ControlMessage.from_str(event.raw_content()) self.controller._event_queue.put(uncast_event) self.controller._event_notice.set() + self.controller._event_queue.join() # block until the event is consumed finally: is_alive_mock.return_value = False self.controller._close()

1 0

[stem/master] Note python interpreter in test output
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 7cd58724f0a3ad8bcbdb7c6f7a11522df8dafb5d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 8 10:56:08 2018 -0700 Note python interpreter in test output Our test output notes our python version but not the intterpreter type. This caused Pypy for instance to look like CPython 2.7. Intercluding the interpreter in the version string if it isn't CPython. --- test/task.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/task.py b/test/task.py index d1c1d6c7..a763e710 100644 --- a/test/task.py +++ b/test/task.py @@ -75,6 +75,13 @@ def _check_tor_version(tor_path): return str(test.tor_version(tor_path)).split()[0] +def _check_python_version(): + interpreter = platform.python_implementation() + version = platform.python_version() + + return version if interpreter == 'CPython' else '%s (%s)' % (interpreter, version) + + def _check_platform_version(): if platform.system() == 'Windows': extra = platform.release() @@ -271,7 +278,7 @@ class StaticCheckTask(Task): STEM_VERSION = Task('stem version', lambda: stem.__version__) TOR_VERSION = Task('tor version', _check_tor_version) -PYTHON_VERSION = Task('python version', lambda: platform.python_version()) +PYTHON_VERSION = Task('python version', _check_python_version) PLATFORM_VERSION = Task('operating system version', _check_platform_version) CRYPTO_VERSION = ModuleVersion('cryptography version', 'cryptography', stem.prereq.is_crypto_available) PYNACL_VERSION = ModuleVersion('pynacl version', 'nacl', stem.prereq._is_pynacl_available)

1 0

[stem/master] Drop vague note about set_process_name FreeBSD issue
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 035432d31234096d41adfac0411e165b90653a3a Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Oct 19 12:02:51 2018 -0700 Drop vague note about set_process_name FreeBSD issue The user disappeared before figuring out what was up and I haven't heard anything since. As such this note is too vague to really be useful to users. --- stem/util/system.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stem/util/system.py b/stem/util/system.py index 3c03bc46..17eeb61d 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -1408,8 +1408,6 @@ def set_process_name(process_name): Renames our current process from "python <args>" to a custom name. This is best-effort, not necessarily working on all platforms. - **Note:** This might have issues on FreeBSD (:trac:`9804`). - :param str process_name: new name for our process """

1 0

[stem/master] Stem development version tag
by atagar＠torproject.org 25 Oct '18

25 Oct '18

commit 7e3a2e9ccb2fca2eddf426340f6a95b554b834e2 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 8 10:51:43 2018 -0700 Stem development version tag As always, flagging our git codebase with a '-dev' version suffix. --- stem/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/__init__.py b/stem/__init__.py index 9a9aec59..771266fb 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -488,7 +488,7 @@ Library for working with the tor process. import stem.util import stem.util.enum -__version__ = '1.7.0' +__version__ = '1.7.0-dev' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' __url__ = 'https://stem.torproject.org/'

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 25 Oct '18

25 Oct '18

commit 28fa2e0ca4d54b092244c639d5e078988d4540b7 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Oct 24 20:16:03 2018 +0000 Update translations for tails-misc --- fr.po | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fr.po b/fr.po index dc2acb4da..dbdc16f67 100644 --- a/fr.po +++ b/fr.po @@ -11,6 +11,7 @@ # Athorcis, 2015 # Domiho Zannou <zannou.gery(a)gmail.com>, 2018 # Emmanuel Simond <emmanuel.simond(a)gmail.com>, 2014 +# Emma Peel, 2018 # AO <ao(a)localizationlab.org>, 2017-2018 # AO <ao(a)localizationlab.org>, 2016-2017 # Gwennole Hangard <gwennole.hangard(a)gmail.com>, 2015 @@ -29,8 +30,8 @@ msgstr "" "Project-Id-Version: Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2018-10-15 14:47+0200\n" -"PO-Revision-Date: 2018-10-24 14:20+0000\n" -"Last-Translator: AO <ao(a)localizationlab.org>\n" +"PO-Revision-Date: 2018-10-24 20:00+0000\n" +"Last-Translator: Emma Peel\n" "Language-Team: French (http://www.transifex.com/otf/torproject/language/fr/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -346,7 +347,7 @@ msgid "" "To do so, install some software using <a href=\"synaptic.desktop\">Synaptic " "Package Manager</a> or <a href=\"org.gnome.Terminal.desktop\">APT on the " "command line</a>." -msgstr "" +msgstr "Pour en ajouter, installez des logiciels en utilisant le <a href=\"synaptic.desktop\">Gestionnaire de Paquets Synaptic</a> ou <a href=\"org.gnome.Terminal.desktop\">APT à partir de la ligne de commande</a>." #: config/chroot_local-includes/usr/local/bin/tails-additional-software-config:188 msgid ""

1 0

[translation/donatepages-messagespot_completed] Update translations for donatepages-messagespot_completed
by translation＠torproject.org 25 Oct '18

25 Oct '18

commit 7a9db3b06534b53d95b410896963d6b1c0ee9a1a Author: Translation commit bot <translation(a)torproject.org> Date: Wed Oct 24 18:15:32 2018 +0000 Update translations for donatepages-messagespot_completed --- locale/es_AR/LC_MESSAGES/messages.po | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/locale/es_AR/LC_MESSAGES/messages.po b/locale/es_AR/LC_MESSAGES/messages.po index 1d36f4c9b..50c08cbc2 100644 --- a/locale/es_AR/LC_MESSAGES/messages.po +++ b/locale/es_AR/LC_MESSAGES/messages.po @@ -1,11 +1,11 @@ # Translators: # erinm, 2018 +# Joaquín Serna, 2018 # Zuhualime Akoochimoya, 2018 # Emma Peel, 2018 -# Joaquín Serna, 2018 msgid "" msgstr "" -"Last-Translator: Joaquín Serna, 2018\n" +"Last-Translator: Emma Peel, 2018\n" "Language-Team: Spanish (Argentina) (https://www.transifex.com/otf/teams/1519/es_AR/)\n" "Language: es_AR\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" @@ -195,7 +195,7 @@ msgstr "Clásico" #: tmp/cache_locale/ce/ce708c1cd991748e8c1c29f932e6ddbd1be5be1b4cc2c5b49b607cae1df80432.php:55 msgid "European" -msgstr "Europeos" +msgstr "Europeo" #: tmp/cache_locale/ce/ce708c1cd991748e8c1c29f932e6ddbd1be5be1b4cc2c5b49b607cae1df80432.php:65 msgid "Size" @@ -207,7 +207,7 @@ msgstr "Elegí talle" #: tmp/cache_locale/ce/ce708c1cd991748e8c1c29f932e6ddbd1be5be1b4cc2c5b49b607cae1df80432.php:73 msgid "S" -msgstr "G" +msgstr "S" #: tmp/cache_locale/ce/ce708c1cd991748e8c1c29f932e6ddbd1be5be1b4cc2c5b49b607cae1df80432.php:77 msgid "M" @@ -317,7 +317,7 @@ msgstr "" #: tmp/cache_locale/c7/c763c19bb6abb9330294c550c8241bb3874e3b4e17fb6e7b15db26c60df8d5fe.php:166 msgid "sticker Pack" -msgstr "Paquete de autoadhesivos" +msgstr "Paquete de stickers autoadhesivos" #: tmp/cache_locale/c7/c763c19bb6abb9330294c550c8241bb3874e3b4e17fb6e7b15db26c60df8d5fe.php:173 msgid "" @@ -336,7 +336,7 @@ msgid "" "Get our limited edition Tor: Powering Digital Resistance or Open Observatory" " of Network Interference (OONI) t-shirt." msgstr "" -"Tené nuestra remera Tor de edición limitada: Potenciando la Resistencia " +"Obtené nuestra remera de Tor edición limitada: Potenciando la Resistencia " "Digital u Observatorio Abierto de Interferencia en la Red (OONI)." #: tmp/cache_locale/c7/c763c19bb6abb9330294c550c8241bb3874e3b4e17fb6e7b15db26c60df8d5fe.php:198 @@ -360,8 +360,8 @@ msgid "" "t-shirt." msgstr "" "Nuestra remera Tor: Potenciando la Resistencia Digital u Observatorio " -"Abierto de Interferencia en la Red (OONI), más una Tor en el Corazón de la " -"Libertad en Internet." +"Abierto de Interferencia en la Red (OONI), más una remera de Tor en el " +"Corazón de la Libertad en Internet." #: tmp/cache_locale/c7/c763c19bb6abb9330294c550c8241bb3874e3b4e17fb6e7b15db26c60df8d5fe.php:243 msgid "sweatshirt"

1 0