January 2017 - tor-commits - lists.torproject.org

[stem/master] Speed tests and descriptor parsing
by atagar＠torproject.org 30 Jan '17

30 Jan '17

commit 10fc8134065a57f5c2c1d012e70cbe13ba6a700f Merge: d713b29 40fd99d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 30 09:55:07 2017 -0800 Speed tests and descriptor parsing During my morning commute I've been looking into Stem's test speed. Faster tests mean quicker development cycles, so nice investment for the future. Main changes are... * Greatly sped up descriptor parsing, especially when there's validation. Should help with memory usage too. * When running our tests with the --verbose argument we now show individual test runtimes. This isn't a capability python's unittest module has so providing it in our test_tools util. Unit tests improved quite a bit on my netbook but has no negligible impact on my PC. Integ tests however have improved quite a bit, dropping from 57.0s to 44.7s (22% faster). docs/change_log.rst | 5 +++ run_tests.py | 14 +++++-- stem/descriptor/__init__.py | 46 ++++++++++++----------- stem/descriptor/networkstatus.py | 10 ++--- stem/process.py | 5 ++- stem/util/test_tools.py | 51 ++++++++++++++++++++++++++ test/integ/control/base_controller.py | 29 +++++++-------- test/integ/control/controller.py | 67 ++++++---------------------------- test/integ/descriptor/remote.py | 4 +- test/integ/process.py | 8 ++-- test/integ/socket/control_message.py | 22 ++++++----- test/integ/util/connection.py | 18 ++++----- test/mocking.py | 10 +++++ test/output.py | 28 ++++++++++++++ test/unit/connection/authentication.py | 28 +++++--------- test/unit/control/controller.py | 2 +- test/unit/version.py | 6 +-- test/util.py | 5 +++ 18 files changed, 211 insertions(+), 147 deletions(-)

1 0

[stem/master] Defer defaulting to speed descriptor parsing
by atagar＠torproject.org 30 Jan '17

30 Jan '17

commit 3eecbe1f5ce468769d5d884172eeca677e2f1b30 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 25 10:58:50 2017 -0800 Defer defaulting to speed descriptor parsing Defaulting descriptor attributes when they're accessed rather than doing this right away. This speeds reading descriptors with validation by ~25%... Before descriptor.microdescriptor... success (15.38s) descriptor.networkstatus... success (23.25s) After descriptor.microdescriptor... success (11.67s) descriptor.networkstatus... success (19.80s) --- docs/change_log.rst | 2 +- stem/descriptor/__init__.py | 48 ++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 477db2e..1111afd 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -45,7 +45,7 @@ The following are only available within Stem's `git repository * **Descriptors** - * Sped descriptor reading by ~5% by not cloning immutable fields + * Sped descriptor reading by ~25% by deferring defaulting when validating * Support for protocol descriptor fields (:spec:`eb4fb3c`) * Shared randomness properties weren't being read in votes (:trac:`21102`) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 67ea374..1262f6a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -437,6 +437,15 @@ def _parse_key_block(keyword, attribute, expected_block_type, value_attribute = return _parse +def _copy(default): + if default is None or isinstance(default, (bool, stem.exit_policy.ExitPolicy)): + return default # immutable + elif default in EMPTY_COLLECTION: + return type(default)() # collection construction tad faster than copy + else: + return copy.copy(default) + + class Descriptor(object): """ Common parent for all types of descriptors. @@ -516,21 +525,6 @@ class Descriptor(object): if parser_for_line is None: parser_for_line = self.PARSER_FOR_LINE - # set defaults - - for attr in self.ATTRIBUTES: - if not hasattr(self, attr): - value = self.ATTRIBUTES[attr][0] - - if value is None or isinstance(value, (bool, stem.exit_policy.ExitPolicy)): - pass # immutable - elif value in EMPTY_COLLECTION: - value = type(value)() # collection construction tad faster than copy - else: - value = copy.copy(value) - - setattr(self, attr, value) - for keyword, values in list(entries.items()): try: if keyword in parser_for_line: @@ -654,19 +648,25 @@ class Descriptor(object): if name in ('_lazy_loading', 'ATTRIBUTES'): return super(Descriptor, self).__getattribute__(name) - # If attribute isn't already present we might be lazy loading it... + # If an attribute we should have isn't present it means either... + # + # a. we still need to lazy load this + # b. we read the whole descriptor but it wasn't present, so needs the default - if self._lazy_loading and name in self.ATTRIBUTES: + if name in self.ATTRIBUTES and not hasattr(self, name): default, parsing_function = self.ATTRIBUTES[name] - try: - parsing_function(self, self._entries) - except (ValueError, KeyError): + if self._lazy_loading: try: - # despite having a validation failure check to see if we set something - return super(Descriptor, self).__getattribute__(name) - except AttributeError: - setattr(self, name, copy.copy(default)) + parsing_function(self, self._entries) + except (ValueError, KeyError): + try: + # despite having a validation failure check to see if we set something + return super(Descriptor, self).__getattribute__(name) + except AttributeError: + setattr(self, name, _copy(default)) + else: + setattr(self, name, _copy(default)) return super(Descriptor, self).__getattribute__(name)

1 0

[stem/master] Drop unnecessary check in descriptor accessor
by atagar＠torproject.org 30 Jan '17

30 Jan '17

commit 50c2539d7cfdf7cb8b07c58791ef29fb91cd9045 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 25 18:28:31 2017 -0800 Drop unnecessary check in descriptor accessor The following conditional already checks if the attribute we're after is among a certain set. No need for this check too. --- stem/descriptor/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 1262f6a..0d4e73e 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -642,12 +642,6 @@ class Descriptor(object): return stem.util.str_tools._to_unicode(digest_hash.hexdigest().upper()) def __getattr__(self, name): - # Our constructor sets these, but when unpickling we might lack them. This - # check is needed to avoid an infinite loop in that case. - - if name in ('_lazy_loading', 'ATTRIBUTES'): - return super(Descriptor, self).__getattribute__(name) - # If an attribute we should have isn't present it means either... # # a. we still need to lazy load this

1 0

[stem/master] Add new GUARD_WAIT circuit status
by atagar＠torproject.org 30 Jan '17

30 Jan '17

commit 91b5b9ce404ca099089d12d8b66e6c67a423e1e9 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 30 08:35:27 2017 -0800 Add new GUARD_WAIT circuit status Newly added circuit status... https://gitweb.torproject.org/torspec.git/commit/?id=6446210 --- docs/change_log.rst | 4 ++++ stem/__init__.py | 23 ++++++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 1111afd..85619b2 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -43,6 +43,10 @@ Unreleased The following are only available within Stem's `git repository <download.html>`_. + * **Controller** + + * Added the GUARD_WAIT :data:`~stem.CircStatus` (:spec:`6446210`) + * **Descriptors** * Sped descriptor reading by ~25% by deferring defaulting when validating diff --git a/stem/__init__.py b/stem/__init__.py index d74e90b..de2ce9d 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -86,15 +86,19 @@ Library for working with the tor process. Statuses that a circuit can be in. Tor may provide statuses not in this enum. - ============ =========== - CircStatus Description - ============ =========== - **LAUNCHED** new circuit was created - **BUILT** circuit finished being created and can accept traffic - **EXTENDED** circuit has been extended by a hop - **FAILED** circuit construction failed - **CLOSED** circuit has been closed - ============ =========== + .. versionchanged:: 1.6.0 + Added the GUARD_WAIT signal. + + ============== =========== + CircStatus Description + ============== =========== + **LAUNCHED** new circuit was created + **BUILT** circuit finished being created and can accept traffic + **GUARD_WAIT** waiting to see if there's a circuit with a better guard before using + **EXTENDED** circuit has been extended by a hop + **FAILED** circuit construction failed + **CLOSED** circuit has been closed + ============== =========== .. data:: CircBuildFlag (enum) @@ -653,6 +657,7 @@ Signal = stem.util.enum.UppercaseEnum( CircStatus = stem.util.enum.UppercaseEnum( 'LAUNCHED', 'BUILT', + 'GUARD_WAIT', 'EXTENDED', 'FAILED', 'CLOSED',

1 0

[stem/master] Validation for params fails due to ordering
by atagar＠torproject.org 30 Jan '17

30 Jan '17

commit 40fd99d8f14261f8221734e837ea0db1028dd127 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 27 19:17:15 2017 -0800 Validation for params fails due to ordering We check that a params attribute is only supplied in consensus methods that should have it. However, we checked it before the value might be set. I'm not sure why this only surfaced with recent changes but unit tests failed for this reason. --- stem/descriptor/__init__.py | 2 +- stem/descriptor/networkstatus.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 0d4e73e..ee0b289 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -643,7 +643,7 @@ class Descriptor(object): def __getattr__(self, name): # If an attribute we should have isn't present it means either... - # + # # a. we still need to lazy load this # b. we read the whole descriptor but it wasn't present, so needs the default diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 9818624..78b453b 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -638,11 +638,6 @@ def _parse_header_parameters_line(descriptor, entries): value = _value('params', entries) - # should only appear in consensus-method 7 or later - - if not descriptor.meets_consensus_method(7): - raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later") - if value != '': descriptor.params = _parse_int_mappings('params', value, True) descriptor._check_params_constraints() @@ -998,6 +993,11 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): self._parse(entries, validate, parser_for_line = self.HEADER_PARSER_FOR_LINE) + # should only appear in consensus-method 7 or later + + if not self.meets_consensus_method(7) and 'params' in list(entries.keys()): + raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later") + _check_for_missing_and_disallowed_fields(self, entries, HEADER_STATUS_DOCUMENT_FIELDS) # default consensus_method and consensus_methods based on if we're a consensus or vote

1 0

[translation/tor-launcher-properties_completed] Update translations for tor-launcher-properties_completed
by translation＠torproject.org 30 Jan '17

30 Jan '17

commit d3fc100d6da7cf57e68a989aed97291e84f98ffc Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 30 17:15:59 2017 +0000 Update translations for tor-launcher-properties_completed --- pt/torlauncher.properties | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pt/torlauncher.properties b/pt/torlauncher.properties index 61b4aa9..4689953 100644 --- a/pt/torlauncher.properties +++ b/pt/torlauncher.properties @@ -1,8 +1,9 @@ -### Copyright (c) 2014, The Tor Project, Inc. +### Copyright (c) 2016, The Tor Project, Inc. ### See LICENSE for licensing information. torlauncher.error_title=Executor Tor +torlauncher.tor_exited_during_startup=O Tor fechou-se durante o arranque. Isto pode ter acontecido devido a erro no seu ficheiro torrc, um erro no Tor ou outro programa no seu sistema, ou falha de hardware. Até que corrija o problema em causa reinicie o Tor, o Navegador Tor não irá iniciar. torlauncher.tor_exited=O Tor encerrou inesperadamente. Isto pode ter acontecido devido a erro dentro do Tor, outro programa no seu sistema ou falha de hardware. Até que reinicie o Tor, o Navegador Tor não irá poder aceder a quaisquer sítios da Web. Se o problema persistir, por favor, envie uma cópia do Registo de Eventos do seu Tor para a equipa de suporte. torlauncher.tor_exited2=Ao reiniciar o Tor não irá fechar os separadores do seu navegador. torlauncher.tor_controlconn_failed=Não foi possível ligar à porta de controlo do Tor. @@ -13,8 +14,8 @@ torlauncher.tor_bootstrap_failed_details=%1$S falhou (%2$S). torlauncher.unable_to_start_tor=Não é possível iniciar o Tor.\n\n %S torlauncher.tor_missing=O executável do Tor está em falta. -torlauncher.torrc_missing=O ficheiro torrc está em falta. -torlauncher.datadir_missing=A pasta dos dados do Tor não existe. +torlauncher.torrc_missing=O ficheiro torc não existe e não pode ser criado. +torlauncher.datadir_missing=O diretório de dados Tor não existe e não pode ser criado. torlauncher.password_hash_missing=Não foi possível obter a senha "hashed". torlauncher.failed_to_get_settings=Não é possível obter as configurações do Tor\n\n %S @@ -36,6 +37,7 @@ torlauncher.quit_win=Fechar torlauncher.done=Concluído torlauncher.forAssistance=Para assistência, contacte %S +torlauncher.forAssistance2=Para assistência, visite %S torlauncher.copiedNLogMessages=Cópia completa. Estão prontas %S mensagens do registo de eventos do Tor para serem coladas num editor de texto ou numa mensagem de correio eletrónico.

1 0

[translation/tor-launcher-properties] Update translations for tor-launcher-properties
by translation＠torproject.org 30 Jan '17

30 Jan '17

commit b3c9eeb7e0279c2e3dec96bc820b9ed6338abea6 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Jan 30 17:15:55 2017 +0000 Update translations for tor-launcher-properties --- pt/torlauncher.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pt/torlauncher.properties b/pt/torlauncher.properties index a01fa61..4689953 100644 --- a/pt/torlauncher.properties +++ b/pt/torlauncher.properties @@ -3,7 +3,7 @@ torlauncher.error_title=Executor Tor -torlauncher.tor_exited_during_startup=Tor exited during startup. This might be due to an error in your torrc file, a bug in Tor or another program on your system, or faulty hardware. Until you fix the underlying problem and restart Tor, Tor Browser will not start. +torlauncher.tor_exited_during_startup=O Tor fechou-se durante o arranque. Isto pode ter acontecido devido a erro no seu ficheiro torrc, um erro no Tor ou outro programa no seu sistema, ou falha de hardware. Até que corrija o problema em causa reinicie o Tor, o Navegador Tor não irá iniciar. torlauncher.tor_exited=O Tor encerrou inesperadamente. Isto pode ter acontecido devido a erro dentro do Tor, outro programa no seu sistema ou falha de hardware. Até que reinicie o Tor, o Navegador Tor não irá poder aceder a quaisquer sítios da Web. Se o problema persistir, por favor, envie uma cópia do Registo de Eventos do seu Tor para a equipa de suporte. torlauncher.tor_exited2=Ao reiniciar o Tor não irá fechar os separadores do seu navegador. torlauncher.tor_controlconn_failed=Não foi possível ligar à porta de controlo do Tor.

1 0

[tor/master] Remove XXXXprop271 comments from test_entrynodes.c
by nickm＠torproject.org 30 Jan '17

30 Jan '17

commit ed4a3dfef20935e6f357c2881a2db6b2b4445566 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 30 10:43:53 2017 -0500 Remove XXXXprop271 comments from test_entrynodes.c These commments were complaints about how I didn't like some aspects of prop271. They have been superseded by ticket 20832. --- src/test/test_entrynodes.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index bcb56e3..249fd1f 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -2044,7 +2044,6 @@ test_entry_guard_select_for_circuit_highlevel_confirm_other(void *arg) u = entry_guard_succeeded(&guard); /* We're on the internet (by fiat), so this guard will get called "confirmed" * and should immediately become primary. - * XXXX prop271 -- I don't like that behavior, but it's what is specified */ tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE); tt_assert(u == GUARD_USABLE_NOW); @@ -2338,9 +2337,6 @@ test_entry_guard_upgrade_a_circuit(void *arg) * primary guards are down, we have two WAITING circuits: one will * get upgraded to COMPLETED! (The one that started first.) */ - /* XXXX prop271 -- perhaps the one that started first should - * also wind up in confirmed_entry_guards earlier? - */ smartlist_t *result = smartlist_new(); int r;

1 0

[tor/master] Remove a couple of stale comments from entrynodes.h
by nickm＠torproject.org 30 Jan '17

30 Jan '17

commit 7d0df8bad857e48252d28445dbdc9c087d2e8168 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 30 10:38:24 2017 -0500 Remove a couple of stale comments from entrynodes.h --- src/or/entrynodes.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index bd501d0..d469df7 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -392,9 +392,6 @@ int num_bridges_usable(void); #define DFLT_MAX_SAMPLE_SIZE 60 /** * We always try to make our sample contain at least this many guards. - * - * XXXX prop271 There was a MIN_SAMPLE_THRESHOLD in the proposal, but I - * removed it in favor of MIN_FILTERED_SAMPLE_SIZE. -NM */ #define DFLT_MIN_FILTERED_SAMPLE_SIZE 20 /** @@ -466,7 +463,6 @@ STATIC int get_nonprimary_guard_idle_timeout(void); STATIC double get_meaningful_restriction_threshold(void); STATIC double get_extreme_restriction_threshold(void); -// ---------- XXXX these functions and definitions are post-prop271. HANDLE_DECL(entry_guard, entry_guard_t, STATIC) STATIC guard_selection_type_t guard_selection_infer_type( guard_selection_type_t type_in,

1 0

[tor/master] entrynodes: Remove "split these functions" XXXXs
by nickm＠torproject.org 30 Jan '17

30 Jan '17

commit cccd3f1dae32075ffd4c53de7709fc83b7db43fb Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jan 30 10:49:40 2017 -0500 entrynodes: Remove "split these functions" XXXXs They now have a ticket: #21349. --- src/or/entrynodes.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 587dec6..c5fb92e 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1157,7 +1157,6 @@ entry_guard_is_listed,(guard_selection_t *gs, const entry_guard_t *guard)) STATIC void sampled_guards_update_from_consensus(guard_selection_t *gs) { - /*XXXX prop271 consider splitting this function up. */ tor_assert(gs); const int REMOVE_UNLISTED_GUARDS_AFTER = (get_remove_unlisted_guards_after_days() * 86400); @@ -1581,7 +1580,6 @@ make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard) STATIC void entry_guards_update_primary(guard_selection_t *gs) { - /*XXXX prop271 consider splitting this function up. */ tor_assert(gs); // prevent recursion. Recursion is potentially very bad here. @@ -1779,7 +1777,6 @@ select_entry_guard_for_circuit(guard_selection_t *gs, const entry_guard_restriction_t *rst, unsigned *state_out) { - /*XXXX prop271 consider splitting this function up. */ tor_assert(gs); tor_assert(state_out);

1 0