[tor-commits] [stem/master] Defer defaulting to speed descriptor parsing

[atagar at torproject.org]

commit 3eecbe1f5ce468769d5d884172eeca677e2f1b30
Author: Damian Johnson <atagar at torproject.org>
Date:   Wed Jan 25 10:58:50 2017 -0800

    Defer defaulting to speed descriptor parsing
    
    Defaulting descriptor attributes when they're accessed rather than doing this
    right away. This speeds reading descriptors with validation by ~25%...
    
      Before
      descriptor.microdescriptor...                        success (15.38s)
      descriptor.networkstatus...                          success (23.25s)
    
      After
      descriptor.microdescriptor...                        success (11.67s)
      descriptor.networkstatus...                          success (19.80s)
---
 docs/change_log.rst         |  2 +-
 stem/descriptor/__init__.py | 48 ++++++++++++++++++++++-----------------------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst
index 477db2e..1111afd 100644
--- a/docs/change_log.rst
+++ b/docs/change_log.rst
@@ -45,7 +45,7 @@ The following are only available within Stem's `git repository
 
  * **Descriptors**
 
-  * Sped descriptor reading by ~5% by not cloning immutable fields
+  * Sped descriptor reading by ~25% by deferring defaulting when validating
   * Support for protocol descriptor fields (:spec:`eb4fb3c`)
   * Shared randomness properties weren't being read in votes (:trac:`21102`)
 
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 67ea374..1262f6a 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -437,6 +437,15 @@ def _parse_key_block(keyword, attribute, expected_block_type, value_attribute =
   return _parse
 
 
+def _copy(default):
+  if default is None or isinstance(default, (bool, stem.exit_policy.ExitPolicy)):
+    return default  # immutable
+  elif default in EMPTY_COLLECTION:
+    return type(default)()  # collection construction tad faster than copy
+  else:
+    return copy.copy(default)
+
+
 class Descriptor(object):
   """
   Common parent for all types of descriptors.
@@ -516,21 +525,6 @@ class Descriptor(object):
     if parser_for_line is None:
       parser_for_line = self.PARSER_FOR_LINE
 
-    # set defaults
-
-    for attr in self.ATTRIBUTES:
-      if not hasattr(self, attr):
-        value = self.ATTRIBUTES[attr][0]
-
-        if value is None or isinstance(value, (bool, stem.exit_policy.ExitPolicy)):
-          pass  # immutable
-        elif value in EMPTY_COLLECTION:
-          value = type(value)()  # collection construction tad faster than copy
-        else:
-          value = copy.copy(value)
-
-        setattr(self, attr, value)
-
     for keyword, values in list(entries.items()):
       try:
         if keyword in parser_for_line:
@@ -654,19 +648,25 @@ class Descriptor(object):
     if name in ('_lazy_loading', 'ATTRIBUTES'):
       return super(Descriptor, self).__getattribute__(name)
 
-    # If attribute isn't already present we might be lazy loading it...
+    # If an attribute we should have isn't present it means either...
+    #    
+    #   a. we still need to lazy load this
+    #   b. we read the whole descriptor but it wasn't present, so needs the default
 
-    if self._lazy_loading and name in self.ATTRIBUTES:
+    if name in self.ATTRIBUTES and not hasattr(self, name):
       default, parsing_function = self.ATTRIBUTES[name]
 
-      try:
-        parsing_function(self, self._entries)
-      except (ValueError, KeyError):
+      if self._lazy_loading:
         try:
-          # despite having a validation failure check to see if we set something
-          return super(Descriptor, self).__getattribute__(name)
-        except AttributeError:
-          setattr(self, name, copy.copy(default))
+          parsing_function(self, self._entries)
+        except (ValueError, KeyError):
+          try:
+            # despite having a validation failure check to see if we set something
+            return super(Descriptor, self).__getattribute__(name)
+          except AttributeError:
+            setattr(self, name, _copy(default))
+      else:
+        setattr(self, name, _copy(default))
 
     return super(Descriptor, self).__getattribute__(name)