[tor-commits] [nyx/master] Log size of cached consensus data

[atagar at torproject.org]

commit 6d8920436d9ee24f104f591bb48ff176cb7958dd
Author: Damian Johnson <atagar at torproject.org>
Date:   Sun Aug 20 15:36:12 2017 -0700

    Log size of cached consensus data
    
    Investigations into nyx memory usage indicate the consensus tracker is more
    than everything else combined. On initial load ~2.3 MB, and for some reason
    when update() is called that grows several fold (from 3% total memory usage
    to 10%). Still investigating, and there's some clear optimizations we can
    make, but first step is more data.
---
 nyx/tracker.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/nyx/tracker.py b/nyx/tracker.py
index 21d5b36..4ac76aa 100644
--- a/nyx/tracker.py
+++ b/nyx/tracker.py
@@ -52,6 +52,7 @@ Background tasks for gathering information about the tor process.
 
 import collections
 import os
+import sys
 import time
 import threading
 
@@ -817,6 +818,7 @@ class ConsensusTracker(object):
     # Stem's get_network_statuses() is slow, and overkill for what we need
     # here. Just parsing the raw GETINFO response to cut startup time down.
 
+    start_time = time.time()
     controller = tor_controller()
     ns_response = controller.get_info('ns/all', None)
 
@@ -834,6 +836,8 @@ class ConsensusTracker(object):
           self._address_cache[fingerprint] = (address, or_port)
           self._nickname_cache[fingerprint] = nickname
 
+      stem.util.log.info('Cached consensus data. Took %0.2fs. Cache size is %s for fingerprints, %s for addresses, and %s for nicknames' % (time.time() - start_time, stem.util.str_tools.size_label(sys.getsizeof(self._fingerprint_cache)), stem.util.str_tools.size_label(sys.getsizeof(self._address_cache)), stem.util.str_tools.size_label(sys.getsizeof(self._nickname_cache))))
+
     controller.add_event_listener(lambda event: self.update(event.desc), stem.control.EventType.NEWCONSENSUS)
 
   def update(self, router_status_entries):
@@ -846,6 +850,8 @@ class ConsensusTracker(object):
     new_fingerprint_cache = {}
     new_address_cache = {}
     new_nickname_cache = {}
+
+    start_time = time.time()
     our_fingerprint = tor_controller().get_info('fingerprint', None)
 
     for desc in router_status_entries:
@@ -861,6 +867,8 @@ class ConsensusTracker(object):
     self._address_cache = new_address_cache
     self._nickname_cache = new_nickname_cache
 
+    stem.util.log.info('Updated consensus cache. Took %0.2fs. Cache size is %s for fingerprints, %s for addresses, and %s for nicknames' % (time.time() - start_time, stem.util.str_tools.size_label(sys.getsizeof(self._fingerprint_cache)), stem.util.str_tools.size_label(sys.getsizeof(self._address_cache)), stem.util.str_tools.size_label(sys.getsizeof(self._nickname_cache))))
+
   def my_router_status_entry(self):
     """
     Provides the router status entry of ourselves. Descriptors are published