[tor-commits] [nyx/master] Don't call 'GETINFO ns/all' if cache is recent

[atagar at torproject.org]

commit 3fd8114d0eb5412adcd26d13b22282c2c63c45a4
Author: Damian Johnson <atagar at torproject.org>
Date:   Sat Sep 2 18:08:25 2017 -0700

    Don't call 'GETINFO ns/all' if cache is recent
    
    If it's been less than an hour there's no point in making this GETINFO call.
    Hopefully this will speed startup a bit.
---
 nyx/__init__.py | 14 ++++++++++++++
 nyx/tracker.py  | 36 ++++++++++++++++++++++--------------
 test/cache.py   | 22 +++++++++++++++++++++-
 3 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/nyx/__init__.py b/nyx/__init__.py
index 2a5cf57..2239956 100644
--- a/nyx/__init__.py
+++ b/nyx/__init__.py
@@ -114,6 +114,9 @@ SCHEMA = (
   'CREATE TABLE schema(version INTEGER)',
   'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION,
 
+  'CREATE TABLE metadata(relays_updated_at REAL)',
+  'INSERT INTO metadata(relays_updated_at) VALUES (0.0)',
+
   'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port INTEGER, nickname TEXT)',
   'CREATE INDEX addresses ON relays(address)',
 )
@@ -442,6 +445,16 @@ class Cache(object):
     result = self._query('SELECT address, or_port FROM relays WHERE fingerprint=?', fingerprint).fetchone()
     return result if result else default
 
+  def relays_updated_at(self):
+    """
+    Provides the unix timestamp when relay information was last updated.
+
+    :returns: **float** with the unix timestamp when relay information was last
+      updated, zero if it has never been set
+    """
+
+    return self._query('SELECT relays_updated_at FROM metadata').fetchone()[0]
+
   def _query(self, query, *param):
     """
     Performs a query on our cache.
@@ -477,6 +490,7 @@ class CacheWriter(object):
       raise ValueError("'%s' isn't a valid port" % or_port)
 
     self._cache._query('INSERT OR REPLACE INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', fingerprint, address, or_port, nickname)
+    self._cache._query('UPDATE metadata SET relays_updated_at=?', time.time())
 
 
 class Interface(object):
diff --git a/nyx/tracker.py b/nyx/tracker.py
index 2530bf2..7a16817 100644
--- a/nyx/tracker.py
+++ b/nyx/tracker.py
@@ -813,25 +813,33 @@ class ConsensusTracker(object):
 
     # Stem's get_network_statuses() is slow, and overkill for what we need
     # here. Just parsing the raw GETINFO response to cut startup time down.
+    #
+    # Only fetching this if our cache is at least an hour old (and hence a new
+    # consensus available).
 
-    start_time = time.time()
-    controller = tor_controller()
-    ns_response = controller.get_info('ns/all', None)
+    cache_age = time.time() - nyx.cache().relays_updated_at()
+
+    if cache_age < 3600:
+      stem.util.log.info('Cached is only %i seconds old, no need to refresh it.' % cache_age)
+    else:
+      start_time = time.time()
+      controller = tor_controller()
+      ns_response = controller.get_info('ns/all', None)
 
-    if ns_response:
-      with nyx.cache().write() as writer:
-        for line in ns_response.splitlines():
-          if line.startswith('r '):
-            r_comp = line.split(' ')
+      if ns_response:
+        with nyx.cache().write() as writer:
+          for line in ns_response.splitlines():
+            if line.startswith('r '):
+              r_comp = line.split(' ')
 
-            address = r_comp[6]
-            or_port = int(r_comp[7])
-            fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2])
-            nickname = r_comp[1]
+              address = r_comp[6]
+              or_port = int(r_comp[7])
+              fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2])
+              nickname = r_comp[1]
 
-            writer.record_relay(fingerprint, address, or_port, nickname)
+              writer.record_relay(fingerprint, address, or_port, nickname)
 
-      stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time))
+        stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time))
 
     controller.add_event_listener(lambda event: self.update(event.desc), stem.control.EventType.NEWCONSENSUS)
 
diff --git a/test/cache.py b/test/cache.py
index 281ad86..4c93c75 100644
--- a/test/cache.py
+++ b/test/cache.py
@@ -4,6 +4,7 @@ Unit tests for nyx.cache.
 
 import re
 import tempfile
+import time
 import unittest
 
 import nyx
@@ -49,7 +50,7 @@ class TestCache(unittest.TestCase):
   @patch('nyx.data_directory', Mock(return_value = None))
   def test_relays_for_address(self):
     """
-    Basic checks for registering and fetching nicknames.
+    Basic checks for fetching relays by their address.
     """
 
     cache = nyx.cache()
@@ -103,6 +104,25 @@ class TestCache(unittest.TestCase):
     self.assertEqual(None, cache.relay_address('66E1D8F00C49820FE8AA26003EC49B6F069E8AE3'))
 
   @patch('nyx.data_directory', Mock(return_value = None))
+  def test_relays_updated_at(self):
+    """
+    Basic checks for getting when relay information was last updated.
+    """
+
+    before = time.time()
+    time.sleep(0.01)
+
+    cache = nyx.cache()
+
+    with cache.write() as writer:
+      writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi1')
+
+    time.sleep(0.01)
+    after = time.time()
+
+    self.assertTrue(before < cache.relays_updated_at() < after)
+
+  @patch('nyx.data_directory', Mock(return_value = None))
   def test_record_relay_when_updating(self):
     cache = nyx.cache()