commit 3fd8114d0eb5412adcd26d13b22282c2c63c45a4 Author: Damian Johnson atagar@torproject.org Date: Sat Sep 2 18:08:25 2017 -0700
Don't call 'GETINFO ns/all' if cache is recent
If it's been less than an hour there's no point in making this GETINFO call. Hopefully this will speed startup a bit. --- nyx/__init__.py | 14 ++++++++++++++ nyx/tracker.py | 36 ++++++++++++++++++++++-------------- test/cache.py | 22 +++++++++++++++++++++- 3 files changed, 57 insertions(+), 15 deletions(-)
diff --git a/nyx/__init__.py b/nyx/__init__.py index 2a5cf57..2239956 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -114,6 +114,9 @@ SCHEMA = ( 'CREATE TABLE schema(version INTEGER)', 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION,
+ 'CREATE TABLE metadata(relays_updated_at REAL)', + 'INSERT INTO metadata(relays_updated_at) VALUES (0.0)', + 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port INTEGER, nickname TEXT)', 'CREATE INDEX addresses ON relays(address)', ) @@ -442,6 +445,16 @@ class Cache(object): result = self._query('SELECT address, or_port FROM relays WHERE fingerprint=?', fingerprint).fetchone() return result if result else default
+ def relays_updated_at(self): + """ + Provides the unix timestamp when relay information was last updated. + + :returns: **float** with the unix timestamp when relay information was last + updated, zero if it has never been set + """ + + return self._query('SELECT relays_updated_at FROM metadata').fetchone()[0] + def _query(self, query, *param): """ Performs a query on our cache. @@ -477,6 +490,7 @@ class CacheWriter(object): raise ValueError("'%s' isn't a valid port" % or_port)
self._cache._query('INSERT OR REPLACE INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', fingerprint, address, or_port, nickname) + self._cache._query('UPDATE metadata SET relays_updated_at=?', time.time())
class Interface(object): diff --git a/nyx/tracker.py b/nyx/tracker.py index 2530bf2..7a16817 100644 --- a/nyx/tracker.py +++ b/nyx/tracker.py @@ -813,25 +813,33 @@ class ConsensusTracker(object):
# Stem's get_network_statuses() is slow, and overkill for what we need # here. Just parsing the raw GETINFO response to cut startup time down. + # + # Only fetching this if our cache is at least an hour old (and hence a new + # consensus available).
- start_time = time.time() - controller = tor_controller() - ns_response = controller.get_info('ns/all', None) + cache_age = time.time() - nyx.cache().relays_updated_at() + + if cache_age < 3600: + stem.util.log.info('Cached is only %i seconds old, no need to refresh it.' % cache_age) + else: + start_time = time.time() + controller = tor_controller() + ns_response = controller.get_info('ns/all', None)
- if ns_response: - with nyx.cache().write() as writer: - for line in ns_response.splitlines(): - if line.startswith('r '): - r_comp = line.split(' ') + if ns_response: + with nyx.cache().write() as writer: + for line in ns_response.splitlines(): + if line.startswith('r '): + r_comp = line.split(' ')
- address = r_comp[6] - or_port = int(r_comp[7]) - fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) - nickname = r_comp[1] + address = r_comp[6] + or_port = int(r_comp[7]) + fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) + nickname = r_comp[1]
- writer.record_relay(fingerprint, address, or_port, nickname) + writer.record_relay(fingerprint, address, or_port, nickname)
- stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time)) + stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time))
controller.add_event_listener(lambda event: self.update(event.desc), stem.control.EventType.NEWCONSENSUS)
diff --git a/test/cache.py b/test/cache.py index 281ad86..4c93c75 100644 --- a/test/cache.py +++ b/test/cache.py @@ -4,6 +4,7 @@ Unit tests for nyx.cache.
import re import tempfile +import time import unittest
import nyx @@ -49,7 +50,7 @@ class TestCache(unittest.TestCase): @patch('nyx.data_directory', Mock(return_value = None)) def test_relays_for_address(self): """ - Basic checks for registering and fetching nicknames. + Basic checks for fetching relays by their address. """
cache = nyx.cache() @@ -103,6 +104,25 @@ class TestCache(unittest.TestCase): self.assertEqual(None, cache.relay_address('66E1D8F00C49820FE8AA26003EC49B6F069E8AE3'))
@patch('nyx.data_directory', Mock(return_value = None)) + def test_relays_updated_at(self): + """ + Basic checks for getting when relay information was last updated. + """ + + before = time.time() + time.sleep(0.01) + + cache = nyx.cache() + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi1') + + time.sleep(0.01) + after = time.time() + + self.assertTrue(before < cache.relays_updated_at() < after) + + @patch('nyx.data_directory', Mock(return_value = None)) def test_record_relay_when_updating(self): cache = nyx.cache()