tor-commits September 2017

tor-commits@lists.torproject.org

16 participants
2950 discussions

[nyx/master] Allow cache to be accessed from multiple threads
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 851159f02ed7cad6db5ead4d99cdd4d6b1f0ce3b Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 13:53:35 2017 -0700 Allow cache to be accessed from multiple threads Huh, didn't expect it to reject multithreading by default. Using the cache fails with... ProgrammingError: SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 140437876885248 and this is thread id 140437570447104 SQLite has a parameter to drop this check. Should be safe since we do the locking ourselves. There's discussion about the option on... https://bugs.python.org/issue27113 --- nyx/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index 26b490d..b3cbafb 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -362,7 +362,7 @@ class Cache(object): if cache_path: try: - self._conn = sqlite3.connect(cache_path) + self._conn = sqlite3.connect(cache_path, check_same_thread = False) schema = self._conn.execute('SELECT version FROM schema').fetchone()[0] except: schema = None @@ -377,13 +377,13 @@ class Cache(object): self._conn.close() os.remove(cache_path) - self._conn = sqlite3.connect(cache_path) + self._conn = sqlite3.connect(cache_path, check_same_thread = False) for cmd in SCHEMA: self._conn.execute(cmd) else: stem.util.log.info('Unable to cache to disk. Using an in-memory cache instead.') - self._conn = sqlite3.connect(':memory:') + self._conn = sqlite3.connect(':memory:', check_same_thread = False) for cmd in SCHEMA: self._conn.execute(cmd)

1 0

[nyx/master] Don't call 'GETINFO ns/all' if cache is recent
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 3fd8114d0eb5412adcd26d13b22282c2c63c45a4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 18:08:25 2017 -0700 Don't call 'GETINFO ns/all' if cache is recent If it's been less than an hour there's no point in making this GETINFO call. Hopefully this will speed startup a bit. --- nyx/__init__.py | 14 ++++++++++++++ nyx/tracker.py | 36 ++++++++++++++++++++++-------------- test/cache.py | 22 +++++++++++++++++++++- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index 2a5cf57..2239956 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -114,6 +114,9 @@ SCHEMA = ( 'CREATE TABLE schema(version INTEGER)', 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, + 'CREATE TABLE metadata(relays_updated_at REAL)', + 'INSERT INTO metadata(relays_updated_at) VALUES (0.0)', + 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port INTEGER, nickname TEXT)', 'CREATE INDEX addresses ON relays(address)', ) @@ -442,6 +445,16 @@ class Cache(object): result = self._query('SELECT address, or_port FROM relays WHERE fingerprint=?', fingerprint).fetchone() return result if result else default + def relays_updated_at(self): + """ + Provides the unix timestamp when relay information was last updated. + + :returns: **float** with the unix timestamp when relay information was last + updated, zero if it has never been set + """ + + return self._query('SELECT relays_updated_at FROM metadata').fetchone()[0] + def _query(self, query, *param): """ Performs a query on our cache. @@ -477,6 +490,7 @@ class CacheWriter(object): raise ValueError("'%s' isn't a valid port" % or_port) self._cache._query('INSERT OR REPLACE INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', fingerprint, address, or_port, nickname) + self._cache._query('UPDATE metadata SET relays_updated_at=?', time.time()) class Interface(object): diff --git a/nyx/tracker.py b/nyx/tracker.py index 2530bf2..7a16817 100644 --- a/nyx/tracker.py +++ b/nyx/tracker.py @@ -813,25 +813,33 @@ class ConsensusTracker(object): # Stem's get_network_statuses() is slow, and overkill for what we need # here. Just parsing the raw GETINFO response to cut startup time down. + # + # Only fetching this if our cache is at least an hour old (and hence a new + # consensus available). - start_time = time.time() - controller = tor_controller() - ns_response = controller.get_info('ns/all', None) + cache_age = time.time() - nyx.cache().relays_updated_at() + + if cache_age < 3600: + stem.util.log.info('Cached is only %i seconds old, no need to refresh it.' % cache_age) + else: + start_time = time.time() + controller = tor_controller() + ns_response = controller.get_info('ns/all', None) - if ns_response: - with nyx.cache().write() as writer: - for line in ns_response.splitlines(): - if line.startswith('r '): - r_comp = line.split(' ') + if ns_response: + with nyx.cache().write() as writer: + for line in ns_response.splitlines(): + if line.startswith('r '): + r_comp = line.split(' ') - address = r_comp[6] - or_port = int(r_comp[7]) - fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) - nickname = r_comp[1] + address = r_comp[6] + or_port = int(r_comp[7]) + fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) + nickname = r_comp[1] - writer.record_relay(fingerprint, address, or_port, nickname) + writer.record_relay(fingerprint, address, or_port, nickname) - stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time)) + stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time)) controller.add_event_listener(lambda event: self.update(event.desc), stem.control.EventType.NEWCONSENSUS) diff --git a/test/cache.py b/test/cache.py index 281ad86..4c93c75 100644 --- a/test/cache.py +++ b/test/cache.py @@ -4,6 +4,7 @@ Unit tests for nyx.cache. import re import tempfile +import time import unittest import nyx @@ -49,7 +50,7 @@ class TestCache(unittest.TestCase): @patch('nyx.data_directory', Mock(return_value = None)) def test_relays_for_address(self): """ - Basic checks for registering and fetching nicknames. + Basic checks for fetching relays by their address. """ cache = nyx.cache() @@ -103,6 +104,25 @@ class TestCache(unittest.TestCase): self.assertEqual(None, cache.relay_address('66E1D8F00C49820FE8AA26003EC49B6F069E8AE3')) @patch('nyx.data_directory', Mock(return_value = None)) + def test_relays_updated_at(self): + """ + Basic checks for getting when relay information was last updated. + """ + + before = time.time() + time.sleep(0.01) + + cache = nyx.cache() + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi1') + + time.sleep(0.01) + after = time.time() + + self.assertTrue(before < cache.relays_updated_at() < after) + + @patch('nyx.data_directory', Mock(return_value = None)) def test_record_relay_when_updating(self): cache = nyx.cache()

1 0

[nyx/master] Implement relay cache
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 8ea32382cc467202707c0161d05b271846978669 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Sep 1 11:39:06 2017 -0700 Implement relay cache There, that's more like what I was after. Adding methods to read and write relay data to the cache along with tests. --- nyx/__init__.py | 72 +++++++++++++++++++++++++++++++++++++++++++++-- test/cache.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 144 insertions(+), 15 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index 3c3c939..39c1e5d 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -31,6 +31,7 @@ Tor curses monitoring application. +- halt - stops daemon panels """ +import contextlib import distutils.spawn import os import sqlite3 @@ -42,8 +43,10 @@ import stem import stem.connection import stem.control import stem.util.conf +import stem.util.connection import stem.util.log import stem.util.system +import stem.util.tor_tools __version__ = '1.4.6-dev' __release_date__ = 'April 28, 2011' @@ -339,7 +342,8 @@ def join(entries, joiner = ' ', size = None): class Cache(object): """ - Cache for frequently used information. + Cache for frequently needed information. This persists to disk if we can, and + otherwise is an in-memory cache. """ def __init__(self): @@ -374,7 +378,43 @@ class Cache(object): for cmd in SCHEMA: self._conn.execute(cmd) - def query(self, query, *param): + @contextlib.contextmanager + def write(self): + """ + Provides a context in which we can modify the cache. + + :returns: :class:`~nyx.CacheWriter` that can modify the cache + """ + + with self._conn: + yield CacheWriter(self) + + def relay_nickname(self, fingerprint): + """ + Provides the nickname associated with the given relay. + + :param str fingerprint: relay to look up + + :returns: **str** with the nickname ("Unnamed" if unset), and **None** if + no such relay exists + """ + + result = self._query('SELECT nickname FROM relays WHERE fingerprint=?', fingerprint).fetchone() + return result[0] if result else None + + def relay_address(self, fingerprint): + """ + Provides the (address, port) tuple where a relay is running. + + :param str fingerprint: fingerprint to be checked + + :returns: **tuple** with a **str** address and **int** port + """ + + result = self._query('SELECT address, or_port FROM relays WHERE fingerprint=?', fingerprint).fetchone() + return result if result else None # TODO: does this raise if fingerprint doesn't exist? + + def _query(self, query, *param): """ Performs a query on our cache. """ @@ -383,6 +423,34 @@ class Cache(object): return self._conn.execute(query, param) +class CacheWriter(object): + def __init__(self, cache): + self._cache = cache + + def record_relay(self, fingerprint, address, or_port, nickname): + """ + Records relay metadata. + + :param str fingerprint: relay fingerprint + :param str address: ipv4 or ipv6 address + :param int or_port: ORPort of the relay + :param str nickname: relay nickname + + :raises: **ValueError** if provided data is malformed + """ + + if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): + raise ValueError("'%s' isn't a valid fingerprint" % fingerprint) + elif not stem.util.tor_tools.is_valid_nickname(nickname): + raise ValueError("'%s' isn't a valid nickname" % nickname) + elif not stem.util.connection.is_valid_ipv4_address(address) and not stem.util.connection.is_valid_ipv6_address(address): + raise ValueError("'%s' isn't a valid address" % address) + elif not stem.util.connection.is_valid_port(or_port): + raise ValueError("'%s' isn't a valid port" % or_port) + + self._cache._query('INSERT OR REPLACE INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', fingerprint, address, or_port, nickname) + + class Interface(object): """ Overall state of the nyx interface. diff --git a/test/cache.py b/test/cache.py index 91e9ab8..dd76c00 100644 --- a/test/cache.py +++ b/test/cache.py @@ -2,6 +2,7 @@ Unit tests for nyx.cache. """ +import re import tempfile import unittest @@ -9,11 +10,6 @@ import nyx from mock import Mock, patch -FINGERPRINT = '3EA8E960F6B94CE30062AA8EF02894C00F8D1E66' -ADDRESS = '208.113.165.162' -PORT = 1443 -NICKNAME = 'caersidi' - class TestCache(unittest.TestCase): def setUp(self): @@ -26,10 +22,12 @@ class TestCache(unittest.TestCase): """ cache = nyx.cache() + self.assertEqual((0, 'main', ''), cache._query('PRAGMA database_list').fetchone()) + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi') - self.assertEqual((0, 'main', ''), cache.query('PRAGMA database_list').fetchone()) - cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) - self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0]) + self.assertEqual('caersidi', cache.relay_nickname('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) def test_file_cache(self): """ @@ -39,12 +37,75 @@ class TestCache(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix = '.sqlite') as tmp: with patch('nyx.data_directory', Mock(return_value = tmp.name)): cache = nyx.cache() + self.assertEqual((0, 'main', tmp.name), cache._query('PRAGMA database_list').fetchone()) - self.assertEqual((0, 'main', tmp.name), cache.query('PRAGMA database_list').fetchone()) - cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) - cache._conn.commit() + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi') nyx.CACHE = None - cache = nyx.cache() - self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0]) + self.assertEqual('caersidi', cache.relay_nickname('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) + + @patch('nyx.data_directory', Mock(return_value = None)) + def test_relay_nickname(self): + """ + Basic checks for registering and fetching nicknames. + """ + + cache = nyx.cache() + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi') + writer.record_relay('9695DFC35FFEB861329B9F1AB04C46397020CE31', '128.31.0.34', 9101, 'moria1') + writer.record_relay('74A910646BCEEFBCD2E874FC1DC997430F968145', '199.254.238.53', 443, 'longclaw') + + self.assertEqual('moria1', cache.relay_nickname('9695DFC35FFEB861329B9F1AB04C46397020CE31')) + self.assertEqual('longclaw', cache.relay_nickname('74A910646BCEEFBCD2E874FC1DC997430F968145')) + self.assertEqual('caersidi', cache.relay_nickname('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) + + self.assertEqual(None, cache.relay_nickname('66E1D8F00C49820FE8AA26003EC49B6F069E8AE3')) + + @patch('nyx.data_directory', Mock(return_value = None)) + def test_relay_address(self): + """ + Basic checks for registering and fetching nicknames. + """ + + cache = nyx.cache() + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi') + writer.record_relay('9695DFC35FFEB861329B9F1AB04C46397020CE31', '128.31.0.34', 9101, 'moria1') + writer.record_relay('74A910646BCEEFBCD2E874FC1DC997430F968145', '199.254.238.53', 443, 'longclaw') + + self.assertEqual(('128.31.0.34', 9101), cache.relay_address('9695DFC35FFEB861329B9F1AB04C46397020CE31')) + self.assertEqual(('199.254.238.53', 443), cache.relay_address('74A910646BCEEFBCD2E874FC1DC997430F968145')) + self.assertEqual(('208.113.165.162', 1443), cache.relay_address('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) + + self.assertEqual(None, cache.relay_address('66E1D8F00C49820FE8AA26003EC49B6F069E8AE3')) + + @patch('nyx.data_directory', Mock(return_value = None)) + def test_record_relay_when_updating(self): + cache = nyx.cache() + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi') + + self.assertEqual('caersidi', cache.relay_nickname('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '128.31.0.34', 9101, 'moria1') + + self.assertEqual('moria1', cache.relay_nickname('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) + + @patch('nyx.data_directory', Mock(return_value = None)) + def test_record_relay_when_invalid(self): + """ + Provide malformed information to record_relay. + """ + + with nyx.cache().write() as writer: + self.assertRaisesRegexp(ValueError, re.escape("'blarg' isn't a valid fingerprint"), writer.record_relay, 'blarg', '208.113.165.162', 1443, 'caersidi') + self.assertRaisesRegexp(ValueError, re.escape("'blarg' isn't a valid address"), writer.record_relay, '3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', 'blarg', 1443, 'caersidi') + self.assertRaisesRegexp(ValueError, re.escape("'blarg' isn't a valid port"), writer.record_relay, '3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 'blarg', 'caersidi') + self.assertRaisesRegexp(ValueError, re.escape("'~blarg' isn't a valid nickname"), writer.record_relay, '3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, '~blarg')

1 0

[nyx/master] Drop 'unknown terminal' check
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit beead811f9cd98337a4510cdcbd5e7d6ccbdd9b4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 18:16:37 2017 -0700 Drop 'unknown terminal' check Just ran into this due to a regression on my part. The message doesn't give enough information to be useful and if it masks our own issues that's clearly a no-go. --- nyx/starter.py | 10 ---------- nyx/tracker.py | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/nyx/starter.py b/nyx/starter.py index 6aa98b7..077ca36 100644 --- a/nyx/starter.py +++ b/nyx/starter.py @@ -36,11 +36,6 @@ Nyx Configuration ({nyxrc_path}): -------------------------------------------------------------------------------- """.strip() -UNKNOWN_TERM = """\ -Unknown $TERM: (%s) -Either update your terminfo database or run nyx using "TERM=xterm nyx". -""" - @uses_settings def main(config): @@ -102,11 +97,6 @@ def main(config): try: nyx.curses.start(nyx.draw_loop, acs_support = config.get('acs_support', True), transparent_background = True, cursor = False) - except UnboundLocalError as exc: - if os.environ['TERM'] != 'xterm': - print(UNKNOWN_TERM % os.environ['TERM']) - else: - raise exc except KeyboardInterrupt: pass # skip printing a stack trace finally: diff --git a/nyx/tracker.py b/nyx/tracker.py index 7a16817..8630529 100644 --- a/nyx/tracker.py +++ b/nyx/tracker.py @@ -818,12 +818,12 @@ class ConsensusTracker(object): # consensus available). cache_age = time.time() - nyx.cache().relays_updated_at() + controller = tor_controller() if cache_age < 3600: stem.util.log.info('Cached is only %i seconds old, no need to refresh it.' % cache_age) else: start_time = time.time() - controller = tor_controller() ns_response = controller.get_info('ns/all', None) if ns_response:

1 0

[nyx/master] Use cache for resolving addresses to relays
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 451cda3f5456f3b3eaf84af1c86f6e618bcd0cae Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 17:53:15 2017 -0700 Use cache for resolving addresses to relays --- nyx/__init__.py | 23 ++++++++++++++++++++--- nyx/tracker.py | 10 +--------- test/cache.py | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index b3cbafb..2a5cf57 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -109,12 +109,13 @@ stem.control.CACHEABLE_GETINFO_PARAMS = list(stem.control.CACHEABLE_GETINFO_PARA stem.control.LOG_CACHE_FETCHES = False -SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following +SCHEMA_VERSION = 2 # version of our scheme, bump this if you change the following SCHEMA = ( - 'CREATE TABLE schema(version NUMBER)', + 'CREATE TABLE schema(version INTEGER)', 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, - 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port NUMBER, nickname TEXT)', + 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port INTEGER, nickname TEXT)', + 'CREATE INDEX addresses ON relays(address)', ) @@ -399,6 +400,22 @@ class Cache(object): with self._conn: yield CacheWriter(self) + def relays_for_address(self, address): + """ + Provides the relays running at a given location. + + :param str address: address to be checked + + :returns: **dict** of ORPorts to their fingerprint + """ + + result = {} + + for entry in self._query('SELECT or_port, fingerprint FROM relays WHERE address=?', address).fetchall(): + result[entry[0]] = entry[1] + + return result + def relay_nickname(self, fingerprint, default = None): """ Provides the nickname associated with the given relay. diff --git a/nyx/tracker.py b/nyx/tracker.py index cbfaedf..2530bf2 100644 --- a/nyx/tracker.py +++ b/nyx/tracker.py @@ -808,8 +808,6 @@ class ConsensusTracker(object): """ def __init__(self): - self._fingerprint_cache = {} # {address => [(port, fingerprint), ..]} for relays - self._my_router_status_entry = None self._my_router_status_entry_time = 0 @@ -831,7 +829,6 @@ class ConsensusTracker(object): fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) nickname = r_comp[1] - self._fingerprint_cache.setdefault(address, []).append((or_port, fingerprint)) writer.record_relay(fingerprint, address, or_port, nickname) stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time)) @@ -845,22 +842,17 @@ class ConsensusTracker(object): :param list router_status_entries: router status entries to populate our cache with """ - new_fingerprint_cache = {} - start_time = time.time() our_fingerprint = tor_controller().get_info('fingerprint', None) with nyx.cache().write() as writer: for desc in router_status_entries: - new_fingerprint_cache.setdefault(desc.address, []).append((desc.or_port, desc.fingerprint)) writer.record_relay(desc.fingerprint, desc.address, desc.or_port, desc.nickname) if desc.fingerprint == our_fingerprint: self._my_router_status_entry = desc self._my_router_status_entry_time = time.time() - self._fingerprint_cache = new_fingerprint_cache - stem.util.log.info('Updated consensus cache, took %0.2fs.' % (time.time() - start_time)) def my_router_status_entry(self): @@ -915,7 +907,7 @@ class ConsensusTracker(object): if fingerprint and ports: return dict([(port, fingerprint) for port in ports]) - return dict([(port, fp) for (port, fp) in self._fingerprint_cache.get(address, [])]) + return nyx.cache().relays_for_address(address) def get_relay_address(self, fingerprint, default): """ diff --git a/test/cache.py b/test/cache.py index dd76c00..281ad86 100644 --- a/test/cache.py +++ b/test/cache.py @@ -47,6 +47,24 @@ class TestCache(unittest.TestCase): self.assertEqual('caersidi', cache.relay_nickname('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66')) @patch('nyx.data_directory', Mock(return_value = None)) + def test_relays_for_address(self): + """ + Basic checks for registering and fetching nicknames. + """ + + cache = nyx.cache() + + with cache.write() as writer: + writer.record_relay('3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', '208.113.165.162', 1443, 'caersidi1') + writer.record_relay('9695DFC35FFEB861329B9F1AB04C46397020CE31', '128.31.0.34', 9101, 'moria1') + writer.record_relay('74A910646BCEEFBCD2E874FC1DC997430F968145', '208.113.165.162', 1543, 'caersidi2') + + self.assertEqual({9101: '9695DFC35FFEB861329B9F1AB04C46397020CE31'}, cache.relays_for_address('128.31.0.34')) + self.assertEqual({1443: '3EA8E960F6B94CE30062AA8EF02894C00F8D1E66', 1543: '74A910646BCEEFBCD2E874FC1DC997430F968145'}, cache.relays_for_address('208.113.165.162')) + + self.assertEqual({}, cache.relays_for_address('199.254.238.53')) + + @patch('nyx.data_directory', Mock(return_value = None)) def test_relay_nickname(self): """ Basic checks for registering and fetching nicknames.

1 0

[nyx/master] Move cache back into base module
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 6678e5021dd626afbccbf9aebb15819d40deae3f Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 31 12:07:28 2017 -0700 Move cache back into base module Well, that didn't last very long. Got annoyed by calling 'nyx.cache.cache()'. Oh well. --- nyx/__init__.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ nyx/cache.py | 83 --------------------------------------------------------- test/cache.py | 12 ++++----- 3 files changed, 78 insertions(+), 89 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index c2032bc..3c3c939 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -33,6 +33,7 @@ Tor curses monitoring application. import distutils.spawn import os +import sqlite3 import sys import threading import time @@ -84,6 +85,7 @@ CONFIG = stem.util.conf.config_dict('nyx', { NYX_INTERFACE = None TOR_CONTROLLER = None +CACHE = None BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-1]) # technically can change but we use this query a *lot* so needs to be cached @@ -94,6 +96,15 @@ stem.control.CACHEABLE_GETINFO_PARAMS = list(stem.control.CACHEABLE_GETINFO_PARA stem.control.LOG_CACHE_FETCHES = False +SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following +SCHEMA = ( + 'CREATE TABLE schema(version NUMBER)', + 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, + + 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port NUMBER, nickname TEXT)', +) + + try: uses_settings = stem.util.conf.uses_settings('nyx', os.path.join(BASE_DIR, 'settings'), lazy_load = False) except IOError as exc: @@ -189,6 +200,21 @@ def tor_controller(): return TOR_CONTROLLER +def cache(): + """ + Provides the sqlite cache for application data. + + :returns: :class:`~nyx.cache.Cache` for our applicaion + """ + + global CACHE + + if CACHE is None: + CACHE = Cache() + + return CACHE + + def show_message(message = None, *attr, **kwargs): """ Shows a message in our header. @@ -311,6 +337,52 @@ def join(entries, joiner = ' ', size = None): return result +class Cache(object): + """ + Cache for frequently used information. + """ + + def __init__(self): + self._conn_lock = threading.RLock() + cache_path = nyx.data_directory('cache.sqlite') + + if cache_path: + try: + self._conn = sqlite3.connect(cache_path) + schema = self._conn.execute('SELECT version FROM schema').fetchone()[0] + except: + schema = None + + if schema == SCHEMA_VERSION: + stem.util.log.info('Cache loaded from %s' % cache_path) + else: + if schema is None: + stem.util.log.info('Cache at %s is missing a schema, clearing it.' % cache_path) + else: + stem.util.log.info('Cache at %s has schema version %s but the current version is %s, clearing it.' % (cache_path, schema, SCHEMA_VERSION)) + + self._conn.close() + os.remove(cache_path) + self._conn = sqlite3.connect(cache_path) + + for cmd in SCHEMA: + self._conn.execute(cmd) + else: + stem.util.log.info('Unable to cache to disk. Using an in-memory cache instead.') + self._conn = sqlite3.connect(':memory:') + + for cmd in SCHEMA: + self._conn.execute(cmd) + + def query(self, query, *param): + """ + Performs a query on our cache. + """ + + with self._conn_lock: + return self._conn.execute(query, param) + + class Interface(object): """ Overall state of the nyx interface. diff --git a/nyx/cache.py b/nyx/cache.py deleted file mode 100644 index f4c6f3f..0000000 --- a/nyx/cache.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Cache for frequently needed information. This persists to disk if we can, and -otherwise is an in-memory cache. -""" - -import os -import sqlite3 -import threading - -import stem.util.log - -import nyx - -CACHE = None - -SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following -SCHEMA = ( - 'CREATE TABLE schema(version NUMBER)', - 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, - - 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port NUMBER, nickname TEXT)', -) - - -def cache(): - """ - Provides the sqlite cache for application data. - - :returns: :class:`~nyx.cache.Cache` for our applicaion - """ - - global CACHE - - if CACHE is None: - CACHE = Cache() - - return CACHE - - -class Cache(object): - """ - Cache for frequently used information. - """ - - def __init__(self): - self._conn_lock = threading.RLock() - cache_path = nyx.data_directory('cache.sqlite') - - if cache_path: - try: - self._conn = sqlite3.connect(cache_path) - schema = self._conn.execute('SELECT version FROM schema').fetchone()[0] - except: - schema = None - - if schema == SCHEMA_VERSION: - stem.util.log.info('Cache loaded from %s' % cache_path) - else: - if schema is None: - stem.util.log.info('Cache at %s is missing a schema, clearing it.' % cache_path) - else: - stem.util.log.info('Cache at %s has schema version %s but the current version is %s, clearing it.' % (cache_path, schema, SCHEMA_VERSION)) - - self._conn.close() - os.remove(cache_path) - self._conn = sqlite3.connect(cache_path) - - for cmd in SCHEMA: - self._conn.execute(cmd) - else: - stem.util.log.info('Unable to cache to disk. Using an in-memory cache instead.') - self._conn = sqlite3.connect(':memory:') - - for cmd in SCHEMA: - self._conn.execute(cmd) - - def query(self, query, *param): - """ - Performs a query on our cache. - """ - - with self._conn_lock: - return self._conn.execute(query, param) diff --git a/test/cache.py b/test/cache.py index f26ede9..91e9ab8 100644 --- a/test/cache.py +++ b/test/cache.py @@ -5,7 +5,7 @@ Unit tests for nyx.cache. import tempfile import unittest -import nyx.cache +import nyx from mock import Mock, patch @@ -17,7 +17,7 @@ NICKNAME = 'caersidi' class TestCache(unittest.TestCase): def setUp(self): - nyx.cache.CACHE = None # drop cached database reference + nyx.CACHE = None # drop cached database reference @patch('nyx.data_directory', Mock(return_value = None)) def test_memory_cache(self): @@ -25,7 +25,7 @@ class TestCache(unittest.TestCase): Create a cache in memory. """ - cache = nyx.cache.cache() + cache = nyx.cache() self.assertEqual((0, 'main', ''), cache.query('PRAGMA database_list').fetchone()) cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) @@ -38,13 +38,13 @@ class TestCache(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix = '.sqlite') as tmp: with patch('nyx.data_directory', Mock(return_value = tmp.name)): - cache = nyx.cache.cache() + cache = nyx.cache() self.assertEqual((0, 'main', tmp.name), cache.query('PRAGMA database_list').fetchone()) cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) cache._conn.commit() - nyx.cache.CACHE = None + nyx.CACHE = None - cache = nyx.cache.cache() + cache = nyx.cache() self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0])

1 0

[nyx/master] Use SQLite cache to reduce memory usage
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 40e25acad51daae38f9888699a9884c47148445e Merge: 7304dd2 beead81 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 18:21:52 2017 -0700 Use SQLite cache to reduce memory usage Creating a little SQLite cache to store lookup tables we presently cram into memory. On my relay this drops memory usage from 42.8 MB to 33.0 MB (23% lower). Hopefully we'll find some other good candidates for this too. nyx/__init__.py | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nyx/starter.py | 10 ---- nyx/tracker.py | 64 ++++++++++---------- test/cache.py | 149 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 364 insertions(+), 42 deletions(-)

1 0

[stem/master] Use INTEGER rather than NUMBER for sqlite type
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 6831e262f4a4832f7ff6cd56504a78c084a5ec8d Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 17:52:12 2017 -0700 Use INTEGER rather than NUMBER for sqlite type First tutorials I saw used NUMBER but seems INTEGER is the preferred type. --- stem/manual.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/manual.py b/stem/manual.py index 290688e6..18a5d34d 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -87,14 +87,14 @@ DATABASE = None # cache database connections SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following SCHEMA = ( - 'CREATE TABLE schema(version NUMBER)', + 'CREATE TABLE schema(version INTEGER)', 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, 'CREATE TABLE metadata(name TEXT, synopsis TEXT, description TEXT, man_commit TEXT, stem_commit TEXT)', 'CREATE TABLE commandline(name TEXT PRIMARY KEY, description TEXT)', 'CREATE TABLE signals(name TEXT PRIMARY KEY, description TEXT)', 'CREATE TABLE files(name TEXT PRIMARY KEY, description TEXT)', - 'CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position NUMBER)', + 'CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position INTEGER)', ) CATEGORY_SECTIONS = OrderedDict((

1 0

[translation-tools/master] Drop tails-greeter and _completed
by colin＠torproject.org 02 Sep '17

02 Sep '17

commit f7e9e9b94f3f882dc127a4ec8ccdf5acb047562b Author: Colin Childs <colin(a)torproject.org> Date: Sat Sep 2 18:43:23 2017 -0500 Drop tails-greeter and _completed --- config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index d2348ec..b5d375b 100644 --- a/config +++ b/config @@ -11,8 +11,7 @@ PROJECTS="bridgedb bridgedb_completed torcheck torcheck_completed whisperback whisperback_completed https_everywhere https_everywhere_completed liveusb-creator liveusb-creator_completed tails-persistence-setup -tails-persistence-setup_completed tails-greeter tails-greeter_completed -torbirdy torbirdy_completed mat-gui mat-gui_completed +tails-persistence-setup_completed torbirdy torbirdy_completed mat-gui mat-gui_completed tor-launcher-properties tor-launcher-properties_completed tor-launcher-progress tor-launcher-progress_completed tor-launcher-network-settings tor-launcher-network-settings_completed

1 0

[tor-messenger-build/esr52] Update changelog for 9accac5
by sukhbir＠torproject.org 02 Sep '17

02 Sep '17

commit 6c8777a36119ac795774d45e9bf63c0d1c9e22a4 Author: Arlo Breault <arlolra(a)gmail.com> Date: Fri Sep 1 19:57:03 2017 -0400 Update changelog for 9accac5 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 26e3181..0781e84 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,7 @@ Tor Messenger 0.5.0b1 -- * Trac 10942: Deterministic builds for Instantbird * Windows * Trac 17469: Tor Messenger is not working on Windows XP + * Trac 10942: Deterministic builds for Instantbird Tor Messenger 0.4.0b3 -- June 13, 2017 * All Platforms

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits September 2017