September 2017 - tor-commits - lists.torproject.org

[tor-messenger-build/master] Compile Linux builds with Selfrando
by sukhbir＠torproject.org 03 Sep '17

03 Sep '17

commit f63b980484f55bdf0bab86720587077b4474b679 Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Sun Sep 3 12:41:38 2017 -0400 Compile Linux builds with Selfrando Changes borrowed from tor-browser-build.git --- keyring/elfutils.gpg | 1 + keyring/selfrando.gpg | 1 + projects/elfutils | 1 + projects/instantbird/build | 8 ++++++++ projects/instantbird/config | 3 +++ projects/selfrando | 1 + tor-browser-build | 2 +- 7 files changed, 16 insertions(+), 1 deletion(-) diff --git a/keyring/elfutils.gpg b/keyring/elfutils.gpg new file mode 120000 index 0000000..6e82beb --- /dev/null +++ b/keyring/elfutils.gpg @@ -0,0 +1 @@ +../tor-browser-build/keyring/elfutils.gpg \ No newline at end of file diff --git a/keyring/selfrando.gpg b/keyring/selfrando.gpg new file mode 120000 index 0000000..fee6b9c --- /dev/null +++ b/keyring/selfrando.gpg @@ -0,0 +1 @@ +../tor-browser-build/keyring/selfrando.gpg \ No newline at end of file diff --git a/projects/elfutils b/projects/elfutils new file mode 120000 index 0000000..b4beda5 --- /dev/null +++ b/projects/elfutils @@ -0,0 +1 @@ +../tor-browser-build/projects/elfutils \ No newline at end of file diff --git a/projects/instantbird/build b/projects/instantbird/build index 7813fe2..d9cbdb0 100644 --- a/projects/instantbird/build +++ b/projects/instantbird/build @@ -28,6 +28,14 @@ mkdir -p /var/tmp/build export PATH="/var/tmp/dist/binutils/bin:$PATH" [% END -%] +[% IF c("var/linux") -%] + tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/selfrando') %] + # Selfrando wrapper + export PATH="/var/tmp/dist/selfrando/Tools/TorBrowser/tc-wrapper/:$PATH" + # We need to avoid the shuffling while building as this breaks compilation + export SELFRANDO_skip_shuffle= +[% END -%] + tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz [% IF c("var/osx") %] diff --git a/projects/instantbird/config b/projects/instantbird/config index ed09aa0..1d89837 100644 --- a/projects/instantbird/config +++ b/projects/instantbird/config @@ -83,6 +83,9 @@ input_files: - name: binutils project: binutils enable: '[% c("var/linux") %]' + - project: selfrando + name: selfrando + enable: '[% c("var/linux") %]' - project: gcc name: gcc enable: '[% c("var/windows") %]' diff --git a/projects/selfrando b/projects/selfrando new file mode 120000 index 0000000..c7e52e1 --- /dev/null +++ b/projects/selfrando @@ -0,0 +1 @@ +../tor-browser-build/projects/selfrando/ \ No newline at end of file diff --git a/tor-browser-build b/tor-browser-build index 86210c6..da7485a 160000 --- a/tor-browser-build +++ b/tor-browser-build @@ -1 +1 @@ -Subproject commit 86210c6821e393e98dd58c7d7bb6223755b496d5 +Subproject commit da7485a0b8f89b9ee2278d6132fc7bb53d5220e8

1 0

[translation/torbutton-torbuttondtd] Update translations for torbutton-torbuttondtd
by translation＠torproject.org 03 Sep '17

03 Sep '17

commit 8835422ad15fbe3d0974dc9884570f4264742931 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Sep 3 03:47:14 2017 +0000 Update translations for torbutton-torbuttondtd --- bg/torbutton.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bg/torbutton.dtd b/bg/torbutton.dtd index 176e1bcb4..e7801963a 100644 --- a/bg/torbutton.dtd +++ b/bg/torbutton.dtd @@ -30,7 +30,7 @@ <!ENTITY torbutton.prefs.restrict_thirdparty_tooltip "Отметнете тази кутийка, за да попречите на различни браузър функции, които биха се възползвали от тях, за да следят как сърфирате в интернет. Тези функции включват blоб URL-и, broadcast канали, кеша на браузъра, бисквити, фавикони, HTTP Аuth headers, предварително следване на линкове, localStorate, mediaStorage URL-и, OCSP зявки, SharedWorkers и сесийни TSL билети."> <!ENTITY torbutton.prefs.resist_fingerprinting "Change details that distinguish you from other Tor Browser users"> <!ENTITY torbutton.prefs.resist_fingerprinting.accesskey "F"> -<!ENTITY torbutton.prefs.resist_fingerprinting_tooltip "Keep this box checked to hide things from websites that could be unique about you, including your computer performance, keyboard layout, locale, the location of installed plugins, the list of installed plugins, your network status, screen orientation, screen size, site-specific zoom levels, supported file types, system colors, and WebGL capabilities."> +<!ENTITY torbutton.prefs.resist_fingerprinting_tooltip "Изберете тази кутийка, за да скриете неща от уебсайтовете, които са уникални за вас, включително натоварването на компютъра, калвишното разположение на клавиатурата, език, местоположение на инсталираните браузърни добавки, списъка с браузърни добавки, мрежово състояние, ориентация на екрана, размер на екрана, ниво на увеличение на текста за отделните уебсайтове, поддържани типове файлове, системни цветове, и WebGL възможности."> <!ENTITY torbutton.prefs.sec_caption "Ниво на защита"> <!ENTITY torbutton.prefs.sec_caption_tooltip "Плъзгача по сигурността разрешава да изключите различни функции на браузъра, които правят браузъра по податлив на опити за хакване."> <!ENTITY torbutton.prefs.sec_low "Ниско (по подразбиране)">

1 0

[translation/https_everywhere_completed] Update translations for https_everywhere_completed
by translation＠torproject.org 03 Sep '17

03 Sep '17

commit a61b1ca02c8a7a557b17b9280b8e298416442f79 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Sep 3 03:45:40 2017 +0000 Update translations for https_everywhere_completed --- bg/https-everywhere.dtd | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bg/https-everywhere.dtd b/bg/https-everywhere.dtd index f37031f65..3482593fa 100644 --- a/bg/https-everywhere.dtd +++ b/bg/https-everywhere.dtd @@ -24,9 +24,22 @@ <!ENTITY https-everywhere.menu.showCounter "Показване на брояч"> <!ENTITY https-everywhere.menu.viewAllRules "Прегледай всички правила"> +<!ENTITY https-everywhere.options.importSettings "Внасяне на настройки"> +<!ENTITY https-everywhere.options.import "Внасяне"> +<!ENTITY https-everywhere.options.imported "Настройките са внесени."> + <!ENTITY https-everywhere.prefs.title "Настройки на HTTPS Everywhere"> <!ENTITY https-everywhere.prefs.enable_all "Включване на всички"> <!ENTITY https-everywhere.prefs.disable_all "Изключване на всички"> +<!ENTITY https-everywhere.prefs.export_settings "Изнасяне на настройки"> +<!ENTITY https-everywhere.prefs.settings_warning "Внимание:"> +<!ENTITY https-everywhere.prefs.settings_warning_delete "Някои от HTTPS Everywhere настройките ще бъдат изтрити!"> +<!ENTITY https-everywhere.prefs.settings_warning_explain "Тази добавка ще бъде обновена изцяло в скоро време. Ако не изнесете настройките си, може да изгубите едно или повече от следните:"> +<!ENTITY https-everywhere.prefs.settings_global "Глобални настройки"> +<!ENTITY https-everywhere.prefs.settings_toggle "Вкл/Изкл настройки за всеки сайт"> +<!ENTITY https-everywhere.prefs.settings_custom "Лични правила за сайтове"> +<!ENTITY https-everywhere.prefs.settings_click_ok "Моля натиснете 'ОК'"> +<!ENTITY https-everywhere.prefs.settings_warning_closing "по-долу, за да се уверите, че настройките са запазени. След като HTTPS Everywhere се обнови, ще трябва да внесете настройките в секцията за настройки на добавките."> <!ENTITY https-everywhere.prefs.reset_defaults "Възстановяване на стандартните"> <!ENTITY https-everywhere.prefs.search "Търсене"> <!ENTITY https-everywhere.prefs.site "Страница">

1 0

[translation/https_everywhere] Update translations for https_everywhere
by translation＠torproject.org 03 Sep '17

03 Sep '17

commit a9266f3110379ae02277b6ab2ec3b038c816ca0b Author: Translation commit bot <translation(a)torproject.org> Date: Sun Sep 3 03:45:31 2017 +0000 Update translations for https_everywhere --- bg/https-everywhere.dtd | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/bg/https-everywhere.dtd b/bg/https-everywhere.dtd index a694dcf29..3482593fa 100644 --- a/bg/https-everywhere.dtd +++ b/bg/https-everywhere.dtd @@ -24,22 +24,22 @@ <!ENTITY https-everywhere.menu.showCounter "Показване на брояч"> <!ENTITY https-everywhere.menu.viewAllRules "Прегледай всички правила"> -<!ENTITY https-everywhere.options.importSettings "Import Settings"> -<!ENTITY https-everywhere.options.import "Import"> -<!ENTITY https-everywhere.options.imported "Your settings have been imported."> +<!ENTITY https-everywhere.options.importSettings "Внасяне на настройки"> +<!ENTITY https-everywhere.options.import "Внасяне"> +<!ENTITY https-everywhere.options.imported "Настройките са внесени."> <!ENTITY https-everywhere.prefs.title "Настройки на HTTPS Everywhere"> <!ENTITY https-everywhere.prefs.enable_all "Включване на всички"> <!ENTITY https-everywhere.prefs.disable_all "Изключване на всички"> -<!ENTITY https-everywhere.prefs.export_settings "Export Settings"> -<!ENTITY https-everywhere.prefs.settings_warning "Warning:"> -<!ENTITY https-everywhere.prefs.settings_warning_delete "Some of your HTTPS Everywhere settings are about to be deleted!"> -<!ENTITY https-everywhere.prefs.settings_warning_explain "This extension will be upgraded substantially in the near future. If you do not export your settings, you could lose one or more of the following:"> -<!ENTITY https-everywhere.prefs.settings_global "Global preferences"> -<!ENTITY https-everywhere.prefs.settings_toggle "Per-site on/off setting"> -<!ENTITY https-everywhere.prefs.settings_custom "Custom site rules"> -<!ENTITY https-everywhere.prefs.settings_click_ok "Please click 'OK'"> -<!ENTITY https-everywhere.prefs.settings_warning_closing "below to ensure your settings are saved. Once HTTPS Everywhere is upgraded, you will later have to import these settings in the extension options section."> +<!ENTITY https-everywhere.prefs.export_settings "Изнасяне на настройки"> +<!ENTITY https-everywhere.prefs.settings_warning "Внимание:"> +<!ENTITY https-everywhere.prefs.settings_warning_delete "Някои от HTTPS Everywhere настройките ще бъдат изтрити!"> +<!ENTITY https-everywhere.prefs.settings_warning_explain "Тази добавка ще бъде обновена изцяло в скоро време. Ако не изнесете настройките си, може да изгубите едно или повече от следните:"> +<!ENTITY https-everywhere.prefs.settings_global "Глобални настройки"> +<!ENTITY https-everywhere.prefs.settings_toggle "Вкл/Изкл настройки за всеки сайт"> +<!ENTITY https-everywhere.prefs.settings_custom "Лични правила за сайтове"> +<!ENTITY https-everywhere.prefs.settings_click_ok "Моля натиснете 'ОК'"> +<!ENTITY https-everywhere.prefs.settings_warning_closing "по-долу, за да се уверите, че настройките са запазени. След като HTTPS Everywhere се обнови, ще трябва да внесете настройките в секцията за настройки на добавките."> <!ENTITY https-everywhere.prefs.reset_defaults "Възстановяване на стандартните"> <!ENTITY https-everywhere.prefs.search "Търсене"> <!ENTITY https-everywhere.prefs.site "Страница">

1 0

[nyx/master] Schema versioning and cache thread safety
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 3b9786e3d22b938e7c25fc5a40179208707fd68b Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Aug 30 10:32:44 2017 -0700 Schema versioning and cache thread safety Versioning nyx's cache schema and ensure use of the cache is done in a thread safe fashion. --- nyx/__init__.py | 40 ++++++++++++++++++++++++++++++++++++---- test/cache.py | 31 ++++++++++++++++--------------- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index c7db0cb..ee146a6 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -31,6 +31,7 @@ Tor curses monitoring application. +- halt - stops daemon panels """ +import contextlib import distutils.spawn import os import sqlite3 @@ -86,6 +87,14 @@ NYX_INTERFACE = None TOR_CONTROLLER = None BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-1]) CACHE = None +CACHE_LOCK = threading.RLock() + +SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following +SCHEMA = ( + 'CREATE TABLE schema(version NUMBER)', + 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, +) + # technically can change but we use this query a *lot* so needs to be cached @@ -252,6 +261,7 @@ def data_directory(filename, config): return os.path.join(data_dir, filename) +(a)contextlib.contextmanager def cache(): """ Provides the sqlite cache for application data. @@ -261,11 +271,33 @@ def cache(): global CACHE - if CACHE is None: - cache_path = data_directory('cache.sqlite') - CACHE = sqlite3.connect(cache_path if cache_path else ':memory:') + with CACHE_LOCK: + if CACHE is None: + cache_path = data_directory('cache.sqlite') + + if cache_path: + try: + CACHE = sqlite3.connect(cache_path) + schema = CACHE.execute('SELECT version FROM schema').fetchone()[0] + except: + schema = 'no schema' + + if schema != SCHEMA_VERSION: + stem.util.log.info('Cache schema of %s is out of date (has %s but current version is %s). Clearing the cache.' % (cache_path, schema, SCHEMA_VERSION)) + + CACHE.close() + os.remove(cache_path) + CACHE = sqlite3.connect(cache_path) + + for cmd in SCHEMA: + CACHE.execute(cmd) + else: + CACHE = sqlite3.connect(':memory:') + + for cmd in SCHEMA: + CACHE.execute(cmd) - return CACHE + yield CACHE @uses_settings diff --git a/test/cache.py b/test/cache.py index dc1fa40..1c30f40 100644 --- a/test/cache.py +++ b/test/cache.py @@ -16,25 +16,26 @@ class TestCache(unittest.TestCase): @patch('nyx.data_directory', Mock(return_value = None)) def test_memory_cache(self): - cache = nyx.cache() - self.assertEqual((0, 'main', ''), cache.execute("PRAGMA database_list").fetchone()) + with nyx.cache() as cache: + self.assertEqual((0, 'main', ''), cache.execute("PRAGMA database_list").fetchone()) - cache.execute('CREATE TABLE aliases(alias TEXT, command TEXT)') - cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('l', 'ls -xF --color=auto')) - cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('ll', 'ls -hlA --color=auto')) - self.assertEqual('ls -hlA --color=auto', cache.execute('SELECT command FROM aliases WHERE alias=?', ('ll',)).fetchone()[0]) + cache.execute('CREATE TABLE aliases(alias TEXT, command TEXT)') + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('l', 'ls -xF --color=auto')) + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('ll', 'ls -hlA --color=auto')) + self.assertEqual('ls -hlA --color=auto', cache.execute('SELECT command FROM aliases WHERE alias=?', ('ll',)).fetchone()[0]) def test_file_cache(self): with tempfile.NamedTemporaryFile(suffix = '.sqlite') as tmp: with patch('nyx.data_directory', Mock(return_value = tmp.name)): - cache = nyx.cache() - self.assertEqual((0, 'main', tmp.name), cache.execute("PRAGMA database_list").fetchone()) - - cache.execute('CREATE TABLE aliases(alias TEXT, command TEXT)') - cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('l', 'ls -xF --color=auto')) - cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('ll', 'ls -hlA --color=auto')) - cache.commit() - cache.close() + with nyx.cache() as cache: + self.assertEqual((0, 'main', tmp.name), cache.execute("PRAGMA database_list").fetchone()) + + cache.execute('CREATE TABLE aliases(alias TEXT, command TEXT)') + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('l', 'ls -xF --color=auto')) + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('ll', 'ls -hlA --color=auto')) + cache.commit() + nyx.CACHE = None - self.assertEqual('ls -hlA --color=auto', nyx.cache().execute('SELECT command FROM aliases WHERE alias=?', ('ll',)).fetchone()[0]) + with nyx.cache() as cache: + self.assertEqual('ls -hlA --color=auto', cache.execute('SELECT command FROM aliases WHERE alias=?', ('ll',)).fetchone()[0])

1 0

[nyx/master] Add cache helper
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit e9fbe44319256790aa69e8e20ebec042511a55d8 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 28 12:12:15 2017 -0700 Add cache helper Simple helper that provides back a sqlite cache for us to use. If we can persist it then great, if not then it resides in memory. --- nyx/__init__.py | 18 ++++++++++++++++++ test/cache.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/nyx/__init__.py b/nyx/__init__.py index 3cf3c85..c7db0cb 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -33,6 +33,7 @@ Tor curses monitoring application. import distutils.spawn import os +import sqlite3 import sys import threading import time @@ -84,6 +85,7 @@ CONFIG = stem.util.conf.config_dict('nyx', { NYX_INTERFACE = None TOR_CONTROLLER = None BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-1]) +CACHE = None # technically can change but we use this query a *lot* so needs to be cached @@ -250,6 +252,22 @@ def data_directory(filename, config): return os.path.join(data_dir, filename) +def cache(): + """ + Provides the sqlite cache for application data. + + :returns: **sqlite3.Connection** for our applicaion cache + """ + + global CACHE + + if CACHE is None: + cache_path = data_directory('cache.sqlite') + CACHE = sqlite3.connect(cache_path if cache_path else ':memory:') + + return CACHE + + @uses_settings def expand_path(path, config): """ diff --git a/test/cache.py b/test/cache.py new file mode 100644 index 0000000..dc1fa40 --- /dev/null +++ b/test/cache.py @@ -0,0 +1,40 @@ +""" +Unit tests for nyx.cache. +""" + +import tempfile +import unittest + +import nyx + +from mock import Mock, patch + + +class TestCache(unittest.TestCase): + def setUp(self): + nyx.CACHE = None # drop cached database reference + + @patch('nyx.data_directory', Mock(return_value = None)) + def test_memory_cache(self): + cache = nyx.cache() + self.assertEqual((0, 'main', ''), cache.execute("PRAGMA database_list").fetchone()) + + cache.execute('CREATE TABLE aliases(alias TEXT, command TEXT)') + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('l', 'ls -xF --color=auto')) + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('ll', 'ls -hlA --color=auto')) + self.assertEqual('ls -hlA --color=auto', cache.execute('SELECT command FROM aliases WHERE alias=?', ('ll',)).fetchone()[0]) + + def test_file_cache(self): + with tempfile.NamedTemporaryFile(suffix = '.sqlite') as tmp: + with patch('nyx.data_directory', Mock(return_value = tmp.name)): + cache = nyx.cache() + self.assertEqual((0, 'main', tmp.name), cache.execute("PRAGMA database_list").fetchone()) + + cache.execute('CREATE TABLE aliases(alias TEXT, command TEXT)') + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('l', 'ls -xF --color=auto')) + cache.execute('INSERT INTO aliases(alias, command) VALUES (?,?)', ('ll', 'ls -hlA --color=auto')) + cache.commit() + cache.close() + nyx.CACHE = None + + self.assertEqual('ls -hlA --color=auto', nyx.cache().execute('SELECT command FROM aliases WHERE alias=?', ('ll',)).fetchone()[0])

1 0

[nyx/master] Move cache to its own module
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 70fe1215a01020a87c9cdee139f4188b1424d006 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 31 10:41:22 2017 -0700 Move cache to its own module Not yet even close to what I'm planning, but moving this out of the base module is a good first step. --- nyx/__init__.py | 53 +------------------------------------- nyx/cache.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/cache.py | 26 +++++++++---------- 3 files changed, 93 insertions(+), 65 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index 8c3d396..c2032bc 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -31,10 +31,8 @@ Tor curses monitoring application. +- halt - stops daemon panels """ -import contextlib import distutils.spawn import os -import sqlite3 import sys import threading import time @@ -55,6 +53,7 @@ __license__ = 'GPLv3' __all__ = [ 'arguments', + 'cache', 'controller', 'curses', 'log', @@ -86,17 +85,6 @@ CONFIG = stem.util.conf.config_dict('nyx', { NYX_INTERFACE = None TOR_CONTROLLER = None BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-1]) -CACHE = None -CACHE_LOCK = threading.RLock() - -SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following -SCHEMA = ( - 'CREATE TABLE schema(version NUMBER)', - 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, - - 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port NUMBER, nickname TEXT)', -) - # technically can change but we use this query a *lot* so needs to be cached @@ -263,45 +251,6 @@ def data_directory(filename, config): return os.path.join(data_dir, filename) -(a)contextlib.contextmanager -def cache(): - """ - Provides the sqlite cache for application data. - - :returns: **sqlite3.Connection** for our applicaion cache - """ - - global CACHE - - with CACHE_LOCK: - if CACHE is None: - cache_path = data_directory('cache.sqlite') - - if cache_path: - try: - CACHE = sqlite3.connect(cache_path) - schema = CACHE.execute('SELECT version FROM schema').fetchone()[0] - except: - schema = 'no schema' - - if schema != SCHEMA_VERSION: - stem.util.log.info('Cache schema of %s is out of date (has %s but current version is %s). Clearing the cache.' % (cache_path, schema, SCHEMA_VERSION)) - - CACHE.close() - os.remove(cache_path) - CACHE = sqlite3.connect(cache_path) - - for cmd in SCHEMA: - CACHE.execute(cmd) - else: - CACHE = sqlite3.connect(':memory:') - - for cmd in SCHEMA: - CACHE.execute(cmd) - - yield CACHE - - @uses_settings def expand_path(path, config): """ diff --git a/nyx/cache.py b/nyx/cache.py new file mode 100644 index 0000000..b989f20 --- /dev/null +++ b/nyx/cache.py @@ -0,0 +1,79 @@ +""" +Cache for frequently needed information. This persists to disk if we can, and +otherwise is an in-memory cache. +""" + +import contextlib +import os +import sqlite3 +import threading + +import stem.util.log + +import nyx + +CACHE = None +CACHE_LOCK = threading.RLock() + +SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following +SCHEMA = ( + 'CREATE TABLE schema(version NUMBER)', + 'INSERT INTO schema(version) VALUES (%i)' % SCHEMA_VERSION, + + 'CREATE TABLE relays(fingerprint TEXT PRIMARY KEY, address TEXT, or_port NUMBER, nickname TEXT)', +) + + +(a)contextlib.contextmanager +def cache(): + """ + Provides the sqlite cache for application data. + + :returns: :class:`~nyx.cache.Cache` for our applicaion + """ + + global CACHE + + with CACHE_LOCK: + if CACHE is None: + CACHE = Cache() + + yield CACHE + + +class Cache(object): + """ + Cache for frequently used information. + """ + + def __init__(self): + cache_path = nyx.data_directory('cache.sqlite') + + if cache_path: + try: + self._conn = sqlite3.connect(cache_path) + schema = self._conn.execute('SELECT version FROM schema').fetchone()[0] + except: + schema = 'no schema' + + if schema != SCHEMA_VERSION: + stem.util.log.info('Cache schema of %s is out of date (has %s but current version is %s). Clearing the cache.' % (cache_path, schema, SCHEMA_VERSION)) + + self._conn.close() + os.remove(cache_path) + self._conn = sqlite3.connect(cache_path) + + for cmd in SCHEMA: + self._conn.execute(cmd) + else: + self._conn = sqlite3.connect(':memory:') + + for cmd in SCHEMA: + self._conn.execute(cmd) + + def query(self, query, *param): + """ + Performs a query on our cache. + """ + + return self._conn.execute(query, param) diff --git a/test/cache.py b/test/cache.py index 01f99af..733fbca 100644 --- a/test/cache.py +++ b/test/cache.py @@ -5,7 +5,7 @@ Unit tests for nyx.cache. import tempfile import unittest -import nyx +import nyx.cache from mock import Mock, patch @@ -17,7 +17,7 @@ NICKNAME = 'caersidi' class TestCache(unittest.TestCase): def setUp(self): - nyx.CACHE = None # drop cached database reference + nyx.cache.CACHE = None # drop cached database reference @patch('nyx.data_directory', Mock(return_value = None)) def test_memory_cache(self): @@ -25,10 +25,10 @@ class TestCache(unittest.TestCase): Create a cache in memory. """ - with nyx.cache() as cache: - self.assertEqual((0, 'main', ''), cache.execute("PRAGMA database_list").fetchone()) - cache.execute('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', (FINGERPRINT, ADDRESS, PORT, NICKNAME)) - self.assertEqual(NICKNAME, cache.execute('SELECT nickname FROM relays WHERE fingerprint=?', (FINGERPRINT,)).fetchone()[0]) + with nyx.cache.cache() as cache: + self.assertEqual((0, 'main', ''), cache.query('PRAGMA database_list').fetchone()) + cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) + self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0]) def test_file_cache(self): """ @@ -37,12 +37,12 @@ class TestCache(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix = '.sqlite') as tmp: with patch('nyx.data_directory', Mock(return_value = tmp.name)): - with nyx.cache() as cache: - self.assertEqual((0, 'main', tmp.name), cache.execute("PRAGMA database_list").fetchone()) - cache.execute('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', (FINGERPRINT, ADDRESS, PORT, NICKNAME)) - cache.commit() + with nyx.cache.cache() as cache: + self.assertEqual((0, 'main', tmp.name), cache.query('PRAGMA database_list').fetchone()) + cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) + cache._conn.commit() - nyx.CACHE = None + nyx.cache.CACHE = None - with nyx.cache() as cache: - self.assertEqual(NICKNAME, cache.execute('SELECT nickname FROM relays WHERE fingerprint=?', (FINGERPRINT,)).fetchone()[0]) + with nyx.cache.cache() as cache: + self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0])

1 0

[nyx/master] Logging in all caching scenarios
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit b9e053241ea5370017f62ea4cb57d70301fc93ca Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 31 11:45:03 2017 -0700 Logging in all caching scenarios Providing a log message in all caching scenarios when we load the cache, rather than only when the schema is out of date. --- nyx/cache.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nyx/cache.py b/nyx/cache.py index b989f20..8892be3 100644 --- a/nyx/cache.py +++ b/nyx/cache.py @@ -54,10 +54,15 @@ class Cache(object): self._conn = sqlite3.connect(cache_path) schema = self._conn.execute('SELECT version FROM schema').fetchone()[0] except: - schema = 'no schema' + schema = None - if schema != SCHEMA_VERSION: - stem.util.log.info('Cache schema of %s is out of date (has %s but current version is %s). Clearing the cache.' % (cache_path, schema, SCHEMA_VERSION)) + if schema == SCHEMA_VERSION: + stem.util.log.info('Cache loaded from %s' % cache_path) + else: + if schema is None: + stem.util.log.info('Cache at %s is missing a schema, clearing it.' % cache_path) + else: + stem.util.log.info('Cache at %s has schema version %s but the current version is %s, clearing it.' % (cache_path, schema, SCHEMA_VERSION)) self._conn.close() os.remove(cache_path) @@ -66,6 +71,7 @@ class Cache(object): for cmd in SCHEMA: self._conn.execute(cmd) else: + stem.util.log.info('Unable to cache to disk. Using an in-memory cache instead.') self._conn = sqlite3.connect(':memory:') for cmd in SCHEMA:

1 0

[nyx/master] Move ConsensusTracker to the relay cache
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 2b8ae69028a4618b5bfaf1c46813b8daeecb1411 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 2 12:51:32 2017 -0700 Move ConsensusTracker to the relay cache --- nyx/__init__.py | 23 +++++++++++++++++------ nyx/tracker.py | 44 ++++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index 39c1e5d..26b490d 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -8,6 +8,7 @@ Tor curses monitoring application. nyx_interface - nyx interface singleton tor_controller - tor connection singleton + cache - provides our application cache show_message - shows a message to the user input_prompt - prompts the user for text input @@ -15,6 +16,15 @@ Tor curses monitoring application. expand_path - expands path with respect to our chroot join - joins a series of strings up to a set length + Cache - application cache + |- write - provides a content where we can write to the cache + | + |- relay_nickname - provides the nickname of a relay + +- relay_address - provides the address and orport of a relay + + CacheWriter - context in which we can write to the cache + +- record_relay - caches information about a relay + Interface - overall nyx interface |- get_page - page we're showing |- set_page - sets the page we're showing @@ -389,30 +399,31 @@ class Cache(object): with self._conn: yield CacheWriter(self) - def relay_nickname(self, fingerprint): + def relay_nickname(self, fingerprint, default = None): """ Provides the nickname associated with the given relay. :param str fingerprint: relay to look up + :param str default: response if no such relay exists - :returns: **str** with the nickname ("Unnamed" if unset), and **None** if - no such relay exists + :returns: **str** with the nickname ("Unnamed" if unset) """ result = self._query('SELECT nickname FROM relays WHERE fingerprint=?', fingerprint).fetchone() - return result[0] if result else None + return result[0] if result else default - def relay_address(self, fingerprint): + def relay_address(self, fingerprint, default = None): """ Provides the (address, port) tuple where a relay is running. :param str fingerprint: fingerprint to be checked + :param str default: response if no such relay exists :returns: **tuple** with a **str** address and **int** port """ result = self._query('SELECT address, or_port FROM relays WHERE fingerprint=?', fingerprint).fetchone() - return result if result else None # TODO: does this raise if fingerprint doesn't exist? + return result if result else default def _query(self, query, *param): """ diff --git a/nyx/tracker.py b/nyx/tracker.py index affe680..cbfaedf 100644 --- a/nyx/tracker.py +++ b/nyx/tracker.py @@ -55,6 +55,7 @@ import os import time import threading +import nyx import stem.control import stem.descriptor.router_status_entry import stem.util.log @@ -808,7 +809,6 @@ class ConsensusTracker(object): def __init__(self): self._fingerprint_cache = {} # {address => [(port, fingerprint), ..]} for relays - self._relay_cache = {} # fingerprint => address, orport, nickname lookup cache self._my_router_status_entry = None self._my_router_status_entry_time = 0 @@ -821,19 +821,20 @@ class ConsensusTracker(object): ns_response = controller.get_info('ns/all', None) if ns_response: - for line in ns_response.splitlines(): - if line.startswith('r '): - r_comp = line.split(' ') + with nyx.cache().write() as writer: + for line in ns_response.splitlines(): + if line.startswith('r '): + r_comp = line.split(' ') - address = r_comp[6] - or_port = int(r_comp[7]) - fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) - nickname = r_comp[1] + address = r_comp[6] + or_port = int(r_comp[7]) + fingerprint = stem.descriptor.router_status_entry._base64_to_hex(r_comp[2]) + nickname = r_comp[1] - self._fingerprint_cache.setdefault(address, []).append((or_port, fingerprint)) - self._relay_cache[fingerprint] = (address, or_port, nickname) + self._fingerprint_cache.setdefault(address, []).append((or_port, fingerprint)) + writer.record_relay(fingerprint, address, or_port, nickname) - stem.util.log.info('Cached consensus data. Took %0.2fs. Cache size is %s for fingerprints, %s for relays' % (time.time() - start_time, stem.util.str_tools.size_label(system.size_of(self._fingerprint_cache), 2), stem.util.str_tools.size_label(system.size_of(self._relay_cache), 2))) + stem.util.log.info('Cached consensus data, took %0.2fs.' % (time.time() - start_time)) controller.add_event_listener(lambda event: self.update(event.desc), stem.control.EventType.NEWCONSENSUS) @@ -845,23 +846,22 @@ class ConsensusTracker(object): """ new_fingerprint_cache = {} - new_relay_cache = {} start_time = time.time() our_fingerprint = tor_controller().get_info('fingerprint', None) - for desc in router_status_entries: - new_fingerprint_cache.setdefault(desc.address, []).append((desc.or_port, desc.fingerprint)) - new_relay_cache[desc.fingerprint] = (desc.address, desc.or_port, desc.nickname) + with nyx.cache().write() as writer: + for desc in router_status_entries: + new_fingerprint_cache.setdefault(desc.address, []).append((desc.or_port, desc.fingerprint)) + writer.record_relay(desc.fingerprint, desc.address, desc.or_port, desc.nickname) - if desc.fingerprint == our_fingerprint: - self._my_router_status_entry = desc - self._my_router_status_entry_time = time.time() + if desc.fingerprint == our_fingerprint: + self._my_router_status_entry = desc + self._my_router_status_entry_time = time.time() self._fingerprint_cache = new_fingerprint_cache - self._relay_cache = new_relay_cache - stem.util.log.info('Updated consensus cache. Took %0.2fs. Cache size is %s for fingerprints, %s for relays' % (time.time() - start_time, stem.util.str_tools.size_label(system.size_of(self._fingerprint_cache), 2), stem.util.str_tools.size_label(system.size_of(self._relay_cache), 2))) + stem.util.log.info('Updated consensus cache, took %0.2fs.' % (time.time() - start_time)) def my_router_status_entry(self): """ @@ -895,7 +895,7 @@ class ConsensusTracker(object): elif fingerprint == controller.get_info('fingerprint', None): return controller.get_conf('Nickname', 'Unnamed') else: - return self._relay_cache.get(fingerprint)[2] + return nyx.cache().relay_nickname(fingerprint) def get_relay_fingerprints(self, address): """ @@ -935,4 +935,4 @@ class ConsensusTracker(object): if my_address and len(my_or_ports) == 1: return (my_address, my_or_ports[0]) - return self._relay_cache.get(fingerprint, default)[:2] + return nyx.cache().relay_address(fingerprint, default)

1 0

[nyx/master] Drop cache context manager
by atagar＠torproject.org 03 Sep '17

03 Sep '17

commit 068ace653d9b3127cdef1c2b9ddd33493af957dc Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 31 11:58:27 2017 -0700 Drop cache context manager Now that we have a class that can do its own locking there's no need to require a context manager. --- nyx/cache.py | 14 ++++++-------- test/cache.py | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/nyx/cache.py b/nyx/cache.py index 8892be3..f4c6f3f 100644 --- a/nyx/cache.py +++ b/nyx/cache.py @@ -3,7 +3,6 @@ Cache for frequently needed information. This persists to disk if we can, and otherwise is an in-memory cache. """ -import contextlib import os import sqlite3 import threading @@ -13,7 +12,6 @@ import stem.util.log import nyx CACHE = None -CACHE_LOCK = threading.RLock() SCHEMA_VERSION = 1 # version of our scheme, bump this if you change the following SCHEMA = ( @@ -24,7 +22,6 @@ SCHEMA = ( ) -(a)contextlib.contextmanager def cache(): """ Provides the sqlite cache for application data. @@ -34,11 +31,10 @@ def cache(): global CACHE - with CACHE_LOCK: - if CACHE is None: - CACHE = Cache() + if CACHE is None: + CACHE = Cache() - yield CACHE + return CACHE class Cache(object): @@ -47,6 +43,7 @@ class Cache(object): """ def __init__(self): + self._conn_lock = threading.RLock() cache_path = nyx.data_directory('cache.sqlite') if cache_path: @@ -82,4 +79,5 @@ class Cache(object): Performs a query on our cache. """ - return self._conn.execute(query, param) + with self._conn_lock: + return self._conn.execute(query, param) diff --git a/test/cache.py b/test/cache.py index 733fbca..f26ede9 100644 --- a/test/cache.py +++ b/test/cache.py @@ -25,10 +25,11 @@ class TestCache(unittest.TestCase): Create a cache in memory. """ - with nyx.cache.cache() as cache: - self.assertEqual((0, 'main', ''), cache.query('PRAGMA database_list').fetchone()) - cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) - self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0]) + cache = nyx.cache.cache() + + self.assertEqual((0, 'main', ''), cache.query('PRAGMA database_list').fetchone()) + cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) + self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0]) def test_file_cache(self): """ @@ -37,12 +38,13 @@ class TestCache(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix = '.sqlite') as tmp: with patch('nyx.data_directory', Mock(return_value = tmp.name)): - with nyx.cache.cache() as cache: - self.assertEqual((0, 'main', tmp.name), cache.query('PRAGMA database_list').fetchone()) - cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) - cache._conn.commit() + cache = nyx.cache.cache() + + self.assertEqual((0, 'main', tmp.name), cache.query('PRAGMA database_list').fetchone()) + cache.query('INSERT INTO relays(fingerprint, address, or_port, nickname) VALUES (?,?,?,?)', FINGERPRINT, ADDRESS, PORT, NICKNAME) + cache._conn.commit() nyx.cache.CACHE = None - with nyx.cache.cache() as cache: - self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0]) + cache = nyx.cache.cache() + self.assertEqual(NICKNAME, cache.query('SELECT nickname FROM relays WHERE fingerprint=?', FINGERPRINT).fetchone()[0])

1 0