commit 7d8572a08eb77dae9fdbe09b16d091a1194ec297 Author: Isis Lovecruft isis@torproject.org Date: Fri Nov 15 13:55:31 2013 +0000
Add bridgedb.persistent module for storing persistent state.
* ADD bridgedb.persistent.State class (along with load and save utilities) for securely storing serialized s-expressions representing Python objects and instances.
* ADD 21 unittests for testing all new methods and classes in the bridgedb.persistent module.
The State class can be used to track config file changes. For example, in bridgedb.Main.run(), several attributes on the config `Conf` object are changed, such as various filenames (to point to their absolute paths). When bridgedb is SIGHUPed, it will reread the config file and reapply all settings. However, this overwrites the variables which have been set in the code (for example, pointing back to non-absolute filepaths).
The persistent.State class stores the `Conf` attributes as it's own attributes, and stores the `Conf` instance as State().config. This way, any changes to settings can be applied directly to the State instance, and if the `Conf` object changes on SIGHUP, BridgeDB will understand that a human (hopefully) has changed the file, rather than its own code. Thus, it is smarter about which settings to reapply. --- lib/bridgedb/persistent.py | 259 +++++++++++++++++++++++ lib/bridgedb/test/test_persistent.py | 170 +++++++++++++++ lib/bridgedb/test/test_persistentSaveAndLoad.py | 93 ++++++++ 3 files changed, 522 insertions(+)
diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py new file mode 100644 index 0000000..89e9ff4 --- /dev/null +++ b/lib/bridgedb/persistent.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_persistent -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# please also see AUTHORS file +# :copyright: (c) 2013 Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Module for functionality to persistently storing state.""" + +import copy +import logging +import os.path + +try: + import cPickle as pickle +except (ImportError, NameError): + import pickle + +from twisted.spread import jelly + +from bridgedb import Filters, Bridges, Dist +#from bridgedb.proxy import ProxySet + +_state = None + +#: Types and classes which are allowed to be jellied: +_security = jelly.SecurityOptions() +#_security.allowInstancesOf(ProxySet) +_security.allowModules(Filters, Bridges, Dist) + + +class MissingState(Exception): + """Raised when the file or class storing global state is missing.""" + + +def _getState(): + """Retrieve the global state instance. + + :rtype: :class:`~bridgedb.persistent.State` + :returns: An unpickled de-sexp'ed state object, which may contain just + about anything, but should contain things like options, loaded config + settings, etc. + """ + return _state + +def _setState(state): + """Set the global state. + + :type state: :class:`~bridgedb.persistent.State` + :param state: The state instance to save. + """ + global _state + _state = state + +def load(stateCls=None): + """Given a :class:`State`, try to unpickle it's ``statefile``. + + :param string stateCls: An instance of :class:`~bridgedb.persistent.State`. If + not given, try loading from ``_statefile`` if that file exists. + :rtype: None or :class:`State` + """ + statefile = None + if stateCls and isinstance(stateCls, State): + cls = stateCls + else: + cls = _getState() + + if not cls: + raise MissingState("Could not find a state instance to load.") + else: + loaded = cls.load() + return loaded + + +class State(jelly.Jellyable): + """Pickled, jellied storage container for persistent state.""" + + #synchronized = ['load', 'save'] + + def __init__(self, config=None, **kwargs): + """Create a persistent state storage mechanism. + + Serialisation of certain classes in BridgeDB doesn't work. Classes and + modules which are known to be unjelliable/unpicklable so far are: + + - bridgedb.Dist + - bridgedb.Bridges.BridgeHolder and all other ``splitter`` classes + + :property statefile: The filename to retrieve a pickled, jellied + :class:`~bridgedb.persistent.State` instance from. (default: + :attr:`bridgedb.persistent.State._statefile`) + """ + self._statefile = os.path.abspath(str(__package__) + '.state') + self.proxyList = None + self.config = None + self.key = None + + if 'STATEFILE' in kwargs: + self.statefile(kwargs['STATEFILE']) + + for key, value in kwargs.items(): + self.__dict__[key] = value + + if config is not None: + for key, value in config.__dict__.items(): + self.__dict__[key] = value + #self.config = config + + _setState(self) + + def _get_statefile(self): + """Retrieve the filename of the global statefile. + + :rtype: string + :returns: The filename of the statefile. + """ + return self._statefile + + def _set_statefile(self, filename): + """Set the global statefile. + + :param string statefile: The filename of the statefile. + """ + if filename.startswith('~'): + filename = os.path.expanduser(filename) + if not os.path.isabs(filename): + filename = os.path.abspath(filename) + + logging.debug("Setting statefile to '%s'" % filename) + self._statefile = filename + + # Create the parent directory if it doesn't exist: + dirname = os.path.dirname(filename) + if not os.path.isdir(dirname): + os.makedirs(dirname) + + # Create the statefile if it doesn't exist: + if not os.path.exists(filename): + open(filename, 'w').close() + + def _del_statefile(self): + """Delete the file containing previously saved state.""" + try: + with open(self._statefile, 'w') as fh: + fh.close() + os.unlink(self._statefile) + self._statefile = None + except (IOError, OSError) as error: + logging.error("There was an error deleting the statefile: '%s'" + % self._statefile) + + statefile = property(_get_statefile, _set_statefile, _del_statefile, + """Filename property of a persisent.State.""") + + def load(self, statefile=None): + """Load a previously saved statefile. + + :rtype: :class:`State` or None + :returns: The state, loaded from :attr:`State.STATEFILE`, or None if + an error occurred. + """ + if not statefile: + if not self.statefile: + raise MissingState("Could not find a state file to load.") + statefile = self.statefile + logging.debug("Retrieving state from: \t'%s'" % statefile) + + quo= fh = None + err = '' + + try: + if isinstance(statefile, basestring): + fh = open(statefile, 'r') + elif not statefile.closed: + fh = statefile + else: + raise TypeError("Nothing worked.") + except (IOError, OSError) as error: + err += "There was an error reading statefile " + err += "'{0}':\n{1}".format(statefile, error) + except (AttributeError, TypeError) as error: + err += "Failed statefile.open() and statefile.closed:" + err += "\n\t{0}\nstatefile type = '{1}'".format( + error.message, type(statefile)) + else: + status = pickle.load(fh) + quo = jelly.unjelly(status) + if fh is not None: + fh.close() + if quo: + return quo + + raise MissingState(err) + + def save(self, statefile=None): + """Save state as a pickled jelly to a file on disk.""" + if not statefile: + if not self._statefile: + raise MissingState("Could not find a state file to load.") + statefile = self._statefile + logging.debug("Saving state to: \t'%s'" % statefile) + + fh = None + err = '' + + try: + fh = open(statefile, 'w') + except MissingState as error: + err += error.message + except (IOError, OSError) as error: + err += "Error writing state file to '%s': %s" % (statefile, error) + else: + try: + pickle.dump(jelly.jelly(self), fh) + except AttributeError as error: + logging.debug("Tried jellying an unjelliable object: %s" + % error.message) + + if fh is not None: + fh.flush() + fh.close() + + def useChangedSettings(self, config): + """Take a new config, compare it to the last one, and update settings. + + Given a ``config`` object created from the configuration file, compare + it to the last :class:`~bridgedb.Main.Conf` that was stored, and apply + any settings which were changed to be attributes of the :class:`State` + instance. + """ + updated = new = [] + + for key, value in config.__dict__.items(): + try: + # If state.config doesn't have the same value as the new + # config, then update the state setting. + # + # Be sure, when updating settings while parsing the config + # file, to assign the new settings as attributes of the + # :class:`bridgedb.Main.Conf` instance. + if value != self.config.__dict__[key]: + setattr(self, key, value) + updated.append(key) + logging.debug("Updated %s setting: %r → %r" + % (key, self.config.__dict__[key], value)) + except KeyError: + setattr(self, key, value) + new.append(key) + logging.debug("New setting: %s = %r" % (key, value)) + + logging.info("Updated setting(s): %s" % ' '.join([x for x in updated])) + logging.info("New setting(s): %s" % ' '.join([x for x in new])) + logging.debug( + "Saving newer config as `state.config` for later comparison") + self.config = config diff --git a/lib/bridgedb/test/test_persistent.py b/lib/bridgedb/test/test_persistent.py new file mode 100644 index 0000000..34b0b25 --- /dev/null +++ b/lib/bridgedb/test/test_persistent.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# please also see AUTHORS file +# :copyright: (c) 2013, Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.persistent` module. + +These tests are meant to ensure that the :mod:`bridgedb.persistent` module is +functioning as expected. +""" + +from __future__ import print_function + +import os.path + +from copy import deepcopy +from io import StringIO + +from bridgedb import persistent +from bridgedb.parse.options import MainOptions +from twisted.python import log +from twisted.trial import unittest + +import sure +from sure import this +from sure import the +from sure import expect + + +TEST_CONFIG_FILE = StringIO(unicode("""\ +BRIDGE_FILES = ['bridge-descriptors', 'bridge-descriptors.new'] +LOGFILE = 'bridgedb.log'""")) + + +class StateTest(unittest.TestCase): + """Tests for :class:`bridgedb.persistent.State`.""" + + timeout = 15 + + def setUp(self): + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') + exec compiled in configuration + config = persistent.Conf(**configuration) + + fakeArgs = ['-v', '--dump-bridges'] + options = MainOptions() + options.parseOptions(fakeArgs) + + self.options = options + self.config = config + self.state = persistent.State(**config.__dict__) + self.state.options = options + self.state.config = config + + def test_configCreation(self): + this(self.config).should.be.ok + this(self.config).should.be.a(persistent.Conf) + + def test_optionsCreation(self): + this(self.options).should.be.ok + this(self.options).should.be.a(dict) + + def test_stateCreation(self): + this(self.state).should.be.ok + + this(self.state).should.have.property('config').being.ok + this(self.state).should.have.property('config').being.equal(self.config) + + this(self.state.options).should.be.ok + this(self.state.options).should.equal(self.options) + + def test_docstring_persistent(self): + persistent.should.have.property('__doc__').being.a(str) + + def test_docstring_persistentState(self): + the(self.state).should.have.property('__doc__').being.a(str) + + def test_state_init(self): + this(self.state).should.have.property('config') + this(self.state).should.have.property('proxyList') + this(self.state).should.have.property('statefile') + + def test_persistent_getState(self): + persistent.should.have.property('_getState').being(callable) + this(persistent._getState()).should.be.a(persistent.State) + + def test_getStateFor(self): + jellyState = self.state.getStateFor(self) + expect(jellyState).to.be.a(dict) + expect(jellyState.keys()).to.contain('LOGFILE') + + def test_STATEFILE(self): + this(self.state).should.have.property('statefile') + the(self.state.statefile).should.be.a(str) + + def test_existsSave(self): + this(self.state).should.have.property('save').being(callable) + + def test_existsLoad(self): + persistent.should.have.property('load').being(callable) + + def test_persistent_state(self): + the(persistent._state).should.be.a(persistent.State) + + def test_before_useChangedSettings_state(self): + this(self.state).shouldnt.have.property('FOO') + this(self.state).shouldnt.have.property('BAR') + this(self.state).should.have.property('LOGFILE').being.a(str) + this(self.state).should.have.property( + 'BRIDGE_FILES').being.a(list) + + def test_before_useChangedSettings_config(self): + this(self.config).shouldnt.have.property('FOO') + this(self.config).shouldnt.have.property('BAR') + this(self.config).should.have.property('LOGFILE').being.a(str) + this(self.config).should.have.property( + 'BRIDGE_FILES').being.a(list) + + def test_before_useChangedSettings_stateConfig(self): + this(self.state.config).shouldnt.have.property('FOO') + this(self.state.config).shouldnt.have.property('BAR') + this(self.state.config).should.have.property('LOGFILE').being.a(str) + this(self.state.config).should.have.property( + 'BRIDGE_FILES').being.a(list) + + def test_useChangedSettings(self): + # This deepcopying must be done to avoid changing the State object + # which is used for the rest of the tests. + + thatConfig = deepcopy(self.config) + thatState = deepcopy(self.state) + + setattr(thatConfig, 'FOO', 'fuuuuu') + setattr(thatConfig, 'BAR', 'all of the things') + setattr(thatConfig, 'LOGFILE', 42) + + this(thatConfig).should.have.property('FOO').being.a(str) + this(thatConfig).should.have.property('BAR').being.a(str) + this(thatConfig).should.have.property('LOGFILE').being.an(int) + this(thatConfig).should.have.property('BRIDGE_FILES').being.a(list) + + the(thatConfig.FOO).must.equal('fuuuuu') + the(thatConfig.BAR).must.equal('all of the things') + the(thatConfig.LOGFILE).must.equal(42) + + the(thatState).should.have.property('useChangedSettings') + the(thatState.useChangedSettings).should.be(callable) + thatState.useChangedSettings(thatConfig) + + the(thatState.FOO).should.equal('fuuuuu') + the(thatState).should.have.property('FOO').being.a(str) + the(thatState).should.have.property('BAR').being.a(str) + the(thatState).should.have.property('LOGFILE').being.an(int) + the(thatState.FOO).must.equal(thatConfig.FOO) + the(thatState.BAR).must.equal(thatConfig.BAR) + the(thatState.LOGFILE).must.equal(thatConfig.LOGFILE) + + this(thatState.config).should.have.property('FOO') + this(thatState.config).should.have.property('BAR') + this(thatState.config).should.have.property('LOGFILE').being.an(int) + this(thatState.config).should.have.property( + 'BRIDGE_FILES').being.a(list) diff --git a/lib/bridgedb/test/test_persistentSaveAndLoad.py b/lib/bridgedb/test/test_persistentSaveAndLoad.py new file mode 100644 index 0000000..10467ce --- /dev/null +++ b/lib/bridgedb/test/test_persistentSaveAndLoad.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 isis@torproject.org +# please also see AUTHORS file +# :copyright: (c) 2013, Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-Clause BSD, see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.persistent` module. + +These tests ensure that :meth:`bridgedb.persistent.State.save`, +:func:`bridgedb.persistent.load`, and :meth:`bridgedb.persistent.State.load` +are all functioning as expected. + +This module should not import :mod:`sure`. +""" + +import os + +from copy import deepcopy +from io import StringIO + +from twisted.trial import unittest + +from bridgedb import persistent + + +TEST_CONFIG_FILE = StringIO(unicode("""\ +BRIDGE_FILES = ['bridge-descriptors', 'bridge-descriptors.new'] +LOGFILE = 'bridgedb.log'""")) + + +class StateSaveAndLoadTests(unittest.TestCase): + """Test save() and load() of :mod:`~bridgedb.persistent`.""" + + timeout = 15 + + def setUp(self): + configuration = {} + TEST_CONFIG_FILE.seek(0) + compiled = compile(TEST_CONFIG_FILE.read(), '<string>', 'exec') + exec compiled in configuration + config = persistent.Conf(**configuration) + + self.config = config + self.state = persistent.State(**config.__dict__) + self.state.config = config + self.state.statefile = os.path.abspath('bridgedb.state') + + def loadedStateAssertions(self, loadedState): + self.assertIsNotNone(loadedState) + self.assertIsInstance(loadedState, persistent.State) + self.assertNotIdentical(self.state, loadedState) + self.assertNotEqual(self.state, loadedState) + self.assertItemsEqual(self.state.__dict__.keys(), + loadedState.__dict__.keys()) + + def savedStateAssertions(self, savedStatefile=None): + self.assertTrue(os.path.isfile(str(self.state.statefile))) + if savedStatefile: + self.assertTrue(os.path.isfile(str(savedStatefile))) + + def test_save(self): + self.state.save() + self.savedStateAssertions() + + def test_stateSaveTempfile(self): + savefile = self.mktemp() + self.state.statefile = savefile + self.state.save(savefile) + savedStatefile = str(self.state.statefile) + + def test_stateLoadTempfile(self): + savefile = self.mktemp() + self.state.statefile = savefile + self.assertTrue(self.state.statefile.endswith(savefile)) + self.state.save(savefile) + self.savedStateAssertions(savefile) + loadedState = self.state.load(savefile) + self.loadedStateAssertions(loadedState) + + def test_stateSaveAndLoad(self): + self.state.save() + loadedState = self.state.load() + self.loadedStateAssertions(loadedState) + + def test_load(self): + self.state.save() + loadedState = persistent.load() + self.loadedStateAssertions(loadedState)