[or-cvs] r23723: {arm} persisting tor config descriptions to reduce startup time (i (in arm/trunk: . src src/interface src/util)
Damian Johnson
atagar1 at gmail.com
Sat Oct 30 05:54:21 UTC 2010
Author: atagar
Date: 2010-10-30 05:54:20 +0000 (Sat, 30 Oct 2010)
New Revision: 23723
Modified:
arm/trunk/README
arm/trunk/armrc.sample
arm/trunk/src/interface/controller.py
arm/trunk/src/interface/logPanel.py
arm/trunk/src/starter.py
arm/trunk/src/util/__init__.py
arm/trunk/src/util/sysTools.py
arm/trunk/src/util/torConfig.py
Log:
persisting tor config descriptions to reduce startup time (idea by nickm)
fix: stripping error number from file related IOError log messages
fix: order of prepopulated arm events was backwards
Modified: arm/trunk/README
===================================================================
--- arm/trunk/README 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/README 2010-10-30 05:54:20 UTC (rev 23723)
@@ -137,6 +137,7 @@
log.py - aggregator for application events
panel.py - wrapper for safely working with curses subwindows
sysTools.py - helper for system calls, providing client side caching
+ torConfig.py - functions for working with the torrc and config options
torTools.py - TorCtl wrapper, providing caching and derived information
uiTools.py - helper functions for presenting the user interface
Modified: arm/trunk/armrc.sample
===================================================================
--- arm/trunk/armrc.sample 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/armrc.sample 2010-10-30 05:54:20 UTC (rev 23723)
@@ -71,6 +71,19 @@
features.config.file.showScrollbars true
features.config.file.maxLinesPerEntry 8
+# Descriptions for tor's configuration options can be loaded from its man page
+# to give usage information on the settings page. They can also be persisted to
+# a file to speed future lookups.
+# ---------------------------
+# enabled
+# allows the descriptions to be fetched from the man page if true
+# persistPath
+# location descriptions should be loaded from and saved to (this feature is
+# disabled if unset)
+
+features.config.descriptions.enabled true
+features.config.descriptions.persistPath /tmp/arm/torConfigDescriptions.txt
+
# General graph parameters
# ------------------------
# height
@@ -176,6 +189,12 @@
log.torrc.readFailed WARN
log.torrc.validation.duplicateEntries NOTICE
log.torrc.validation.torStateDiffers NOTICE
+log.configDescriptions.readManPageSuccess INFO
+log.configDescriptions.readManPageFailed WARN
+log.configDescriptions.persistance.loadSuccess INFO
+log.configDescriptions.persistance.loadFailed INFO
+log.configDescriptions.persistance.saveSuccess NOTICE
+log.configDescriptions.persistance.saveFailed NOTICE
log.connLookupFailed INFO
log.connLookupFailover NOTICE
log.connLookupAbandon WARN
Modified: arm/trunk/src/interface/controller.py
===================================================================
--- arm/trunk/src/interface/controller.py 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/src/interface/controller.py 2010-10-30 05:54:20 UTC (rev 23723)
@@ -367,9 +367,7 @@
try:
loadedTorrc.load()
except IOError, exc:
- excMsg = str(exc)
- if excMsg.startswith("[Errno "): excMsg = excMsg[10:]
- msg = "Unable to load torrc (%s)" % excMsg
+ msg = "Unable to load torrc (%s)" % sysTools.getFileErrorMsg(exc)
log.log(CONFIG["log.torrc.readFailed"], msg)
if loadedTorrc.isLoaded():
@@ -886,7 +884,7 @@
panels["control"].redraw(True)
time.sleep(2)
except IOError, exc:
- panels["control"].setMsg("Unable to save snapshot: %s" % str(exc), curses.A_STANDOUT)
+ panels["control"].setMsg("Unable to save snapshot: %s" % sysTools.getFileErrorMsg(exc), curses.A_STANDOUT)
panels["control"].redraw(True)
time.sleep(2)
@@ -1451,7 +1449,7 @@
try:
torTools.getConn().reload()
except IOError, exc:
- log.log(log.ERR, "Error detected when reloading tor: %s" % str(exc))
+ log.log(log.ERR, "Error detected when reloading tor: %s" % sysTools.getFileErrorMsg(exc))
#errorMsg = " (%s)" % str(err) if str(err) else ""
#panels["control"].setMsg("Sighup failed%s" % errorMsg, curses.A_STANDOUT)
Modified: arm/trunk/src/interface/logPanel.py
===================================================================
--- arm/trunk/src/interface/logPanel.py 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/src/interface/logPanel.py 2010-10-30 05:54:20 UTC (rev 23723)
@@ -564,7 +564,7 @@
for level, msg, eventTime in log._getEntries(setRunlevels):
runlevelStr = log.RUNLEVEL_STR[level]
armEventEntry = LogEntry(eventTime, "ARM_" + runlevelStr, msg, RUNLEVEL_EVENT_COLOR[runlevelStr])
- armEventBacklog.append(armEventEntry)
+ armEventBacklog.insert(0, armEventEntry)
# joins armEventBacklog and torEventBacklog chronologically into msgLog
while armEventBacklog or torEventBacklog:
@@ -602,7 +602,7 @@
self.logFile = open(logPath, "a")
log.log(self._config["log.logPanel.logFileOpened"], "arm %s opening log file (%s)" % (VERSION, logPath))
except IOError, exc:
- log.log(self._config["log.logPanel.logFileWriteFailed"], "Unable to write to log file: %s" % exc)
+ log.log(self._config["log.logPanel.logFileWriteFailed"], "Unable to write to log file: %s" % sysTools.getFileErrorMsg(exc))
self.logFile = None
def registerEvent(self, event):
@@ -624,7 +624,7 @@
self.logFile.write(event.getDisplayMessage(True) + "\n")
self.logFile.flush()
except IOError, exc:
- log.log(self._config["log.logPanel.logFileWriteFailed"], "Unable to write to log file: %s" % exc)
+ log.log(self._config["log.logPanel.logFileWriteFailed"], "Unable to write to log file: %s" % sysTools.getFileErrorMsg(exc))
self.logFile = None
if self._isPaused:
Modified: arm/trunk/src/starter.py
===================================================================
--- arm/trunk/src/starter.py 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/src/starter.py 2010-10-30 05:54:20 UTC (rev 23723)
@@ -27,11 +27,19 @@
import TorCtl.TorUtil
DEFAULT_CONFIG = os.path.expanduser("~/.armrc")
-DEFAULTS = {"startup.controlPassword": None,
- "startup.interface.ipAddress": "127.0.0.1",
- "startup.interface.port": 9051,
- "startup.blindModeEnabled": False,
- "startup.events": "N3"}
+CONFIG = {"startup.controlPassword": None,
+ "startup.interface.ipAddress": "127.0.0.1",
+ "startup.interface.port": 9051,
+ "startup.blindModeEnabled": False,
+ "startup.events": "N3",
+ "features.config.descriptions.enabled": True,
+ "features.config.descriptions.persistPath": "/tmp/arm/torConfigDescriptions.txt",
+ "log.configDescriptions.readManPageSuccess": util.log.INFO,
+ "log.configDescriptions.readManPageFailed": util.log.WARN,
+ "log.configDescriptions.persistance.loadSuccess": util.log.INFO,
+ "log.configDescriptions.persistance.loadFailed": util.log.INFO,
+ "log.configDescriptions.persistance.saveSuccess": util.log.INFO,
+ "log.configDescriptions.persistance.saveFailed": util.log.NOTICE}
OPT = "i:c:be:vh"
OPT_EXPANDED = ["interface=", "config=", "blind", "event=", "version", "help"]
@@ -50,8 +58,20 @@
Example:
arm -b -i 1643 hide connection data, attaching to control port 1643
arm -e we -c /tmp/cfg use this configuration file with 'WARN'/'ERR' events
-""" % (DEFAULTS["startup.interface.ipAddress"], DEFAULTS["startup.interface.port"], DEFAULT_CONFIG, DEFAULTS["startup.events"], interface.logPanel.EVENT_LISTING)
+""" % (CONFIG["startup.interface.ipAddress"], CONFIG["startup.interface.port"], DEFAULT_CONFIG, CONFIG["startup.events"], interface.logPanel.EVENT_LISTING)
+# messages related to loading the tor configuration descriptions
+DESC_LOAD_SUCCESS_MSG = "Loaded configuration descriptions from '%s' (runtime: %0.3f)"
+DESC_LOAD_FAILED_MSG = "Unable to load configuration descriptions (%s)"
+DESC_READ_MAN_SUCCESS_MSG = "Read descriptions for tor's configuration options from its man page (runtime %0.3f)"
+DESC_READ_MAN_FAILED_MSG = "Unable to read descriptions for tor's configuration options from its man page (%s)"
+DESC_SAVE_SUCCESS_MSG = "Saved configuration descriptions to '%s' (runtime: %0.3f)"
+DESC_SAVE_FAILED_MSG = "Unable to save configuration descriptions (%s)"
+
+NO_INTERNAL_CFG_MSG = "Failed to load the parsing configuration. This will be problematic for a few things like torrc validation and log duplication detection (%s)"
+STANDARD_CFG_LOAD_FAILED_MSG = "Failed to load configuration (using defaults): \"%s\""
+STANDARD_CFG_NOT_FOUND_MSG = "No configuration found at '%s', using defaults"
+
def isValidIpAddr(ipStr):
"""
Returns true if input is a valid IPv4 address, false otherwise.
@@ -75,10 +95,62 @@
return True
+def _loadConfigurationDescriptions():
+ """
+ Attempts to load descriptions for tor's configuration options, fetching them
+ from the man page and persisting them to a file to speed future startups.
+ """
+
+ # It is important that this is loaded before entering the curses context,
+ # otherwise the man call pegs the cpu for around a minute (I'm not sure
+ # why... curses must mess the terminal in a way that's important to man).
+
+ if CONFIG["features.config.descriptions.enabled"]:
+ isConfigDescriptionsLoaded = False
+ descriptorPath = CONFIG["features.config.descriptions.persistPath"]
+
+ # attempts to load persisted configuration descriptions
+ if descriptorPath:
+ try:
+ loadStartTime = time.time()
+ util.torConfig.loadOptionDescriptions(descriptorPath)
+ isConfigDescriptionsLoaded = True
+
+ msg = DESC_LOAD_SUCCESS_MSG % (descriptorPath, time.time() - loadStartTime)
+ util.log.log(CONFIG["log.configDescriptions.persistance.loadSuccess"], msg)
+ except IOError, exc:
+ msg = DESC_LOAD_FAILED_MSG % util.sysTools.getFileErrorMsg(exc)
+ util.log.log(CONFIG["log.configDescriptions.persistance.loadFailed"], msg)
+
+ if not isConfigDescriptionsLoaded:
+ try:
+ # fetches configuration options from the man page
+ loadStartTime = time.time()
+ util.torConfig.loadOptionDescriptions()
+ isConfigDescriptionsLoaded = True
+
+ msg = DESC_READ_MAN_SUCCESS_MSG % (time.time() - loadStartTime)
+ util.log.log(CONFIG["log.configDescriptions.readManPageSuccess"], msg)
+ except IOError, exc:
+ msg = DESC_READ_MAN_FAILED_MSG % util.sysTools.getFileErrorMsg(exc)
+ util.log.log(CONFIG["log.configDescriptions.readManPageFailed"], msg)
+
+ # persists configuration descriptions
+ if isConfigDescriptionsLoaded and descriptorPath:
+ try:
+ loadStartTime = time.time()
+ util.torConfig.saveOptionDescriptions(descriptorPath)
+
+ msg = DESC_SAVE_SUCCESS_MSG % (descriptorPath, time.time() - loadStartTime)
+ util.log.log(CONFIG["log.configDescriptions.persistance.loadSuccess"], msg)
+ except IOError, exc:
+ msg = DESC_SAVE_FAILED_MSG % util.sysTools.getFileErrorMsg(exc)
+ util.log.log(CONFIG["log.configDescriptions.persistance.saveFailed"], msg)
+
if __name__ == '__main__':
startTime = time.time()
- param = dict([(key, None) for key in DEFAULTS.keys()])
- configPath = DEFAULT_CONFIG # path used for customized configuration
+ param = dict([(key, None) for key in CONFIG.keys()])
+ configPath = DEFAULT_CONFIG # path used for customized configuration
# parses user input, noting any issues
try:
@@ -127,11 +199,7 @@
config.load("%ssettings.cfg" % pathPrefix)
except IOError, exc:
- # Strips off the error number prefix from the message. Example error msg:
- # [Errno 2] No such file or directory
- excMsg = str(exc)
- if excMsg.startswith("[Errno "): excMsg = excMsg[10:]
- msg = "Failed to load the parsing configuration. This will be problematic for a few things like torrc validation and log duplication detection (%s)" % excMsg
+ msg = NO_INTERNAL_CFG_MSG % util.sysTools.getFileErrorMsg(exc)
util.log.log(util.log.WARN, msg)
# loads user's personal armrc if available
@@ -139,15 +207,15 @@
try:
config.load(configPath)
except IOError, exc:
- msg = "Failed to load configuration (using defaults): \"%s\"" % str(exc)
+ msg = STANDARD_CFG_LOAD_FAILED_MSG % util.sysTools.getFileErrorMsg(exc)
util.log.log(util.log.WARN, msg)
else:
# no armrc found, falling back to the defaults in the source
- msg = "No configuration found at '%s', using defaults" % configPath
+ msg = STANDARD_CFG_NOT_FOUND_MSG % configPath
util.log.log(util.log.NOTICE, msg)
# revises defaults to match user's configuration
- config.update(DEFAULTS)
+ config.update(CONFIG)
# loads user preferences for utilities
for utilModule in (util.conf, util.connections, util.hostnames, util.log, util.panel, util.sysTools, util.torConfig, util.torTools, util.uiTools):
@@ -155,7 +223,7 @@
# overwrites undefined parameters with defaults
for key in param.keys():
- if param[key] == None: param[key] = DEFAULTS[key]
+ if param[key] == None: param[key] = CONFIG[key]
# validates that input has a valid ip address and port
controlAddr = param["startup.interface.ipAddress"]
@@ -182,22 +250,13 @@
# sets up TorCtl connection, prompting for the passphrase if necessary and
# sending problems to stdout if they arise
TorCtl.INCORRECT_PASSWORD_MSG = "Controller password found in '%s' was incorrect" % configPath
- authPassword = config.get("startup.controlPassword", DEFAULTS["startup.controlPassword"])
+ authPassword = config.get("startup.controlPassword", CONFIG["startup.controlPassword"])
conn = TorCtl.TorCtl.connect(controlAddr, controlPort, authPassword)
if conn == None: sys.exit(1)
- # It is important that this is loaded before entering the curses context,
- # otherwise the man call pegs the cpu for around a minute (I'm not sure
- # why... curses must mess the terminal in a way that's important to man).
+ # fetches descriptions for tor's configuration options
+ _loadConfigurationDescriptions()
- # TODO: Moving into an async call isn't helping with the startup time. Next,
- # try caching the parsed results to disk (idea by nickm).
- #import threading
- #t = threading.Thread(target = util.torConfig.loadOptionDescriptions)
- #t.setDaemon(True)
- #t.start()
- util.torConfig.loadOptionDescriptions()
-
# initializing the connection may require user input (for the password)
# scewing the startup time results so this isn't counted
initTime = time.time() - startTime
Modified: arm/trunk/src/util/__init__.py
===================================================================
--- arm/trunk/src/util/__init__.py 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/src/util/__init__.py 2010-10-30 05:54:20 UTC (rev 23723)
@@ -4,5 +4,5 @@
and safely working with curses (hiding some of the gory details).
"""
-__all__ = ["conf", "connections", "hostnames", "log", "panel", "sysTools", "torTools", "uiTools"]
+__all__ = ["conf", "connections", "hostnames", "log", "panel", "sysTools", "torConfig", "torTools", "uiTools"]
Modified: arm/trunk/src/util/sysTools.py
===================================================================
--- arm/trunk/src/util/sysTools.py 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/src/util/sysTools.py 2010-10-30 05:54:20 UTC (rev 23723)
@@ -54,6 +54,26 @@
CMD_AVAILABLE_CACHE[command] = cmdExists
return cmdExists
+def getFileErrorMsg(exc):
+ """
+ Strips off the error number prefix for file related IOError messages. For
+ instance, instead of saying:
+ [Errno 2] No such file or directory
+
+ this would return:
+ no such file or directory
+
+ Arguments:
+ exc - file related IOError exception
+ """
+
+ excStr = str(exc)
+ if excStr.startswith("[Errno ") and "] " in excStr:
+ excStr = excStr[excStr.find("] ") + 2:].strip()
+ excStr = excStr[0].lower() + excStr[1:]
+
+ return excStr
+
def call(command, cacheAge=0, suppressExc=False, quiet=True):
"""
Convenience function for performing system calls, providing:
Modified: arm/trunk/src/util/torConfig.py
===================================================================
--- arm/trunk/src/util/torConfig.py 2010-10-29 17:10:00 UTC (rev 23722)
+++ arm/trunk/src/util/torConfig.py 2010-10-30 05:54:20 UTC (rev 23723)
@@ -2,6 +2,7 @@
Helper functions for working with tor's configuration file.
"""
+import os
import curses
import threading
@@ -38,6 +39,7 @@
TORRC = None # singleton torrc instance
MAN_OPT_INDENT = 7 # indentation before options in the man page
MAN_EX_INDENT = 15 # indentation used for man page examples
+PERSIST_ENTRY_DIVIDER = "-" * 80 + "\n" # splits config entries when saving to a file
def loadConfig(config):
CONFIG["torrc.multiline"] = config.get("torrc.multiline", [])
@@ -59,11 +61,19 @@
if TORRC == None: TORRC = Torrc()
return TORRC
-def loadOptionDescriptions():
+def loadOptionDescriptions(loadPath = None):
"""
Fetches and parses descriptions for tor's configuration options from its man
page. This can be a somewhat lengthy call, and raises an IOError if issues
occure.
+
+ If available, this can load the configuration descriptions from a file where
+ they were previously persisted to cut down on the load time (latency for this
+ is around 200ms).
+
+ Arguments:
+ loadPath - if set, this attempts to fetch the configuration descriptions
+ from the given path instead of the man page
"""
CONFIG_DESCRIPTIONS_LOCK.acquire()
@@ -71,52 +81,105 @@
raisedExc = None
try:
- manCallResults = sysTools.call("man tor")
-
- lastOption, lastArg = None, None
- lastDescription = ""
- for line in manCallResults:
- strippedLine = line.strip()
+ if loadPath:
+ # Input file is expected to be of the form:
+ # <option>
+ # <arg description>
+ # <description, possibly multiple lines>
+ # <PERSIST_ENTRY_DIVIDER>
+ inputFile = open(loadPath, "r")
+ inputFileContents = inputFile.readlines()
+ inputFile.close()
- # we have content, but an indent less than an option (ignore line)
- if strippedLine and not line.startswith(" " * MAN_OPT_INDENT): continue
+ try:
+ while inputFileContents:
+ option = inputFileContents.pop(0).rstrip()
+ argument = inputFileContents.pop(0).rstrip()
+
+ description, loadedLine = "", inputFileContents.pop(0)
+ while loadedLine != PERSIST_ENTRY_DIVIDER:
+ description += loadedLine
+
+ if inputFileContents: loadedLine = inputFileContents.pop(0)
+ else: break
+
+ CONFIG_DESCRIPTIONS[option] = (argument, description.rstrip())
+ except IndexError:
+ CONFIG_DESCRIPTIONS.clear()
+ raise IOError("input file format is invalid")
+ else:
+ manCallResults = sysTools.call("man tor")
- # line starts with an indent equivilant to a new config option
- isOptIndent = line.startswith(" " * MAN_OPT_INDENT) and line[MAN_OPT_INDENT] != " "
-
- if isOptIndent:
- # Most lines with this indent that aren't config options won't have
- # any description set at this point (not a perfect filter, but cuts
- # down on the noise).
- strippedDescription = lastDescription.strip()
- if lastOption and strippedDescription:
- CONFIG_DESCRIPTIONS[lastOption] = (lastArg, strippedDescription)
- lastDescription = ""
+ lastOption, lastArg = None, None
+ lastDescription = ""
+ for line in manCallResults:
+ strippedLine = line.strip()
- # parses the option and argument
- line = line.strip()
- divIndex = line.find(" ")
- if divIndex != -1:
- lastOption, lastArg = line[:divIndex], line[divIndex + 1:]
- else:
- # Appends the text to the running description. Empty lines and lines
- # starting with a specific indentation are used for formatting, for
- # instance the ExitPolicy and TestingTorNetwork entries.
- if lastDescription and lastDescription[-1] != "\n":
- lastDescription += " "
+ # we have content, but an indent less than an option (ignore line)
+ if strippedLine and not line.startswith(" " * MAN_OPT_INDENT): continue
- if not strippedLine:
- lastDescription += "\n\n"
- elif line.startswith(" " * MAN_EX_INDENT):
- lastDescription += " %s\n" % strippedLine
- else: lastDescription += strippedLine
-
+ # line starts with an indent equivilant to a new config option
+ isOptIndent = line.startswith(" " * MAN_OPT_INDENT) and line[MAN_OPT_INDENT] != " "
+
+ if isOptIndent:
+ # Most lines with this indent that aren't config options won't have
+ # any description set at this point (not a perfect filter, but cuts
+ # down on the noise).
+ strippedDescription = lastDescription.strip()
+ if lastOption and strippedDescription:
+ CONFIG_DESCRIPTIONS[lastOption] = (lastArg, strippedDescription)
+ lastDescription = ""
+
+ # parses the option and argument
+ line = line.strip()
+ divIndex = line.find(" ")
+ if divIndex != -1:
+ lastOption, lastArg = line[:divIndex], line[divIndex + 1:]
+ else:
+ # Appends the text to the running description. Empty lines and lines
+ # starting with a specific indentation are used for formatting, for
+ # instance the ExitPolicy and TestingTorNetwork entries.
+ if lastDescription and lastDescription[-1] != "\n":
+ lastDescription += " "
+
+ if not strippedLine:
+ lastDescription += "\n\n"
+ elif line.startswith(" " * MAN_EX_INDENT):
+ lastDescription += " %s\n" % strippedLine
+ else: lastDescription += strippedLine
except IOError, exc:
raisedExc = exc
CONFIG_DESCRIPTIONS_LOCK.release()
if raisedExc: raise raisedExc
+def saveOptionDescriptions(path):
+ """
+ Preserves the current configuration descriptors to the given path. This
+ raises an IOError if unable to do so.
+
+ Arguments:
+ path - location to persist configuration descriptors
+ """
+
+ # make dir if the path doesn't already exist
+ baseDir = os.path.dirname(path)
+ if not os.path.exists(baseDir): os.makedirs(baseDir)
+ outputFile = open(path, "w")
+
+ CONFIG_DESCRIPTIONS_LOCK.acquire()
+ sortedOptions = CONFIG_DESCRIPTIONS.keys()
+ sortedOptions.sort()
+
+ for i in range(len(sortedOptions)):
+ option = sortedOptions[i]
+ argument, description = getConfigDescription(option)
+ outputFile.write("%s\n%s\n%s\n" % (option, argument, description))
+ if i != len(sortedOptions) - 1: outputFile.write(PERSIST_ENTRY_DIVIDER)
+
+ outputFile.close()
+ CONFIG_DESCRIPTIONS_LOCK.release()
+
def getConfigDescription(option):
"""
Provides a tuple with arguments and description for the given tor
@@ -355,7 +418,7 @@
self.valsLock.acquire()
returnVal = list(self.contents) if self.contents else None
- self.valsLock.relese()
+ self.valsLock.release()
return returnVal
def getDisplayContents(self, strip = False):
More information about the tor-commits
mailing list