[or-cvs] r22799: {arm} Moving additional functionality from the interface to torToo (in arm/trunk: . interface util)
Damian Johnson
atagar1 at gmail.com
Wed Aug 4 03:53:24 UTC 2010
Author: atagar
Date: 2010-08-04 03:53:24 +0000 (Wed, 04 Aug 2010)
New Revision: 22799
Modified:
arm/trunk/TODO
arm/trunk/armrc.sample
arm/trunk/interface/controller.py
arm/trunk/util/torTools.py
Log:
Moving additional functionality from the interface to torTools utility.
- Moving heartbeat functionality out of the log panel. This is in preparation for the panel's rewrite.
- Now that torTools.Controller is making use of several event types it makes more sense for its setControllerEvents method's mandatory events listing to live here. This also simplifies /interface/controller.py a little.
Modified: arm/trunk/TODO
===================================================================
--- arm/trunk/TODO 2010-08-04 01:17:41 UTC (rev 22798)
+++ arm/trunk/TODO 2010-08-04 03:53:24 UTC (rev 22799)
@@ -16,6 +16,8 @@
sorting, etc
- provide notice if tor supports events that arm doesn't
getInfo("events/names") provides the space-separated listing
+ - check what events TorCtl can provide us, and give notice if any are
+ missing
[ ] conf panel
- move torrc validation into util
- condense tor/arm log listing types if they're the same
@@ -35,6 +37,10 @@
- connection uptime to associate inbound/outbound connections?
- Identify controller connections (if it's arm, vidalia, etc) with
special detail page for them
+ - provide bridge / client country statistics
+ Include bridge related data via GETINFO option (feature request
+ by waltman and ioerror).
+ - Country data for client connections (requested by ioerror)
[-] controller (for version 1.3.8)
[ ] provide performance ARM-DEBUG events
Help with diagnosing performance bottlenecks. This is pending the
@@ -124,7 +130,7 @@
* rdns and whois lookups
To avoid disclosing connection data to third parties this needs to be
an all-or-nothing operation (ie, needs to fetch information on all
- relays or none of them. Plan is something like:
+ relays or none of them). Plan is something like:
* add resolving/caching capabilities to fetch information on all relays
and distil whois entries to just what we care about (hosting provider
or ISP), by default updating the cache on a daily basis
@@ -146,10 +152,6 @@
http://www.codexon.com/posts/clearing-passwords-in-memory-with-python
* escaping function for uiTools' formatted strings
* tor-weather like functionality (email notices)
- * provide bridge / client country statistics
- - Include bridge related data via GETINFO option (feature request by
- waltman).
- - Country data for client connections (requested by ioerror)
* make update rates configurable via the ui
Also provide option for saving these settings to the config
* config option to cap resource usage
Modified: arm/trunk/armrc.sample
===================================================================
--- arm/trunk/armrc.sample 2010-08-04 01:17:41 UTC (rev 22798)
+++ arm/trunk/armrc.sample 2010-08-04 03:53:24 UTC (rev 22799)
@@ -7,6 +7,20 @@
features.colorInterface true
+# log panel parameters
+# prepopulate: attempts to read past events from the log file if true
+# prepopulateAddLimit: maximum entries added from the log file
+# prepopulateReadLimit: maximum entries read from the log file
+#
+# Limits are to prevent big log files from causing a slow startup time. For
+# instance, if arm's only listening for ERR entries but the log has all
+# runlevels then this will add the first <prepopulateAddLimit> ERR entries and
+# stop reading after <prepopulateReadLimit> lines
+
+features.log.prepopulate true
+features.log.prepopulateAddLimit 1000
+features.log.prepopulateReadLimit 5000
+
# general graph parameters
# interval: 0 -> each second, 1 -> 5 seconds, 2 -> 30 seconds,
# 3 -> minutely, 4 -> half hour, 5 -> hourly, 6 -> daily
@@ -55,6 +69,7 @@
cache.sysCalls.size 600
cache.hostnames.size 700000
cache.hostnames.trimSize 200000
+cache.logPanel.size 1000
cache.armLog.size 1000
cache.armLog.trimSize 200
Modified: arm/trunk/interface/controller.py
===================================================================
--- arm/trunk/interface/controller.py 2010-08-04 01:17:41 UTC (rev 22798)
+++ arm/trunk/interface/controller.py 2010-08-04 03:53:24 UTC (rev 22799)
@@ -265,26 +265,10 @@
for eventType in events:
if eventType not in logPanel.TOR_EVENT_TYPES.values(): toDiscard += [eventType]
- for eventType in list(toDiscard):
- events.discard(eventType)
+ for eventType in list(toDiscard): events.discard(eventType)
- # makes a mapping instead
- events = dict([(eventType, None) for eventType in events])
+ setEvents = torTools.getConn().setControllerEvents(list(events))
- # add mandatory events (those needed for arm functionaity)
- reqEvents = {"BW": "(bandwidth graph won't function)",
- "NEWDESC": "(information related to descriptors will grow stale)",
- "NS": "(information related to the consensus will grow stale)",
- "NEWCONSENSUS": "(information related to the consensus will grow stale)"}
-
- if not isBlindMode:
- reqEvents["CIRC"] = "(may cause issues in identifying client connections)"
-
- for eventType, msg in reqEvents.items():
- events[eventType] = (log.ERR, "Unsupported event type: %s %s" % (eventType, msg))
-
- setEvents = torTools.getConn().setControllerEvents(events)
-
# temporary hack for providing user selected events minus those that failed
# (wouldn't be a problem if I wasn't storing tor and non-tor events together...)
returnVal = list(selectedEvents.difference(torTools.FAILED_EVENTS))
@@ -325,6 +309,14 @@
config.update(CONFIG)
config.update(graphing.graphPanel.CONFIG)
+ # adds events needed for arm functionality to the torTools REQ_EVENTS mapping
+ # (they're then included with any setControllerEvents call, and log a more
+ # helpful error if unavailable)
+ torTools.REQ_EVENTS["BW"] = "bandwidth graph won't function"
+
+ if not isBlindMode:
+ torTools.REQ_EVENTS["CIRC"] = "may cause issues in identifying client connections"
+
# pauses/unpauses connection resolution according to if tor's connected or not
torTools.getConn().addStatusListener(connResetListener)
@@ -497,14 +489,17 @@
panels[panelKey].setTop(tmpStartY)
tmpStartY += panels[panelKey].getHeight()
- # if it's been at least ten seconds since the last BW event Tor's probably done
- if not isUnresponsive and not panels["log"].controlPortClosed and panels["log"].getHeartbeat() >= 10:
- isUnresponsive = True
- log.log(log.NOTICE, "Relay unresponsive (last heartbeat: %s)" % time.ctime(panels["log"].lastHeartbeat))
- elif not panels["log"].controlPortClosed and (isUnresponsive and panels["log"].getHeartbeat() < 10):
- # shouldn't happen unless Tor freezes for a bit - BW events happen every second...
- isUnresponsive = False
- log.log(log.NOTICE, "Relay resumed")
+ # provides a notice if there's been ten seconds since the last BW event
+ if torTools.getConn().isAlive():
+ lastHeartbeat = torTools.getConn().getHeartbeat()
+
+ if not isUnresponsive and (time.time() - lastHeartbeat) >= 10:
+ isUnresponsive = True
+ log.log(log.NOTICE, "Relay unresponsive (last heartbeat: %s)" % time.ctime(lastHeartbeat))
+ elif isUnresponsive and (time.time() - lastHeartbeat) < 10:
+ # really shouldn't happen (meant Tor froze for a bit)
+ isUnresponsive = False
+ log.log(log.NOTICE, "Relay resumed")
panels["conn"].reset()
Modified: arm/trunk/util/torTools.py
===================================================================
--- arm/trunk/util/torTools.py 2010-08-04 01:17:41 UTC (rev 22798)
+++ arm/trunk/util/torTools.py 2010-08-04 03:53:24 UTC (rev 22799)
@@ -16,7 +16,7 @@
import thread
import threading
-from TorCtl import TorCtl
+from TorCtl import TorCtl, TorUtil
import log
import sysTools
@@ -26,9 +26,8 @@
# TOR_CLOSED - control port closed
TOR_INIT, TOR_CLOSED = range(1, 3)
-# Message logged by default when a controller event type can't be set (message
-# has the event type inserted into it). This skips logging entirely if None.
-DEFAULT_FAILED_EVENT_ENTRY = (log.WARN, "Unsupported event type: %s")
+# message logged by default when a controller can't set an event type
+DEFAULT_FAILED_EVENT_MSG = "Unsupported event type: %s"
# TODO: check version when reattaching to controller and if version changes, flush?
# Skips attempting to set events we've failed to set before. This avoids
@@ -45,9 +44,20 @@
CACHE_ARGS = ("nsEntry", "descEntry", "bwRate", "bwBurst", "bwObserved",
"bwMeasured", "flags", "fingerprint", "pid")
+TOR_CTL_CLOSE_MSG = "Tor closed control connection. Exiting event thread."
UNKNOWN = "UNKNOWN" # value used by cached information if undefined
CONFIG = {"log.torGetInfo": log.DEBUG, "log.torGetConf": log.DEBUG}
+# events used for controller functionality:
+# BW - used to check for a periodic heartbeat
+# NOTICE - used to detect when tor is shut down
+# NEWDESC, NS, and NEWCONSENSUS - used for cache invalidation
+REQ_EVENTS = {"BW": "unable to check for a periodic heartbeat",
+ "NOTICE": "this will be unable to detect when tor is shut down",
+ "NEWDESC": "information related to descriptors will grow stale",
+ "NS": "information related to the consensus will grow stale",
+ "NEWCONSENSUS": "information related to the consensus will grow stale"}
+
def loadConfig(config):
config.update(CONFIG)
@@ -277,14 +287,20 @@
self.conn = None # None if uninitialized or controller's been closed
self.connLock = threading.RLock()
self.eventListeners = [] # instances listening for tor controller events
+ self.torctlListeners = [] # callback functions for TorCtl events
self.statusListeners = [] # callback functions for tor's state changes
- self.controllerEvents = {} # mapping of successfully set controller events to their failure level/msg
+ self.controllerEvents = [] # list of successfully set controller events
self._isReset = False # internal flag for tracking resets
self._status = TOR_CLOSED # current status of the attached control port
self._statusTime = 0 # unix time-stamp for the duration of the status
+ self.lastHeartbeat = 0 # time of the last bw event
# cached getInfo parameters (None if unset or possibly changed)
self._cachedParam = dict([(arg, "") for arg in CACHE_ARGS])
+
+ # directs TorCtl to notify us of events
+ TorUtil.loglevel = "DEBUG"
+ TorUtil.logfile = self
def init(self, conn=None):
"""
@@ -355,6 +371,15 @@
self.connLock.release()
return result
+ def getHeartbeat(self):
+ """
+ Provides the time of the last registered BW event (this should occure every
+ second if relay's still responsive). This returns zero if there has never
+ been an attached tor instance.
+ """
+
+ return self.lastHeartbeat
+
def getTorCtl(self):
"""
Provides the current TorCtl connection. If unset or closed then this
@@ -572,6 +597,18 @@
if self.isAlive(): self.conn.add_event_listener(listener)
self.connLock.release()
+ def addTorCtlListener(self, callback):
+ """
+ Directs further TorCtl events to the callback function. Events are composed
+ of a runlevel and message tuple.
+
+ Arguments:
+ callback - functor that'll accept the events, expected to be of the form:
+ myFunction(runlevel, msg)
+ """
+
+ self.torctlListeners.append(callback)
+
def addStatusListener(self, callback):
"""
Directs further events related to tor's controller status to the callback
@@ -598,27 +635,31 @@
return True
else: return False
- def setControllerEvents(self, eventsToMsg):
+ def setControllerEvents(self, events):
"""
- Sets the events being provided via any associated tor controller, logging
- messages for event types that aren't supported (possibly due to version
- issues). This remembers the successfully set events and tries to apply them
- to any controllers attached later too (again logging and dropping
- unsuccessful event types). This returns the listing of event types that
- were successfully set. If no controller is available or events can't be set
- then this is a no-op.
+ Sets the events being requested from any attached tor instance, logging
+ warnings for event types that aren't supported (possibly due to version
+ issues). Events in REQ_EVENTS will also be included, logging at the error
+ level with an additional description in case of failure.
+ This remembers the successfully set events and tries to request them from
+ any tor instance it attaches to in the future too (again logging and
+ dropping unsuccessful event types).
+
+ This returns the listing of event types that were successfully set. If not
+ currently attached to a tor instance then all events are assumed to be ok,
+ then attempted when next attached to a control port.
+
Arguments:
- eventsToMsg - mapping of event types to a tuple of the (runlevel, msg) it
- should log in case of failure (uses DEFAULT_FAILED_EVENT_ENTRY
- if mapped to None)
+ events - listing of events to be set
"""
self.connLock.acquire()
returnVal = []
if self.isAlive():
- events = set(eventsToMsg.keys())
+ events = set(events)
+ events = events.union(set(REQ_EVENTS.keys()))
unavailableEvents = set()
# removes anything we've already failed to set
@@ -626,7 +667,8 @@
unavailableEvents.update(events.intersection(FAILED_EVENTS))
events.difference_update(FAILED_EVENTS)
- # initial check for event availability
+ # initial check for event availability, using the 'events/names' GETINFO
+ # option to detect invalid events
validEvents = self.getInfo("events/names")
if validEvents:
@@ -634,7 +676,7 @@
unavailableEvents.update(events.difference(validEvents))
events.intersection_update(validEvents)
- # attempt to set events
+ # attempt to set events via trial and error
isEventsSet, isAbandoned = False, False
while not isEventsSet and not isAbandoned:
@@ -661,19 +703,20 @@
FAILED_EVENTS.update(unavailableEvents)
if not isAbandoned:
- # removes failed events and logs warnings
+ # logs warnings or errors for failed events
for eventType in unavailableEvents:
- if eventsToMsg[eventType]:
- lvl, msg = eventsToMsg[eventType]
- log.log(lvl, msg)
- elif DEFAULT_FAILED_EVENT_ENTRY:
- lvl, msg = DEFAULT_FAILED_EVENT_ENTRY
- log.log(lvl, msg % eventType)
-
- del eventsToMsg[eventType]
+ defaultMsg = DEFAULT_FAILED_EVENT_MSG % eventType
+ if eventType in REQ_EVENTS:
+ log.log(log.ERR, defaultMsg + " (%s)" % REQ_EVENTS[eventType])
+ else:
+ log.log(log.WARN, defaultMsg)
- self.controllerEvents = eventsToMsg
- returnVal = eventsToMsg.keys()
+ self.controllerEvents = list(events)
+ returnVal = list(events)
+ else:
+ # attempts to set the events when next attached to a control port
+ self.controllerEvents = list(events)
+ returnVal = list(events)
self.connLock.release()
return returnVal
@@ -765,6 +808,9 @@
thread.start_new_thread(self._notifyStatusListeners, (TOR_INIT,))
+ def bandwidth_event(self, event):
+ self.lastHeartbeat = time.time()
+
def ns_event(self, event):
myFingerprint = self.getMyFingerprint()
if myFingerprint:
@@ -790,6 +836,23 @@
self._cachedParam["descEntry"] = None
self._cachedParam["bwObserved"] = None
+ def write(self, msg):
+ """
+ Tracks TorCtl events. Ugly hack since TorCtl/TorUtil.py expects a file.
+ """
+
+ timestampStart, timestampEnd = msg.find("["), msg.find("]")
+ level = msg[:timestampStart]
+ msg = msg[timestampEnd + 2:].strip()
+
+ # notifies listeners of TorCtl events
+ for callback in self.torctlListeners: callback(level, msg)
+
+ # checks if TorCtl is providing a notice that control port is closed
+ if TOR_CTL_CLOSE_MSG in msg: self.close()
+
+ def flush(self): pass
+
def _getRelayAttr(self, key, default, cacheUndefined = True):
"""
Provides information associated with this relay, using the cached value if
More information about the tor-commits
mailing list