commit 7a5131d401a3981e16dc5c6056afbb0067ef36bb Author: Damian Johnson atagar@torproject.org Date: Mon Sep 23 00:10:07 2013 -0700

Using stem's connection resolution

Dropping use of our connection resolvers in favor of stem's. Unfortunately there's still quite a bit more work needed to get rid of this module... --- arm/logPanel.py | 4 +- arm/util/connections.py | 205 +++-------------------------------------------- 2 files changed, 10 insertions(+), 199 deletions(-)

diff --git a/arm/logPanel.py b/arm/logPanel.py index 101a7e8..b5ee962 100644 --- a/arm/logPanel.py +++ b/arm/logPanel.py @@ -566,9 +566,7 @@ class LogPanel(panel.Panel, threading.Thread, logging.Handler): stem_logger.addHandler(self)

def emit(self, record): - if record.levelname == "ERROR": - record.levelname = "ERR" - elif record.levelname == "WARNING": + if record.levelname == "WARNING": record.levelname = "WARN"

eventColor = RUNLEVEL_EVENT_COLOR[record.levelname] diff --git a/arm/util/connections.py b/arm/util/connections.py index bdb3c3f..d43a85d 100644 --- a/arm/util/connections.py +++ b/arm/util/connections.py @@ -22,55 +22,13 @@ import os import time import threading

-from stem.util import conf, enum, log, proc, system - -# enums for connection resolution utilities -Resolver = enum.Enum(("PROC", "proc"), - ("NETSTAT", "netstat"), - ("SS", "ss"), - ("LSOF", "lsof"), - ("SOCKSTAT", "sockstat"), - ("BSD_SOCKSTAT", "sockstat (bsd)"), - ("BSD_PROCSTAT", "procstat (bsd)")) +from stem.util import conf, connection, enum, log, proc, system

# If true this provides new instantiations for resolvers if the old one has # been stopped. This can make it difficult ensure all threads are terminated # when accessed concurrently. RECREATE_HALTED_RESOLVERS = False

-# formatted strings for the commands to be executed with the various resolvers -# options are: -# n = prevents dns lookups, p = include process -# output: -# tcp 0 0 127.0.0.1:9051 127.0.0.1:53308 ESTABLISHED 9912/tor -# *note: bsd uses a different variant ('-t' => '-p tcp', but worse an -# equivilant -p doesn't exist so this can't function) -RUN_NETSTAT = "netstat -np" - -# n = numeric ports, p = include process, t = tcp sockets, u = udp sockets -# output: -# ESTAB 0 0 127.0.0.1:9051 127.0.0.1:53308 users:(("tor",9912,20)) -# *note: under freebsd this command belongs to a spreadsheet program -RUN_SS = "ss -nptu" - -# n = prevent dns lookups, P = show port numbers (not names), i = ip only, -# -w = no warnings -# output: -# tor 3873 atagar 45u IPv4 40994 0t0 TCP 10.243.55.20:45724->194.154.227.109:9001 (ESTABLISHED) -# -# oddly, using the -p flag via: -# lsof lsof -nPi -p <pid> | grep "^<process>.*(ESTABLISHED)" -# is much slower (11-28% in tests I ran) -RUN_LSOF = "lsof -wnPi" - -# output: -# atagar tor 3475 tcp4 127.0.0.1:9051 127.0.0.1:38942 ESTABLISHED -# *note: this isn't available by default under ubuntu -RUN_SOCKSTAT = "sockstat" - -RUN_BSD_SOCKSTAT = "sockstat -4c" -RUN_BSD_PROCSTAT = "procstat -f %s" - RESOLVERS = [] # connection resolvers available via the singleton constructor RESOLVER_FAILURE_TOLERANCE = 3 # number of subsequent failures before moving on to another resolver RESOLVER_SERIAL_FAILURE_MSG = "Unable to query connections with %s, trying %s" @@ -118,130 +76,6 @@ def getPortUsage(port):

return PORT_USAGE.get(port)

-def getResolverCommand(resolutionCmd, processName, processPid = ""): - """ - Provides the command and line filter that would be processed for the given - resolver type. This raises a ValueError if either the resolutionCmd isn't - recognized or a pid was requited but not provided. - - Arguments: - resolutionCmd - command to use in resolving the address - processName - name of the process for which connections are fetched - processPid - process ID (this helps improve accuracy) - """ - - if not processPid: - # the pid is required for procstat resolution - if resolutionCmd == Resolver.BSD_PROCSTAT: - raise ValueError("procstat resolution requires a pid") - - # if the pid was undefined then match any in that field - processPid = "[0-9]*" - - no_op_filter = lambda line: True - - if resolutionCmd == Resolver.PROC: return ("", no_op_filter) - elif resolutionCmd == Resolver.NETSTAT: - return ( - RUN_NETSTAT, - lambda line: "ESTABLISHED %s/%s" % (processPid, processName) in line - ) - elif resolutionCmd == Resolver.SS: - return ( - RUN_SS, - lambda line: ("ESTAB" in line) and (""%s",%s" % (processName, processPid) in line) - ) - elif resolutionCmd == Resolver.LSOF: - return ( - RUN_LSOF, - lambda line: re.match("^%s *%s.*((UDP.*)|((ESTABLISHED)))" % (processName, processPid)) - ) - elif resolutionCmd == Resolver.SOCKSTAT: - return ( - RUN_SOCKSTAT, - lambda line: re.match("%s *%s.*ESTABLISHED" % (processName, processPid)) - ) - elif resolutionCmd == Resolver.BSD_SOCKSTAT: - return ( - RUN_BSD_SOCKSTAT, - lambda line: re.match("%s *%s" % (processName, processPid)) - ) - elif resolutionCmd == Resolver.BSD_PROCSTAT: - return ( - RUN_BSD_PROCSTAT % processPid, - lambda line: "TCP" in line and "0.0.0.0:0" not in line - ) - else: raise ValueError("Unrecognized resolution type: %s" % resolutionCmd) - -def getConnections(resolutionCmd, processName, processPid = ""): - """ - Retrieves a list of the current connections for a given process, providing a - tuple list of the form: - [(local_ipAddr1, local_port1, foreign_ipAddr1, foreign_port1), ...] - this raises an IOError if no connections are available or resolution fails - (in most cases these appear identical). Common issues include: - - insufficient permissions - - resolution command is unavailable - - usage of the command is non-standard (particularly an issue for BSD) - - Arguments: - resolutionCmd - command to use in resolving the address - processName - name of the process for which connections are fetched - processPid - process ID (this helps improve accuracy) - """ - - if resolutionCmd == Resolver.PROC: - # Attempts resolution via checking the proc contents. - if not processPid: - raise ValueError("proc resolution requires a pid") - - try: - return proc.get_connections(processPid) - except Exception, exc: - raise IOError(str(exc)) - else: - # Queries a resolution utility (netstat, lsof, etc). This raises an - # IOError if the command fails or isn't available. - cmd, cmd_filter = getResolverCommand(resolutionCmd, processName, processPid) - results = system.call(cmd) - results = filter(cmd_filter, results) - - if not results: raise IOError("No results found using: %s" % cmd) - - # parses results for the resolution command - conn = [] - for line in results: - if resolutionCmd == Resolver.LSOF: - # Different versions of lsof have different numbers of columns, so - # stripping off the optional 'established' entry so we can just use - # the last one. - comp = line.replace("(ESTABLISHED)", "").strip().split() - else: comp = line.split() - - if resolutionCmd == Resolver.NETSTAT: - localIp, localPort = comp[3].split(":") - foreignIp, foreignPort = comp[4].split(":") - elif resolutionCmd == Resolver.SS: - localIp, localPort = comp[4].split(":") - foreignIp, foreignPort = comp[5].split(":") - elif resolutionCmd == Resolver.LSOF: - local, foreign = comp[-1].split("->") - localIp, localPort = local.split(":") - foreignIp, foreignPort = foreign.split(":") - elif resolutionCmd == Resolver.SOCKSTAT: - localIp, localPort = comp[4].split(":") - foreignIp, foreignPort = comp[5].split(":") - elif resolutionCmd == Resolver.BSD_SOCKSTAT: - localIp, localPort = comp[5].split(":") - foreignIp, foreignPort = comp[6].split(":") - elif resolutionCmd == Resolver.BSD_PROCSTAT: - localIp, localPort = comp[9].split(":") - foreignIp, foreignPort = comp[10].split(":") - - conn.append((localIp, localPort, foreignIp, foreignPort)) - - return conn - def isResolverAlive(processName, processPid = ""): """ This provides true if a singleton resolver instance exists for the given @@ -289,30 +123,6 @@ def getResolver(processName, processPid = "", alias=None): else: RESOLVERS[haltedIndex] = r return r

-def getSystemResolvers(osType = None): - """ - Provides the types of connection resolvers available on this operating - system. - - Arguments: - osType - operating system type, fetched from the os module if undefined - """ - - if osType == None: osType = os.uname()[0] - - if osType == "FreeBSD": - resolvers = [Resolver.BSD_SOCKSTAT, Resolver.BSD_PROCSTAT, Resolver.LSOF] - elif osType in ("OpenBSD", "Darwin"): - resolvers = [Resolver.LSOF] - else: - resolvers = [Resolver.NETSTAT, Resolver.SOCKSTAT, Resolver.LSOF, Resolver.SS] - - # proc resolution, by far, outperforms the others so defaults to this is able - if proc.is_available(): - resolvers = [Resolver.PROC] + resolvers - - return resolvers - class ConnectionResolver(threading.Thread): """ Service that periodically queries for a process' current connections. This @@ -379,11 +189,11 @@ class ConnectionResolver(threading.Thread): self.defaultRate = CONFIG["queries.connections.minRate"] self.lastLookup = -1 self.overwriteResolver = None - self.defaultResolver = Resolver.PROC + self.defaultResolver = connection.Resolver.PROC

- osType = os.uname()[0] - self.resolverOptions = getSystemResolvers(osType) + self.resolverOptions = connection.get_system_resolvers()

+ osType = os.uname()[0] log.info("Operating System: %s, Connection Resolvers: %s" % (osType, ", ".join(self.resolverOptions)))

# sets the default resolver to be the first found in the system's PATH @@ -393,7 +203,7 @@ class ConnectionResolver(threading.Thread): # resolvers. resolverCmd = resolver.replace(" (bsd)", "")

- if resolver == Resolver.PROC or system.is_available(resolverCmd): + if resolver == connection.Resolver.PROC or system.is_available(resolverCmd): self.defaultResolver = resolver break

@@ -452,7 +262,10 @@ class ConnectionResolver(threading.Thread):

try: resolveStart = time.time() - connResults = getConnections(resolver, self.processName, self.processPid) + time.sleep(2) + from stem.util import log + connResults = [(conn.local_address, conn.local_port, conn.remote_address, conn.remote_port) for conn in connection.get_connections(resolver, process_pid = self.processPid, process_name = self.processName)] + lookupTime = time.time() - resolveStart

self._connections = connResults