commit 75ed2eef3750eaa8d6d8ba56c55b545e5a5b8053 Author: Damian Johnson atagar@torproject.org Date: Wed Jan 2 18:46:24 2013 -0800

Using stem's proc util

Stem's proc util is a direct counterpart for arm's. The only big difference is of course the tests. :) --- src/util/__init__.py | 2 +- src/util/connections.py | 8 +- src/util/procTools.py | 303 ----------------------------------------------- src/util/sysTools.py | 20 ++-- src/util/torTools.py | 14 +- 5 files changed, 22 insertions(+), 325 deletions(-)

diff --git a/src/util/__init__.py b/src/util/__init__.py index 9834bb1..16b28ee 100644 --- a/src/util/__init__.py +++ b/src/util/__init__.py @@ -4,5 +4,5 @@ application's status, making cross platform system calls, parsing tor data, and safely working with curses (hiding some of the gory details). """

-__all__ = ["connections", "hostnames", "panel", "procTools", "procName", "sysTools", "textInput", "torConfig", "torTools", "uiTools"] +__all__ = ["connections", "hostnames", "panel", "procName", "sysTools", "textInput", "torConfig", "torTools", "uiTools"]

diff --git a/src/util/connections.py b/src/util/connections.py index 5c66c1d..d633c03 100644 --- a/src/util/connections.py +++ b/src/util/connections.py @@ -21,9 +21,9 @@ import os import time import threading

-from util import procTools, sysTools +from util import sysTools

-from stem.util import conf, enum, log +from stem.util import conf, enum, log, proc

# enums for connection resolution utilities Resolver = enum.Enum(("PROC", "proc"), @@ -226,7 +226,7 @@ def getConnections(resolutionCmd, processName, processPid = ""): raise ValueError("proc resolution requires a pid")

try: - return procTools.getConnections(processPid) + return proc.get_connections(processPid) except Exception, exc: raise IOError(str(exc)) else: @@ -337,7 +337,7 @@ def getSystemResolvers(osType = None): resolvers = [Resolver.NETSTAT, Resolver.SOCKSTAT, Resolver.LSOF, Resolver.SS]

# proc resolution, by far, outperforms the others so defaults to this is able - if procTools.isProcAvailable(): + if proc.is_available(): resolvers = [Resolver.PROC] + resolvers

return resolvers diff --git a/src/util/procTools.py b/src/util/procTools.py deleted file mode 100644 index 8a895fe..0000000 --- a/src/util/procTools.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -Helper functions for querying process and system information from the /proc -contents. Fetching information this way provides huge performance benefits -over lookups via system utilities (ps, netstat, etc). For instance, resolving -connections this way cuts the runtime by around 90% verses the alternatives. -These functions may not work on all platforms (only Linux?). - -All functions raise IOErrors if unable to read their respective proc files. - -The method for reading these files (and some of the code) are borrowed from -psutil: -https://code.google.com/p/psutil/ -which was written by Jay Loden, Dave Daeschler, Giampaolo Rodola' and is under -the BSD license. -""" - -import os -import sys -import time -import socket -import base64 - -from stem.util import conf, enum, log - -# cached system values -SYS_START_TIME, SYS_PHYSICAL_MEMORY = None, None -CLOCK_TICKS = os.sysconf(os.sysconf_names["SC_CLK_TCK"]) -Stat = enum.Enum("COMMAND", "CPU_UTIME", "CPU_STIME", "START_TIME") - -CONFIG = conf.config_dict("arm", { - "queries.useProc": True, -}) - -def isProcAvailable(): - """ - Provides true if configured to use proc resolution and it's available on the - platform, false otherwise. - """ - - return CONFIG["queries.useProc"] and os.uname()[0] == "Linux" - -def getSystemStartTime(): - """ - Provides the unix time (seconds since epoch) when the system started. - """ - - global SYS_START_TIME - if not SYS_START_TIME: - startTime = time.time() - statFile = open('/proc/stat') - statLines = statFile.readlines() - statFile.close() - - for line in statLines: - if line.startswith('btime'): - SYS_START_TIME = float(line.strip().split()[1]) - break - - _logProcRuntime("system start time", "/proc/stat[btime]", startTime) - - return SYS_START_TIME - -def getPhysicalMemory(): - """ - Provides the total physical memory on the system in bytes. - """ - - global SYS_PHYSICAL_MEMORY - if not SYS_PHYSICAL_MEMORY: - startTime = time.time() - memFile = open('/proc/meminfo') - memLines = memFile.readlines() - memFile.close() - - for line in memLines: - if line.startswith('MemTotal:'): - SYS_PHYSICAL_MEMORY = int(line.split()[1]) * 1024 - - _logProcRuntime("system physical memory", "/proc/meminfo[MemTotal]", startTime) - - return SYS_PHYSICAL_MEMORY - -def getPwd(pid): - """ - Provides the current working directory for the given process. - - Arguments: - pid - queried process - """ - - startTime = time.time() - if pid == 0: cwd = "" - else: cwd = os.readlink("/proc/%s/cwd" % pid) - _logProcRuntime("cwd", "/proc/%s/cwd" % pid, startTime) - return cwd - -def getUid(pid): - """ - Provides the user ID the given process is running under. This is None if it - can't be determined. - - Arguments: - pid - queried process - """ - - startTime = time.time() - statusFile = open("/proc/%s/status" % pid) - statusFileLines = statusFile.readlines() - statusFile.close() - - result = None - for line in statusFileLines: - if line.startswith("Uid:"): - lineComp = line.split() - - if len(lineComp) >= 2 and lineComp[1].isdigit(): - result = lineComp[1] - - _logProcRuntime("uid", "/proc/%s/status[Uid]" % pid, startTime) - return result - -def getMemoryUsage(pid): - """ - Provides the memory usage in bytes for the given process of the form: - (residentSize, virtualSize) - - Arguments: - pid - queried process - """ - - # checks if this is the kernel process - if pid == 0: return (0, 0) - - startTime = time.time() - statusFile = open("/proc/%s/status" % pid) - statusFileLines = statusFile.readlines() - statusFile.close() - - residentSize, virtualSize = None, None - for line in statusFileLines: - if line.startswith("VmRSS"): - residentSize = int(line.split()[1]) * 1024 - if virtualSize != None: break - elif line.startswith("VmSize:"): - virtualSize = int(line.split()[1]) * 1024 - if residentSize != None: break - - _logProcRuntime("memory usage", "/proc/%s/status[VmRSS|VmSize]" % pid, startTime) - return (residentSize, virtualSize) - -def getStats(pid, *statTypes): - """ - Provides process specific information. Options are: - Stat.COMMAND command name under which the process is running - Stat.CPU_UTIME total user time spent on the process - Stat.CPU_STIME total system time spent on the process - Stat.START_TIME when this process began, in unix time - - Arguments: - pid - queried process - statTypes - information to be provided back - """ - - startTime = time.time() - statFilePath = "/proc/%s/stat" % pid - statFile = open(statFilePath) - statContents = statFile.read().strip() - statFile.close() - - # contents are of the form: - # 8438 (tor) S 8407 8438 8407 34818 8438 4202496... - statComp = [] - cmdStart, cmdEnd = statContents.find("("), statContents.find(")") - - if cmdStart != -1 and cmdEnd != -1: - statComp.append(statContents[:cmdStart]) - statComp.append(statContents[cmdStart + 1:cmdEnd]) - statComp += statContents[cmdEnd + 1:].split() - - if len(statComp) != 44: - raise IOError("stat file had an unexpected format: %s" % statFilePath) - - results, queriedStats = [], [] - for statType in statTypes: - if statType == Stat.COMMAND: - queriedStats.append("command") - if pid == 0: results.append("sched") - else: results.append(statComp[1]) - elif statType == Stat.CPU_UTIME: - queriedStats.append("utime") - if pid == 0: results.append("0") - else: results.append(str(float(statComp[13]) / CLOCK_TICKS)) - elif statType == Stat.CPU_STIME: - queriedStats.append("stime") - if pid == 0: results.append("0") - else: results.append(str(float(statComp[14]) / CLOCK_TICKS)) - elif statType == Stat.START_TIME: - queriedStats.append("start time") - if pid == 0: return getSystemStartTime() - else: - # According to documentation, starttime is in field 21 and the unit is - # jiffies (clock ticks). We divide it for clock ticks, then add the - # uptime to get the seconds since the epoch. - pStartTime = float(statComp[21]) / CLOCK_TICKS - results.append(str(pStartTime + getSystemStartTime())) - - _logProcRuntime("process %s" % ", ".join(queriedStats), "/proc/%s/stat" % pid, startTime) - return results - -def getConnections(pid): - """ - Provides a listing of connection tuples of the form: - [(local_ipAddr1, local_port1, foreign_ipAddr1, foreign_port1), ...] - - If the information about a connection can't be queried (often due to - permission issues) then it's excluded from the listing. - - Arguments: - pid - ID of the process to be resolved - """ - - if pid == "0": return [] - - # fetches the inode numbers for socket file descriptors - startTime = time.time() - inodes = [] - for fd in os.listdir("/proc/%s/fd" % pid): - try: - # File descriptor link, such as 'socket:[30899]' - fdName = os.readlink("/proc/%s/fd/%s" % (pid, fd)) - - if fdName.startswith('socket:['): - inodes.append(fdName[8:-1]) - except OSError: - pass # most likely couldn't be read due to permissions - - if not inodes: - # unable to fetch any connections for this process - return [] - - # check for the connection information from the /proc/net contents - conn = [] - for procFilePath in ("/proc/net/tcp", "/proc/net/udp"): - procFile = open(procFilePath) - procFile.readline() # skip the first line - - for line in procFile: - _, lAddr, fAddr, status, _, _, _, _, _, inode = line.split()[:10] - - if inode in inodes: - # if a tcp connection, skip if it isn't yet established - if procFilePath.endswith("/tcp") and status != "01": - continue - - localIp, localPort = _decodeProcAddressEncoding(lAddr) - foreignIp, foreignPort = _decodeProcAddressEncoding(fAddr) - conn.append((localIp, localPort, foreignIp, foreignPort)) - - procFile.close() - - _logProcRuntime("process connections", "/proc/net/[tcp|udp]", startTime) - - return conn - -def _decodeProcAddressEncoding(addr): - """ - Translates an address entry in the /proc/net/* contents to a human readable - form, for instance: - "0500000A:0016" -> ("10.0.0.5", "22") - - Reference: - http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html - - Arguments: - addr - proc address entry to be decoded - """ - - ip, port = addr.split(':') - - # the port is represented as a two-byte hexadecimal number - port = str(int(port, 16)) - - if sys.version_info >= (3,): - ip = ip.encode('ascii') - - # The IPv4 address portion is a little-endian four-byte hexadecimal number. - # That is, the least significant byte is listed first, so we need to reverse - # the order of the bytes to convert it to an IP address. - # - # This needs to account for the endian ordering as per... - # http://code.google.com/p/psutil/issues/detail?id=201 - # https://trac.torproject.org/projects/tor/ticket/4777 - - if sys.byteorder == 'little': - ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)[::-1]) - else: - ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)) - - return (ip, port) - -def _logProcRuntime(parameter, procLocation, startTime): - log.debug("proc call (%s): %s (runtime: %0.4f)" % (parameter, procLocation, time.time() - startTime)) - diff --git a/src/util/sysTools.py b/src/util/sysTools.py index 48bb9ed..2be73fd 100644 --- a/src/util/sysTools.py +++ b/src/util/sysTools.py @@ -6,9 +6,9 @@ import os import time import threading

-from util import procTools, uiTools +from util import uiTools

-from stem.util import conf, log +from stem.util import conf, log, proc

# Mapping of commands to if they're available or not. This isn't always # reliable, failing for some special commands. For these the cache is @@ -120,9 +120,9 @@ def getProcessName(pid, default = None, cacheFailure = True): processName, raisedExc = "", None

# fetch it from proc contents if available - if procTools.isProcAvailable(): + if proc.is_available(): try: - processName = procTools.getStats(pid, procTools.Stat.COMMAND)[0] + processName = proc.get_stats(pid, proc.Stat.COMMAND)[0] except IOError, exc: raisedExc = exc

@@ -163,9 +163,9 @@ def getPwd(pid): elif pid in PWD_CACHE: return PWD_CACHE[pid]

# try fetching via the proc contents if available - if procTools.isProcAvailable(): + if proc.is_available(): try: - pwd = procTools.getPwd(pid) + pwd = proc.get_cwd(pid) PWD_CACHE[pid] = pwd return pwd except IOError: pass # fall back to pwdx @@ -403,7 +403,7 @@ class ResourceTracker(threading.Thread): self.memUsagePercentage = 0.0 # percentage cpu usage

# resolves usage via proc results if true, ps otherwise - self._useProc = procTools.isProcAvailable() + self._useProc = proc.is_available()

# used to get the deltas when querying cpu time self._lastCpuTotal = 0 @@ -469,15 +469,15 @@ class ResourceTracker(threading.Thread): newValues = {} try: if self._useProc: - utime, stime, startTime = procTools.getStats(self.processPid, procTools.Stat.CPU_UTIME, procTools.Stat.CPU_STIME, procTools.Stat.START_TIME) + utime, stime, startTime = proc.get_stats(self.processPid, proc.Stat.CPU_UTIME, proc.Stat.CPU_STIME, proc.Stat.START_TIME) totalCpuTime = float(utime) + float(stime) cpuDelta = totalCpuTime - self._lastCpuTotal newValues["cpuSampling"] = cpuDelta / timeSinceReset newValues["cpuAvg"] = totalCpuTime / (time.time() - float(startTime)) newValues["_lastCpuTotal"] = totalCpuTime

- memUsage = int(procTools.getMemoryUsage(self.processPid)[0]) - totalMemory = procTools.getPhysicalMemory() + memUsage = int(proc.get_memory_usage(self.processPid)[0]) + totalMemory = proc.get_physical_memory() newValues["memUsage"] = memUsage newValues["memUsagePercentage"] = float(memUsage) / totalMemory else: diff --git a/src/util/torTools.py b/src/util/torTools.py index 0ad914b..bce3481 100644 --- a/src/util/torTools.py +++ b/src/util/torTools.py @@ -16,9 +16,9 @@ import stem import stem.control import stem.descriptor

-from util import connections, procTools, sysTools, uiTools +from util import connections, sysTools, uiTools

-from stem.util import conf, enum, log +from stem.util import conf, enum, log, proc

# enums for tor's controller state: # INIT - attached to a new controller @@ -737,7 +737,7 @@ class Controller: self.connLock.acquire()

result = None - if self.isAlive() and procTools.isProcAvailable(): + if self.isAlive() and proc.is_available(): myPid = self.getMyPid()

if myPid: @@ -1640,9 +1640,9 @@ class Controller: if myPid: # if proc contents are available then fetch the pid from there and # convert it to the username - if procTools.isProcAvailable(): + if proc.is_available(): try: - myUid = procTools.getUid(myPid) + myUid = proc.get_uid(myPid) if myUid and myUid.isdigit(): result = pwd.getpwuid(int(myUid)).pw_name except: pass @@ -1709,9 +1709,9 @@ class Controller: myPid = self.getMyPid()

if myPid: - if procTools.isProcAvailable(): + if proc.is_available(): try: - result = float(procTools.getStats(myPid, procTools.Stat.START_TIME)[0]) + result = float(proc.get_stats(myPid, proc.Stat.START_TIME)[0]) except: pass

if not result: