- tor-commits - lists.torproject.org

[arm/release] fix: Accounting for multiple results from lsof
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 12077adddb88f1a2e7ae2f8e0078c54ce3e70fd2 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Apr 26 08:32:50 2011 -0700 fix: Accounting for multiple results from lsof When querying lsof from the python process this can provide multiple results with the same pid. The second is from the lsof query itself (this was observed with an earlier change for querying our own process). Accounting for this by checking all pid results from the query to see if they're in agreement. --- src/util/torTools.py | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/util/torTools.py b/src/util/torTools.py index 08b04c0..f4a29af 100644 --- a/src/util/torTools.py +++ b/src/util/torTools.py @@ -224,8 +224,20 @@ def getPid(controlPort=9051, pidFilePath=None): try: results = sysTools.call("lsof -wnPi | egrep \"^tor.*:%i\"" % controlPort) - if len(results) == 1 and len(results[0].split()) > 1: - pid = results[0].split()[1] + + # This can result in multiple entries with the same pid (maybe from the + # query itself?). Checking all lines to see if they have the same pid. + + if results: + pid = "" + + for line in results: + lineComp = line.split() + + if len(lineComp) >= 2 and (not pid or lineComp[1] == pid): + pid = lineComp[1] + else: raise IOError + if pid.isdigit(): return pid except IOError: pass

1 0

[arm/release] Separating inital BW totals into separate params
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit b4ea3bd612d54ff1e33c4b52d4982ccfc207d9fb Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 23 20:34:31 2011 -0700 Separating inital BW totals into separate params Mixing the bandwidth totals fetched from Tor with the GraphStats param caused the average measurement to be skewed. Separating this total into a parameter just used for the total label. --- armrc.sample | 4 ---- src/interface/graphing/bandwidthStats.py | 14 +++++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/armrc.sample b/armrc.sample index 11b62ea..4cda197 100644 --- a/armrc.sample +++ b/armrc.sample @@ -140,9 +140,6 @@ features.graph.showIntermediateBounds true # prepopulate # attempts to use tor's state file to prepopulate the bandwidth graph at the # 15-minute interval (this requires the minimum of a day's worth of uptime) -# prepopulateTotal -# populates the total stat from the state file if true (this only contains -# the last day's worth of information, so this metric isn't the true total) # transferInBystes # shows rate measurments in bytes if true, bits otherwise # accounting.show @@ -153,7 +150,6 @@ features.graph.showIntermediateBounds true # provides verbose measurements of time if true features.graph.bw.prepopulate true -features.graph.bw.prepopulateTotal false features.graph.bw.transferInBytes false features.graph.bw.accounting.show true features.graph.bw.accounting.rate 10 diff --git a/src/interface/graphing/bandwidthStats.py b/src/interface/graphing/bandwidthStats.py index 70a1e16..f8e3020 100644 --- a/src/interface/graphing/bandwidthStats.py +++ b/src/interface/graphing/bandwidthStats.py @@ -60,19 +60,18 @@ class BandwidthStats(graphPanel.GraphStats): # https://trac.torproject.org/projects/tor/ticket/2345 # # further updates are still handled via BW events to avoid unnecessary - # GETINFO requests. This needs to update the pause buffer too because - # instances start paused, causing the primary value to be clobbered once - # just after initalization. + # GETINFO requests. + + self.initialPrimaryTotal = 0 + self.initialSecondaryTotal = 0 readTotal = conn.getInfo("traffic/read") if readTotal and readTotal.isdigit(): - self.primaryTotal = int(readTotal) / 1024 # Bytes -> KB - self._pauseBuffer.primaryTotal = int(readTotal) / 1024 + self.initialPrimaryTotal = int(readTotal) / 1024 # Bytes -> KB writeTotal = conn.getInfo("traffic/written") if writeTotal and writeTotal.isdigit(): - self.secondaryTotal = int(writeTotal) / 1024 # Bytes -> KB - self._pauseBuffer.secondaryTotal = int(writeTotal) / 1024 + self.initialSecondaryTotal = int(writeTotal) / 1024 # Bytes -> KB def resetListener(self, conn, eventType): # updates title parameters and accounting status if they changed @@ -346,6 +345,7 @@ class BandwidthStats(graphPanel.GraphStats): def _getTotalLabel(self, isPrimary): total = self.primaryTotal if isPrimary else self.secondaryTotal + total += self.initialPrimaryTotal if isPrimary else self.initialSecondaryTotal return "total: %s" % uiTools.getSizeLabel(total * 1024, 1) def _updateAccountingInfo(self):

1 0

[arm/release] Initializing bandwidth totals from Tor when able
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 301085e92801173d884e8a1282539282331244d8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 23 20:24:55 2011 -0700 Initializing bandwidth totals from Tor when able Ticket 2345 (https://trac.torproject.org/projects/tor/ticket/2345) introduced GETINFO options for the total read and written bandwidth values. If these are available then they're used to populate the total values in arm. --- src/interface/graphing/bandwidthStats.py | 19 +++++++++++++++++++ src/settings.cfg | 4 ++++ 2 files changed, 23 insertions(+), 0 deletions(-) diff --git a/src/interface/graphing/bandwidthStats.py b/src/interface/graphing/bandwidthStats.py index 1955465..3f34e3c 100644 --- a/src/interface/graphing/bandwidthStats.py +++ b/src/interface/graphing/bandwidthStats.py @@ -55,6 +55,25 @@ class BandwidthStats(graphPanel.GraphStats): self._titleStats, self.isAccounting = [], False self.resetListener(conn, torTools.State.INIT) # initializes values conn.addStatusListener(self.resetListener) + + # Initialized the bandwidth totals to the values reported by Tor. This + # uses a controller options introduced in ticket 2345: + # https://trac.torproject.org/projects/tor/ticket/2345 + # + # further updates are still handled via BW events to avoid unnecessary + # GETINFO requests. This needs to update the pause buffer too because + # instances start paused, causing the primary value to be clobbered once + # just after initalization. + + readTotal = conn.getInfo("traffic/read") + if readTotal and readTotal.isdigit(): + self.primaryTotal = int(readTotal) / 1024 # Bytes -> KB + self._pauseBuffer.primaryTotal = int(readTotal) / 1024 + + writeTotal = conn.getInfo("traffic/written") + if writeTotal and writeTotal.isdigit(): + self.secondaryTotal = int(writeTotal) / 1024 # Bytes -> KB + self._pauseBuffer.secondaryTotal = int(writeTotal) / 1024 def resetListener(self, conn, eventType): # updates title parameters and accounting status if they changed diff --git a/src/settings.cfg b/src/settings.cfg index 0e0e395..1ee63a1 100644 --- a/src/settings.cfg +++ b/src/settings.cfg @@ -273,6 +273,8 @@ config.summary.TestingEstimatedDescriptorPropagationTime delay before clients at # [ARM_DEBUG] recreating panel 'graph' with the dimensions of 14/124 # [ARM_DEBUG] redrawing the log panel with the corrected content height (estimat was off by 4) # [ARM_DEBUG] GETINFO accounting/bytes-left (runtime: 0.0006) +# [ARM_DEBUG] GETINFO traffic/read (runtime: 0.0004) +# [ARM_DEBUG] GETINFO traffic/written (runtime: 0.0002) # [ARM_DEBUG] GETCONF MyFamily (runtime: 0.0007) # [ARM_DEBUG] Unable to query process resource usage from ps, waiting 6.25 seconds (unrecognized output from ps: ...) @@ -311,6 +313,8 @@ msg.ARM_DEBUG GETINFO accounting/bytes msg.ARM_DEBUG GETINFO accounting/bytes-left msg.ARM_DEBUG GETINFO accounting/interval-end msg.ARM_DEBUG GETINFO accounting/hibernating +msg.ARM_DEBUG GETINFO traffic/read +msg.ARM_DEBUG GETINFO traffic/written msg.ARM_DEBUG GETCONF msg.ARM_DEBUG Unable to query process resource usage from ps

1 0

[arm/release] Ignoring vim temporary swap files
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit b38cd993ee6d717372f479329236a7ab0a019f79 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 23 20:22:17 2011 -0700 Ignoring vim temporary swap files --- .gitignore | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index 9b955dd..6377435 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc src/TorCtl/ +*.swp

1 0

[arm/release] Dropping optional total prepopulation state file
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 4f3f7cfd957bc55f956e21e8af82888a24ec7ff4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 23 20:30:41 2011 -0700 Dropping optional total prepopulation state file The total value provided by the 'features.graph.bw.prepopulateTotal' config option was confusing, pointless, and conflicted with the new functionality we have to fetch the true totals from Tor. --- src/interface/graphing/bandwidthStats.py | 7 +------ 1 files changed, 1 insertions(+), 6 deletions(-) diff --git a/src/interface/graphing/bandwidthStats.py b/src/interface/graphing/bandwidthStats.py index 3f34e3c..70a1e16 100644 --- a/src/interface/graphing/bandwidthStats.py +++ b/src/interface/graphing/bandwidthStats.py @@ -20,8 +20,7 @@ ACCOUNTING_ARGS = ("status", "resetTime", "read", "written", "readLimit", "writt PREPOPULATE_SUCCESS_MSG = "Read the last day of bandwidth history from the state file" PREPOPULATE_FAILURE_MSG = "Unable to prepopulate bandwidth information (%s)" -DEFAULT_CONFIG = {"features.graph.bw.prepopulateTotal": False, - "features.graph.bw.transferInBytes": False, +DEFAULT_CONFIG = {"features.graph.bw.transferInBytes": False, "features.graph.bw.accounting.show": True, "features.graph.bw.accounting.rate": 10, "features.graph.bw.accounting.isTimeLong": False, @@ -347,10 +346,6 @@ class BandwidthStats(graphPanel.GraphStats): def _getTotalLabel(self, isPrimary): total = self.primaryTotal if isPrimary else self.secondaryTotal - - if self._config["features.graph.bw.prepopulateTotal"]: - total += self.prepopulatePrimaryTotal if isPrimary else self.prepopulateSecondaryTotal - return "total: %s" % uiTools.getSizeLabel(total * 1024, 1) def _updateAccountingInfo(self):

1 0

[arm/release] Better commenting for new controller option usage
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 1ff5c3d8307a38485f850ba4936d4ab2e16bed7d Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 22 19:18:50 2011 -0700 Better commenting for new controller option usage --- src/util/torTools.py | 13 ++++++++----- 1 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/util/torTools.py b/src/util/torTools.py index 5e77f2e..b5913ed 100644 --- a/src/util/torTools.py +++ b/src/util/torTools.py @@ -1626,6 +1626,12 @@ class Controller(TorCtl.PostEventListener): lookups if true """ + # Several controller options were added in ticket 2291... + # https://trac.torproject.org/projects/tor/ticket/2291 + # which is only available with newer tor versions (tested them against + # Tor v0.2.3.0-alpha-dev). When using these options we need to be + # especially careful to have good fallback logic. + currentVal = self._cachedParam.get(key) if currentVal != None: if currentVal == UNKNOWN: return default @@ -1697,8 +1703,7 @@ class Controller(TorCtl.PostEventListener): if not result: result = getPid(int(self.getOption("ControlPort", 9051)), self.getOption("PidFile")) elif key == "user": - # This was added in Tor 0.2.3.x-final so it's quite likely unavailable. - # Even if it is, it might fail and return an empty string. + # provides the empty string if the query fails queriedUser = self.getInfo("process/user") if queriedUser != None and queriedUser != "": @@ -1721,9 +1726,7 @@ class Controller(TorCtl.PostEventListener): psResults = sysTools.call("ps -o user %s" % myPid) if psResults and len(psResults) >= 2: result = psResults[1].strip() elif key == "fdLimit": - # This was added in Tor 0.2.3.x-final so it's quite likely unavailable. - # Even if it is, it might fail and return -1. - + # provides -1 if the query fails queriedLimit = self.getInfo("process/descriptor-limit") if queriedLimit != None and queriedLimit != "-1":

1 0

[arm/release] fix: Capitalizing unused cfg option log message
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 9ba121e859d8c551be08e340728004fa48ec1c4d Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 22 19:05:40 2011 -0700 fix: Capitalizing unused cfg option log message --- src/interface/controller.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/interface/controller.py b/src/interface/controller.py index 558c107..b88152f 100644 --- a/src/interface/controller.py +++ b/src/interface/controller.py @@ -649,7 +649,7 @@ def drawTorMonitor(stdscr, startTime, loggedEvents, isBlindMode): # provides notice about any unused config keys for key in config.getUnusedKeys(): - log.log(CONFIG["log.configEntryUndefined"], "unused configuration entry: %s" % key) + log.log(CONFIG["log.configEntryUndefined"], "Unused configuration entry: %s" % key) lastPerformanceLog = 0 # ensures we don't do performance logging too frequently redrawStartTime = time.time()

1 0

[arm/release] Putting TorCtl install prompt in its own function
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 6534807224d5f88b0c5450e996e5fa1f8c470763 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Apr 21 08:52:36 2011 -0700 Putting TorCtl install prompt in its own function --- src/prereq.py | 41 ++++++++++++++++++++++++++--------------- 1 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/prereq.py b/src/prereq.py index ee5c768..50aaa91 100644 --- a/src/prereq.py +++ b/src/prereq.py @@ -20,6 +20,30 @@ def isTorCtlAvailable(): except ImportError: return False +def promptTorCtlInstall(): + """ + Asks the user to install TorCtl. This returns True if it was installed and + False otherwise (if it was either declined or failed to be fetched. + """ + + userInput = raw_input("Arm requires TorCtl to run, but it's unavailable. Would you like to install it? (y/n): ") + + # if user says no then terminate + if not userInput.lower() in ("y", "yes"): return False + + # attempt to install TorCtl, printing the issue if unsuccessful + try: + installTorCtl() + + if not isTorCtlAvailable(): + raise IOError("Unable to install TorCtl, sorry") + + print "TorCtl successfully installed" + return True + except IOError, exc: + print exc + return False + def installTorCtl(): """ Checks out the current git head release for TorCtl and bundles it with arm. @@ -65,19 +89,6 @@ if __name__ == '__main__': sys.exit(1) if not isTorCtlAvailable(): - userInput = raw_input("Arm requires TorCtl to run, but it's unavailable. Would you like to install it? (y/n): ") - - # if user says no then terminate - if not userInput.lower() in ("y", "yes"): - sys.exit(1) - - # attempt to install TorCtl, printing the issue if unsuccessful - try: - installTorCtl() - - if not isTorCtlAvailable(): - raise IOError("Unable to install TorCtl, sorry") - except IOError, exc: - print exc - sys.exit(1) + isInstalled = promptTorCtlInstall() + if not isInstalled: sys.exit(1)

1 0

[arm/release] Dropping raw torctl access from the descriptor popup (simpler, takes advantage of caching, and allows for restarts)
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit 900955fedaca8f87576e67c4c7af474f1ab5453a Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Apr 12 16:15:00 2011 +0000 Dropping raw torctl access from the descriptor popup (simpler, takes advantage of caching, and allows for restarts) svn:r24594 --- src/interface/controller.py | 4 ++-- src/interface/descriptorPopup.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/interface/controller.py b/src/interface/controller.py index 5761861..167be0a 100644 --- a/src/interface/controller.py +++ b/src/interface/controller.py @@ -1462,7 +1462,7 @@ def drawTorMonitor(stdscr, startTime, loggedEvents, isBlindMode): if key in (curses.KEY_DOWN, curses.KEY_UP, curses.KEY_PPAGE, curses.KEY_NPAGE): panels["conn"].handleKey(key) elif key in (ord('d'), ord('D')): - descriptorPopup.showDescriptorPopup(panels["popup"], stdscr, conn, panels["conn"]) + descriptorPopup.showDescriptorPopup(panels["popup"], stdscr, panels["conn"]) panels["conn"].redraw(True) panels["conn"].showLabel = True @@ -1481,7 +1481,7 @@ def drawTorMonitor(stdscr, startTime, loggedEvents, isBlindMode): panels["conn"].showLabel = False panels["conn"].redraw(True) - descriptorPopup.showDescriptorPopup(panels["popup"], stdscr, conn, panels["conn"]) + descriptorPopup.showDescriptorPopup(panels["popup"], stdscr, panels["conn"]) setPauseState(panels, isPaused, page) curses.halfdelay(REFRESH_RATE * 10) # reset normal pausing behavior diff --git a/src/interface/descriptorPopup.py b/src/interface/descriptorPopup.py index 1fabcf4..51ea2e3 100644 --- a/src/interface/descriptorPopup.py +++ b/src/interface/descriptorPopup.py @@ -9,7 +9,7 @@ from TorCtl import TorCtl import controller import connPanel -from util import panel, uiTools +from util import panel, torTools, uiTools # field keywords used to identify areas for coloring LINE_NUM_COLOR = "yellow" @@ -28,8 +28,7 @@ class PopupProperties: State attributes of popup window for consensus descriptions. """ - def __init__(self, conn): - self.conn = conn + def __init__(self): self.fingerprint = "" self.entryColor = "white" self.text = [] @@ -47,18 +46,19 @@ class PopupProperties: self.showLineNum = False self.text.append(UNRESOLVED_MSG) else: + conn = torTools.getConn() + try: self.showLineNum = True - nsCommand = "ns/id/%s" % fingerprint - self.text.append(nsCommand) - self.text = self.text + self.conn.get_info(nsCommand)[nsCommand].split("\n") + self.text.append("ns/id/%s" % fingerprint) + self.text += conn.getConsensusEntry(fingerprint).split("\n") except (socket.error, TorCtl.ErrorReply, TorCtl.TorCtlClosed): self.text = self.text + [ERROR_MSG, ""] try: descCommand = "desc/id/%s" % fingerprint - self.text.append(descCommand) - self.text = self.text + self.conn.get_info(descCommand)[descCommand].split("\n") + self.text.append("desc/id/%s" % fingerprint) + self.text += conn.getDescriptorEntry(fingerprint).split("\n") except (socket.error, TorCtl.ErrorReply, TorCtl.TorCtlClosed): self.text = self.text + [ERROR_MSG] @@ -68,7 +68,7 @@ class PopupProperties: elif key == curses.KEY_PPAGE: self.scroll = max(self.scroll - height, 0) elif key == curses.KEY_NPAGE: self.scroll = max(0, min(self.scroll + height, len(self.text) - height)) -def showDescriptorPopup(popup, stdscr, conn, connectionPanel): +def showDescriptorPopup(popup, stdscr, connectionPanel): """ Presents consensus descriptor in popup window with the following controls: Up, Down, Page Up, Page Down - scroll descriptor @@ -76,7 +76,7 @@ def showDescriptorPopup(popup, stdscr, conn, connectionPanel): Enter, Space, d, D - close popup """ - properties = PopupProperties(conn) + properties = PopupProperties() isVisible = True if not panel.CURSES_LOCK.acquire(False): return

1 0

[arm/release] Offering to install TorCtl if unavailable
by atagar＠torproject.org 17 Jul '11

17 Jul '11

commit d1d54c08e245083660394d0e11b41c593099fc2f Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Apr 20 19:38:17 2011 -0700 Offering to install TorCtl if unavailable --- src/prereq.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 60 insertions(+), 0 deletions(-) diff --git a/src/prereq.py b/src/prereq.py index 90cb03a..ee5c768 100644 --- a/src/prereq.py +++ b/src/prereq.py @@ -2,7 +2,50 @@ Provides a warning and error code if python version isn't compatible. """ +import os import sys +import shutil +import tempfile + +TORCTL_REPO = "git://git.torproject.org/pytorctl.git" + +def isTorCtlAvailable(): + """ + True if TorCtl is already available on the platform, false otherwise. + """ + + try: + import TorCtl + return True + except ImportError: + return False + +def installTorCtl(): + """ + Checks out the current git head release for TorCtl and bundles it with arm. + This raises an IOError if unsuccessful. + """ + + if isTorCtlAvailable(): return + + # temporary destination for TorCtl's git clone, guarenteed to be unoccupied + # (to avoid conflicting with files that are already there) + tmpFilename = tempfile.mktemp("/torctl") + + # fetches TorCtl + exitStatus = os.system("git clone %s %s > /dev/null" % (TORCTL_REPO, tmpFilename)) + if exitStatus: raise IOError("Unable to get TorCtl from %s. Is git installed?" % TORCTL_REPO) + + # the destination for TorCtl will be our directory + ourDir = os.path.dirname(os.path.realpath(__file__)) + + # exports TorCtl to our location + exitStatus = os.system("(cd %s && git archive --format=tar --prefix=TorCtl/ master) | (cd %s && tar xf - 2> /dev/null)" % (tmpFilename, ourDir)) + if exitStatus: raise IOError("Unable to install TorCtl to %s" % ourDir) + + # Clean up the temporary contents. This isn't vital so quietly fails in case + # of errors. + shutil.rmtree(tmpFilename, ignore_errors=True) if __name__ == '__main__': majorVersion = sys.version_info[0] @@ -20,4 +63,21 @@ if __name__ == '__main__': except ImportError: print("arm requires curses - try installing the python-curses package\n") sys.exit(1) + + if not isTorCtlAvailable(): + userInput = raw_input("Arm requires TorCtl to run, but it's unavailable. Would you like to install it? (y/n): ") + + # if user says no then terminate + if not userInput.lower() in ("y", "yes"): + sys.exit(1) + + # attempt to install TorCtl, printing the issue if unsuccessful + try: + installTorCtl() + + if not isTorCtlAvailable(): + raise IOError("Unable to install TorCtl, sorry") + except IOError, exc: + print exc + sys.exit(1)

1 0