tor-commits October 2013

tor-commits@lists.torproject.org

17 participants
1284 discussions

[translation/torbirdy] Update translations for torbirdy
by translation＠torproject.org 29 Oct '13

29 Oct '13

commit b7daf08dbb1fc31515b896824a6fcafd2e539920 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Oct 29 05:16:00 2013 +0000 Update translations for torbirdy --- sk/torbirdy.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sk/torbirdy.properties b/sk/torbirdy.properties index 5baf8ee..14c35bd 100644 --- a/sk/torbirdy.properties +++ b/sk/torbirdy.properties @@ -13,7 +13,7 @@ torbirdy.email.advanced=Prosím berte na vedomie, že zmena pokročilých nastav torbirdy.email.advanced.nextwarning=Zobraziť túto výstrahu nabudúce torbirdy.email.advanced.title=Pokročilé nastavenia TorBirdy -# torbirdy.restart=You must restart Thunderbird for the time zone preference to take effect. +torbirdy.restart=Musíte reštartovať Thunderbird, aby sa prejavili zmeny v nastavení časového pásma. torbirdy.firstrun=Práve používate TorBirdy.\n\nPre ochranu Vašej anonymity, TorBirdy vynúti svoje nastavenia Thunderbird-u, aby nemohli byť zmenené Vami alebo nejakým doplnkom. Niektoré nastavenia je možné zmeniť a sú prístupné cez nastavenia TorBirdy. Ak odinštalujete TorBirdy, tak sa všetky nastavenia vrátia na východzie hodnoty (hodnoty po inštalácii TorBirdy).\n\nAk ste nový používateľ, je odporúčané aby ste si prečítali stránku TorBirdy a pochopili čo sa snažíme docieliť pre užívateľov TorBirdy. torbirdy.website=https://trac.torproject.org/projects/tor/wiki/torbirdy

1 0

[arm/master] Using get_resource_usage() results as a named tuple
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit 737e6e7c3d1856780317a74ad53b7b3c87705bfd Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 10:42:58 2013 -0700 Using get_resource_usage() results as a named tuple Referencing the returned results by their attribute name rather than position. This allows the caller's code to be a bit more self-documenting. --- arm/graphing/resourceStats.py | 6 +++--- arm/headerPanel.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arm/graphing/resourceStats.py b/arm/graphing/resourceStats.py index 03a8654..5be4ba8 100644 --- a/arm/graphing/resourceStats.py +++ b/arm/graphing/resourceStats.py @@ -47,9 +47,9 @@ class ResourceStats(graphPanel.GraphStats): resourceTracker = arm.util.tracker.get_resource_tracker() if resourceTracker and not resourceTracker.last_query_failed(): - primary, _, secondary, _ = resourceTracker.get_resource_usage() - primary *= 100 # decimal percentage to whole numbers - secondary /= 1048576 # translate size to MB so axis labels are short + resources = resourceTracker.get_resource_usage() + primary = resources.cpu_sample * 100 # decimal percentage to whole numbers + secondary = resources.memory_bytes / 1048576 # translate size to MB so axis labels are short self.runCount = resourceTracker.run_counter() self._processEvent(primary, secondary) diff --git a/arm/headerPanel.py b/arm/headerPanel.py index b559ec8..629c84c 100644 --- a/arm/headerPanel.py +++ b/arm/headerPanel.py @@ -571,11 +571,11 @@ class HeaderPanel(panel.Panel, threading.Thread): self.vals["stat/rss"] = "0" self.vals["stat/%mem"] = "0" else: - cpuUsage, _, memUsage, memUsagePercent = resourceTracker.get_resource_usage() + resources = resourceTracker.get_resource_usage() self._lastResourceFetch = resourceTracker.run_counter() - self.vals["stat/%torCpu"] = "%0.1f" % (100 * cpuUsage) - self.vals["stat/rss"] = str(memUsage) - self.vals["stat/%mem"] = "%0.1f" % (100 * memUsagePercent) + self.vals["stat/%torCpu"] = "%0.1f" % (100 * resources.cpu_sample) + self.vals["stat/rss"] = str(resources.memory_bytes) + self.vals["stat/%mem"] = "%0.1f" % (100 * resources.memory_percent) # determines the cpu time for the arm process (including user and system # time of both the primary and child processes)

1 0

[arm/master] Don't attempt to get process information without a pid
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit 7da10157bda24cb94a6342af4c32b206bd77f5e0 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 12:14:09 2013 -0700 Don't attempt to get process information without a pid Making the resource and connection trackers no-ops when we're unable to determine tor's pid. --- arm/util/tracker.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 4f22dee..c982114 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -121,10 +121,13 @@ class Daemon(threading.Thread): continue # done waiting, try again with self._daemon_lock: - is_successful = self._task(self._process_pid, self._process_name) + if self._process_pid is not None: + is_successful = self._task(self._process_pid, self._process_name) + else: + is_successful = False - if is_successful: - self._run_counter += 1 + if is_successful: + self._run_counter += 1 self._last_ran = time.time() @@ -132,6 +135,9 @@ class Daemon(threading.Thread): """ Task the resolver is meant to perform. This should be implemented by subclasses. + + :param int process_pid: pid of the process we're tracking + :param str process_name: name of the process we're tracking """ pass

1 0

[arm/master] Better thread safety for tracker operations
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit fbaa098f4731a6d3bd2713033efc4bfbf62b913a Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 11:45:32 2013 -0700 Better thread safety for tracker operations Resolving tor's pid and process name under a daemon lock, which is also used for the tracker tasks (where it's used). --- arm/util/tracker.py | 61 +++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 8e93e58..647acee 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -3,7 +3,7 @@ Background tasks for gathering informatino about the tor process. :: - get_connection_tracker - provides a ConnectionResolver for our tor process + get_connection_tracker - provides a ConnectionTracker for our tor process get_resource_tracker - provides a ResourceTracker for our tor process Daemon - common parent for resolvers @@ -13,7 +13,7 @@ Background tasks for gathering informatino about the tor process. |- set_paused - pauses or continues work +- stop - stops further work by the daemon - ConnectionResolver - periodically checks the connections established by tor + ConnectionTracker - periodically checks the connections established by tor |- get_custom_resolver - provide the custom conntion resolver we're using |- set_custom_resolver - overwrites automatic resolver selecion with a custom resolver +- get_connections - provides our latest connection results @@ -32,8 +32,8 @@ import arm.util.torTools from stem.control import State from stem.util import conf, connection, log, proc, str_tools, system -CONFIG = conf.config_dict("arm", { - "queries.resourceUsage.rate": 5, +CONFIG = conf.config_dict('arm', { + 'queries.resourceUsage.rate': 5, 'queries.connections.minRate': 5, 'msg.unable_to_use_resolver': '', 'msg.unable_to_use_all_resolvers': '', @@ -64,7 +64,7 @@ def get_connection_tracker(): global CONNECTION_TRACKER if CONNECTION_TRACKER is None: - CONNECTION_TRACKER = ConnectionResolver() + CONNECTION_TRACKER = ConnectionTracker() return CONNECTION_TRACKER @@ -91,6 +91,10 @@ class Daemon(threading.Thread): threading.Thread.__init__(self) self.daemon = True + # PID and process name of the tor process we're tracking. These should only + # be used under the daemon's lock. + + self._daemon_lock = threading.RLock() self._process_name = None self._process_pid = None @@ -99,8 +103,8 @@ class Daemon(threading.Thread): self._run_counter = 0 # counter for the number of successful runs self._is_paused = False + self._pause_condition = threading.Condition() self._halt = False # terminates thread if true - self._cond = threading.Condition() # used for pausing the thread controller = arm.util.torTools.getConn().controller controller.add_status_listener(self._tor_status_listener) @@ -113,24 +117,24 @@ class Daemon(threading.Thread): if self._is_paused or time_since_last_ran < self._rate: sleep_duration = max(0.2, self._rate - time_since_last_ran) - self._cond.acquire() - if not self._halt: - self._cond.wait(sleep_duration) - self._cond.release() + with self._pause_condition: + if not self._halt: + self._pause_condition.wait(sleep_duration) continue # done waiting, try again - is_successful = self.task() + with self._daemon_lock: + is_successful = self._task() if is_successful: self._run_counter += 1 self._last_ran = time.time() - def task(self): + def _task(self): """ Task the resolver is meant to perform. This should be implemented by - subclasses. + subclasses. This is executed under the daemon's lock. """ pass @@ -175,24 +179,24 @@ class Daemon(threading.Thread): Halts further work and terminates the thread. """ - self._cond.acquire() - self._halt = True - self._cond.notifyAll() - self._cond.release() + with self._pause_condition: + self._halt = True + self._pause_condition.notifyAll() def _tor_status_listener(self, controller, event_type, _): - if event_type in (State.INIT, State.RESET): - tor_pid = controller.get_pid(None) - tor_cmd = system.get_name_by_pid(tor_pid) if tor_pid else None + with self._daemon_lock: + if not self._halt and event_type in (State.INIT, State.RESET): + tor_pid = controller.get_pid(None) + tor_cmd = system.get_name_by_pid(tor_pid) if tor_pid else None - if tor_cmd is None: - tor_cmd = 'tor' + if tor_cmd is None: + tor_cmd = 'tor' - self._process_name = tor_cmd - self._process_pid = tor_pid + self._process_name = tor_cmd + self._process_pid = tor_pid -class ConnectionResolver(Daemon): +class ConnectionTracker(Daemon): """ Periodically retrieves the connections established by tor. """ @@ -210,7 +214,7 @@ class ConnectionResolver(Daemon): self._failure_count = 0 self._rate_too_low_count = 0 - def task(self): + def _task(self): if self._custom_resolver: resolver = self._custom_resolver is_default_resolver = False @@ -325,7 +329,6 @@ class ResourceTracker(Daemon): self._last_cpu_total = 0 self.last_lookup = -1 - self._val_lock = threading.RLock() # sequential times we've failed with this method of resolution self._failure_count = 0 @@ -336,7 +339,7 @@ class ResourceTracker(Daemon): (cpuUsage_sampling, cpuUsage_avg, memUsage_bytes, memUsage_percent) """ - with self._val_lock: + with self._daemon_lock: return self._last_sample if self._last_sample else Resources(0.0, 0.0, 0, 0.0) def last_query_failed(self): @@ -347,7 +350,7 @@ class ResourceTracker(Daemon): return self._failure_count != 0 - def task(self): + def _task(self): if self._procss_pid is None: return

1 0

[arm/master] Moving process tracking into the Daemon class
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit cf04513aa02523b319e36ebc520c0c6014a2476a Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 11:31:07 2013 -0700 Moving process tracking into the Daemon class Now that we're using a singleton for just tracking the tor process we can use the controller singleton within the Daemon class itself. This lets it transparently keep up to date with tor's pid and process name. --- arm/controller.py | 11 ----------- arm/headerPanel.py | 2 -- arm/util/tracker.py | 46 +++++++++++++++++++++------------------------- 3 files changed, 21 insertions(+), 38 deletions(-) diff --git a/arm/controller.py b/arm/controller.py index 69e52ce..a68fe41 100644 --- a/arm/controller.py +++ b/arm/controller.py @@ -118,7 +118,6 @@ def initController(stdscr, startTime): tor_cmd = "tor" resolver = arm.util.tracker.get_connection_tracker() - resolver.set_process(tor_pid, tor_cmd) log.info("Operating System: %s, Connection Resolvers: %s" % (os.uname()[0], ", ".join(resolver._resolvers))) resolver.start() else: @@ -510,16 +509,6 @@ def connResetListener(controller, eventType, _): if getController().getPanel("torrc") == None: torConfig.getTorrc().load(True) - try: - tor_cmd = system.get_name_by_pid(tor_pid) - - if tor_cmd is None: - tor_cmd = "tor" - - resolver.set_process(controller.get_pid(), tor_cmd) - except ValueError: - pass - def start_arm(stdscr): """ Main draw loop context. diff --git a/arm/headerPanel.py b/arm/headerPanel.py index 629c84c..d3a8d9d 100644 --- a/arm/headerPanel.py +++ b/arm/headerPanel.py @@ -417,7 +417,6 @@ class HeaderPanel(panel.Panel, threading.Thread): isChanged = False if self.vals["tor/pid"]: resourceTracker = arm.util.tracker.get_resource_tracker() - resourceTracker.set_process(self.vals["tor/pid"]) isChanged = self._lastResourceFetch != resourceTracker.run_counter() if isChanged or currentTime - self._lastUpdate >= 20: @@ -564,7 +563,6 @@ class HeaderPanel(panel.Panel, threading.Thread): # ps or proc derived resource usage stats if self.vals["tor/pid"]: resourceTracker = arm.util.tracker.get_resource_tracker() - resourceTracker.set_process(self.vals["tor/pid"]) if resourceTracker.last_query_failed(): self.vals["stat/%torCpu"] = "0" diff --git a/arm/util/tracker.py b/arm/util/tracker.py index c027975..8e93e58 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -14,13 +14,11 @@ Background tasks for gathering informatino about the tor process. +- stop - stops further work by the daemon ConnectionResolver - periodically checks the connections established by tor - |- set_process - set the pid and process name used for lookups |- get_custom_resolver - provide the custom conntion resolver we're using |- set_custom_resolver - overwrites automatic resolver selecion with a custom resolver +- get_connections - provides our latest connection results ResourceTracker - periodically checks the resource usage of tor - |- set_process - set the pid used for lookups |- get_resource_usage - provides our latest resource usage results +- last_query_failed - checks if we failed to fetch newer results """ @@ -29,6 +27,9 @@ import collections import time import threading +import arm.util.torTools + +from stem.control import State from stem.util import conf, connection, log, proc, str_tools, system CONFIG = conf.config_dict("arm", { @@ -90,6 +91,9 @@ class Daemon(threading.Thread): threading.Thread.__init__(self) self.daemon = True + self._process_name = None + self._process_pid = None + self._rate = rate self._last_ran = -1 # time when we last ran self._run_counter = 0 # counter for the number of successful runs @@ -98,6 +102,10 @@ class Daemon(threading.Thread): self._halt = False # terminates thread if true self._cond = threading.Condition() # used for pausing the thread + controller = arm.util.torTools.getConn().controller + controller.add_status_listener(self._tor_status_listener) + self._tor_status_listener(controller, State.INIT, None) + def run(self): while not self._halt: time_since_last_ran = time.time() - self._last_ran @@ -172,6 +180,17 @@ class Daemon(threading.Thread): self._cond.notifyAll() self._cond.release() + def _tor_status_listener(self, controller, event_type, _): + if event_type in (State.INIT, State.RESET): + tor_pid = controller.get_pid(None) + tor_cmd = system.get_name_by_pid(tor_pid) if tor_pid else None + + if tor_cmd is None: + tor_cmd = 'tor' + + self._process_name = tor_cmd + self._process_pid = tor_pid + class ConnectionResolver(Daemon): """ @@ -185,9 +204,6 @@ class ConnectionResolver(Daemon): self._connections = [] self._custom_resolver = None - self._process_pid = None - self._process_name = None - # Number of times in a row we've either failed with our current resolver or # concluded that our rate is too low. @@ -258,17 +274,6 @@ class ConnectionResolver(Daemon): return False - def set_process(self, pid, name): - """ - Sets the process we retrieve connections for. - - :param int pid: process id - :param str name: name of the process - """ - - self._process_pid = pid - self._process_name = name - def get_custom_resolver(self): """ Provides the custom resolver the user has selected. This is **None** if @@ -325,15 +330,6 @@ class ResourceTracker(Daemon): # sequential times we've failed with this method of resolution self._failure_count = 0 - def set_process(self, pid): - """ - Sets the process we retrieve resources for. - - :param int pid: process id - """ - - self._process_pid = pid - def get_resource_usage(self): """ Provides the last cached resource usage as a named tuple of the form:

1 0

[arm/master] Providing pid and process name to the task() method
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit efdce8091daeb3724a3105ca6126d1df98f1782d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 11:50:19 2013 -0700 Providing pid and process name to the task() method Having the Daemon class provide the pid and process name to subclasses via their _task() method. This provides better encapulation, so subclasses don't need to reference private attributes. Oh, and also fixing the resource tracker since I completely broke it earlier. --- arm/util/tracker.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 647acee..4f22dee 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -89,10 +89,7 @@ class Daemon(threading.Thread): def __init__(self, rate): threading.Thread.__init__(self) - self.daemon = True - - # PID and process name of the tor process we're tracking. These should only - # be used under the daemon's lock. + self.setDaemon(True) self._daemon_lock = threading.RLock() self._process_name = None @@ -124,17 +121,17 @@ class Daemon(threading.Thread): continue # done waiting, try again with self._daemon_lock: - is_successful = self._task() + is_successful = self._task(self._process_pid, self._process_name) if is_successful: self._run_counter += 1 self._last_ran = time.time() - def _task(self): + def _task(self, process_pid, process_name): """ Task the resolver is meant to perform. This should be implemented by - subclasses. This is executed under the daemon's lock. + subclasses. """ pass @@ -214,7 +211,7 @@ class ConnectionTracker(Daemon): self._failure_count = 0 self._rate_too_low_count = 0 - def _task(self): + def _task(self, process_pid, process_name): if self._custom_resolver: resolver = self._custom_resolver is_default_resolver = False @@ -229,8 +226,8 @@ class ConnectionTracker(Daemon): self._connections = connection.get_connections( resolver, - process_pid = self._process_pid, - process_name = self._process_name, + process_pid = process_pid, + process_name = process_name, ) runtime = time.time() - start_time @@ -319,7 +316,6 @@ class ResourceTracker(Daemon): def __init__(self): Daemon.__init__(self, CONFIG["queries.resourceUsage.rate"]) - self._procss_pid = None self._last_sample = None # resolves usage via proc results if true, ps otherwise @@ -350,8 +346,8 @@ class ResourceTracker(Daemon): return self._failure_count != 0 - def _task(self): - if self._procss_pid is None: + def _task(self, process_pid, process_name): + if process_pid is None: return time_since_reset = time.time() - self.last_lookup @@ -359,14 +355,14 @@ class ResourceTracker(Daemon): try: if self._use_proc: - utime, stime, start_time = proc.get_stats(self._procss_pid, proc.Stat.CPU_UTIME, proc.Stat.CPU_STIME, proc.Stat.START_TIME) + utime, stime, start_time = proc.get_stats(process_pid, proc.Stat.CPU_UTIME, proc.Stat.CPU_STIME, proc.Stat.START_TIME) total_cpu_time = float(utime) + float(stime) cpu_delta = total_cpu_time - self._last_cpu_total new_values["cpuSampling"] = cpu_delta / time_since_reset new_values["cpuAvg"] = total_cpu_time / (time.time() - float(start_time)) new_values["_lastCpuTotal"] = total_cpu_time - mem_usage = int(proc.get_memory_usage(self._procss_pid)[0]) + mem_usage = int(proc.get_memory_usage(process_pid)[0]) total_memory = proc.get_physical_memory() new_values["memUsage"] = mem_usage new_values["memUsagePercentage"] = float(mem_usage) / total_memory @@ -381,7 +377,7 @@ class ResourceTracker(Daemon): # TIME ELAPSED RSS %MEM # 0:04.40 37:57 18772 0.9 - ps_call = system.call("ps -p %s -o cputime,etime,rss,%%mem" % self._procss_pid) + ps_call = system.call("ps -p %s -o cputime,etime,rss,%%mem" % process_pid) is_successful = False if ps_call and len(ps_call) >= 2:

1 0

[arm/master] Minor Daemon pydoc revisions
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit 10331da38036aba48f954d0f5e4da17851d5356d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 12:23:09 2013 -0700 Minor Daemon pydoc revisions Cleaning up the pydocs a bit, adding attributes I missed. --- arm/util/tracker.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index c982114..f5ac768 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -84,7 +84,8 @@ def get_resource_tracker(): class Daemon(threading.Thread): """ - Daemon that can perform a unit of work at a given rate. + Daemon that can perform a given action at a set rate. Subclasses are expected + to implement our _task() method with the work to be done. """ def __init__(self, rate): @@ -92,8 +93,8 @@ class Daemon(threading.Thread): self.setDaemon(True) self._daemon_lock = threading.RLock() - self._process_name = None self._process_pid = None + self._process_name = None self._rate = rate self._last_ran = -1 # time when we last ran @@ -138,21 +139,26 @@ class Daemon(threading.Thread): :param int process_pid: pid of the process we're tracking :param str process_name: name of the process we're tracking + + :returns: **bool** indicating if our run was successful or not """ pass def run_counter(self): """ - Provides the number of successful runs so far. This can be used to - determine if the daemon's results are new for the caller or not. + Provides the number of times we've successful runs so far. This can be used + by callers to determine if our results have been seen by them before or + not. + + :returns: **int** for the run count we're on """ return self._run_counter def get_rate(self): """ - Provides the rate at which we perform our given task. + Provides the rate at which we perform our task. :returns: **float** for the rate in seconds at which we perform our task """ @@ -192,11 +198,8 @@ class Daemon(threading.Thread): tor_pid = controller.get_pid(None) tor_cmd = system.get_name_by_pid(tor_pid) if tor_pid else None - if tor_cmd is None: - tor_cmd = 'tor' - - self._process_name = tor_cmd self._process_pid = tor_pid + self._process_name = tor_cmd if tor_cmd else 'tor' class ConnectionTracker(Daemon):

1 0

[arm/master] Renaming query related config options
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit edfba0e97ffbe7f92633959b86f49f86e93bd35e Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 12:30:41 2013 -0700 Renaming query related config options Minor change to the naming convention for our query rates. Also dropping 'queries.useProc' from the sample armrc since it no longer exists (stem is smart enough to know when proc is available or not). --- arm/util/tracker.py | 8 ++++---- armrc.sample | 11 ++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index f5ac768..7f8ae02 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -33,8 +33,8 @@ from stem.control import State from stem.util import conf, connection, log, proc, str_tools, system CONFIG = conf.config_dict('arm', { - 'queries.resourceUsage.rate': 5, - 'queries.connections.minRate': 5, + 'queries.resources.rate': 5, + 'queries.connections.rate': 5, 'msg.unable_to_use_resolver': '', 'msg.unable_to_use_all_resolvers': '', }) @@ -208,7 +208,7 @@ class ConnectionTracker(Daemon): """ def __init__(self): - Daemon.__init__(self, CONFIG["queries.connections.minRate"]) + Daemon.__init__(self, CONFIG['queries.connections.rate']) self._resolvers = connection.get_system_resolvers() self._connections = [] @@ -323,7 +323,7 @@ class ResourceTracker(Daemon): """ def __init__(self): - Daemon.__init__(self, CONFIG["queries.resourceUsage.rate"]) + Daemon.__init__(self, CONFIG['queries.resources.rate']) self._last_sample = None diff --git a/armrc.sample b/armrc.sample index 682db5e..a4f7a71 100644 --- a/armrc.sample +++ b/armrc.sample @@ -4,8 +4,10 @@ startup.events N3 startup.dataDirectory ~/.arm # Seconds between querying information -queries.resourceUsage.rate 5 -queries.connections.minRate 5 + +queries.resources.rate 5 +queries.connections.rate 5 + queries.refreshRate.rate 5 # allows individual panels to be included/excluded @@ -15,11 +17,6 @@ features.panels.show.connection true features.panels.show.config true features.panels.show.torrc true -# Read the proc contents directly instead of calling ps, netstat, and other -# resolvers. This provides very sizable performance benefits (around 90% -# faster lookups) but this is only available on Linux. -queries.useProc true - # Renders the interface with color if set and the terminal supports it features.colorInterface true

1 0

[arm/master] Adding total cpu usage and timestamp to Resource
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit 91232de6293a3b136a815467872c725a68fb3f73 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 19:41:53 2013 -0700 Adding total cpu usage and timestamp to Resource Our ResourceTracker tracked the total cpu usage and fetch time since it's necessary for the derived attributes. So why not add it to the Resource named tuple itself? Also swapping us over to proper sphinx documentation for it. --- arm/util/tracker.py | 53 +++++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 2e964db..f8eb108 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -20,6 +20,17 @@ Background tasks for gathering informatino about the tor process. ResourceTracker - periodically checks the resource usage of tor +- get_resource_usage - provides our latest resource usage results + +.. data:: Resources + + Resource usage information retrieved about the tor process. + + :var float cpu_sample: average cpu usage since we last checked + :var float cpu_average: average cpu usage since we first started tracking the process + :var float cpu_total: total cpu time the process has used since starting + :var int memory_bytes: memory usage of the process in bytes + :var float memory_precent: percentage of our memory used by this process + :var float timestamp: unix timestamp for when this information was fetched """ import collections @@ -41,18 +52,13 @@ CONFIG = conf.config_dict('arm', { CONNECTION_TRACKER = None RESOURCE_TRACKER = None -# Process resources we poll... -# -# cpu_sample - average cpu usage since we last checked -# cpu_average - average cpu usage since we first started tracking the process -# memory_bytes - memory usage of the process in bytes -# memory_precent - percentage of our memory used by this process - Resources = collections.namedtuple('Resources', [ 'cpu_sample', 'cpu_average', + 'cpu_total', 'memory_bytes', 'memory_percent', + 'timestamp', ]) def get_connection_tracker(): @@ -330,11 +336,6 @@ class ResourceTracker(Daemon): # resolves usage via proc results if true, ps otherwise self._use_proc = proc.is_available() - # used to get the deltas when querying cpu time - self._last_cpu_total = 0 - - self.last_lookup = -1 - # sequential times we've failed with this method of resolution self._failure_count = 0 @@ -346,20 +347,20 @@ class ResourceTracker(Daemon): """ result = self._resources - return result if result else Resources(0.0, 0.0, 0, 0.0) + return result if result else Resources(0.0, 0.0, 0.0, 0, 0.0, 0.0) def _task(self, process_pid, process_name): - if process_pid is None: - return + last_cpu_total = self._resources.cpu_total if self._resources else 0 + last_lookup = self._resources.timestamp if self._resources else -1 - time_since_reset = time.time() - self.last_lookup + time_since_reset = time.time() - last_lookup new_values = {} try: if self._use_proc: utime, stime, start_time = proc.get_stats(process_pid, proc.Stat.CPU_UTIME, proc.Stat.CPU_STIME, proc.Stat.START_TIME) total_cpu_time = float(utime) + float(stime) - cpu_delta = total_cpu_time - self._last_cpu_total + cpu_delta = total_cpu_time - last_cpu_total new_values["cpuSampling"] = cpu_delta / time_since_reset new_values["cpuAvg"] = total_cpu_time / (time.time() - float(start_time)) new_values["_lastCpuTotal"] = total_cpu_time @@ -379,7 +380,7 @@ class ResourceTracker(Daemon): # TIME ELAPSED RSS %MEM # 0:04.40 37:57 18772 0.9 - ps_call = system.call("ps -p %s -o cputime,etime,rss,%%mem" % process_pid) + ps_call = system.call("ps -p {pid} -o cputime,etime,rss,%mem".format(pid = process_pid)) is_successful = False if ps_call and len(ps_call) >= 2: @@ -389,7 +390,7 @@ class ResourceTracker(Daemon): try: total_cpu_time = str_tools.parse_short_time_label(stats[0]) uptime = str_tools.parse_short_time_label(stats[1]) - cpu_delta = total_cpu_time - self._last_cpu_total + cpu_delta = total_cpu_time - last_cpu_total new_values["cpuSampling"] = cpu_delta / time_since_reset new_values["cpuAvg"] = total_cpu_time / uptime new_values["_lastCpuTotal"] = total_cpu_time @@ -426,13 +427,19 @@ class ResourceTracker(Daemon): # If this is the first run then the cpuSampling stat is meaningless # (there isn't a previous tick to sample from so it's zero at this # point). Setting it to the average, which is a fairer estimate. - if self.last_lookup == -1: + if last_lookup == -1: new_values["cpuSampling"] = new_values["cpuAvg"] with self.val_lock: - self._resources = Resources(new_values["cpuSampling"], new_values["cpuAvg"], new_values["memUsage"], new_values["memUsagePercentage"]) - self._last_cpu_total = new_values["_lastCpuTotal"] - self.last_lookup = time.time() + self._resources = Resources( + cpu_sample = new_values["cpuSampling"], + cpu_average = new_values["cpuAvg"], + cpu_total = new_values["_lastCpuTotal"], + memory_bytes = new_values["memUsage"], + memory_precent = new_values["memUsagePercentage"], + timestamp = time.time(), + ) + self._failure_count = 0 return True else:

1 0

[arm/master] Dropping resource tracker's last_query_failed()
by atagar＠torproject.org 29 Oct '13

29 Oct '13

commit 07c7847c3eeb8aea09a0b2e2c80cc950e7828871 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Oct 28 13:25:59 2013 -0700 Dropping resource tracker's last_query_failed() Ok, I'm not entirely sure why we had this method. We used to to render 0% resource usage when fresh results failed to be fetched. Why not show the last results we've seen? If we really want to track something reguarding staleness then that's what run_counter() is for. --- arm/graphing/resourceStats.py | 4 +++- arm/headerPanel.py | 15 +++++-------- arm/util/tracker.py | 49 ++++++++++++++++++----------------------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/arm/graphing/resourceStats.py b/arm/graphing/resourceStats.py index 5be4ba8..352c8ec 100644 --- a/arm/graphing/resourceStats.py +++ b/arm/graphing/resourceStats.py @@ -17,6 +17,7 @@ class ResourceStats(graphPanel.GraphStats): def __init__(self): graphPanel.GraphStats.__init__(self) self.queryPid = torTools.getConn().controller.get_pid(None) + self.lastCounter = None def clone(self, newCopy=None): if not newCopy: newCopy = ResourceStats() @@ -46,8 +47,9 @@ class ResourceStats(graphPanel.GraphStats): if self.queryPid: resourceTracker = arm.util.tracker.get_resource_tracker() - if resourceTracker and not resourceTracker.last_query_failed(): + if resourceTracker and resourceTracker.run_counter() != self.lastCounter: resources = resourceTracker.get_resource_usage() + self.lastCounter = resourceTracker.run_counter() primary = resources.cpu_sample * 100 # decimal percentage to whole numbers secondary = resources.memory_bytes / 1048576 # translate size to MB so axis labels are short self.runCount = resourceTracker.run_counter() diff --git a/arm/headerPanel.py b/arm/headerPanel.py index d3a8d9d..3420ade 100644 --- a/arm/headerPanel.py +++ b/arm/headerPanel.py @@ -564,16 +564,11 @@ class HeaderPanel(panel.Panel, threading.Thread): if self.vals["tor/pid"]: resourceTracker = arm.util.tracker.get_resource_tracker() - if resourceTracker.last_query_failed(): - self.vals["stat/%torCpu"] = "0" - self.vals["stat/rss"] = "0" - self.vals["stat/%mem"] = "0" - else: - resources = resourceTracker.get_resource_usage() - self._lastResourceFetch = resourceTracker.run_counter() - self.vals["stat/%torCpu"] = "%0.1f" % (100 * resources.cpu_sample) - self.vals["stat/rss"] = str(resources.memory_bytes) - self.vals["stat/%mem"] = "%0.1f" % (100 * resources.memory_percent) + resources = resourceTracker.get_resource_usage() + self._lastResourceFetch = resourceTracker.run_counter() + self.vals["stat/%torCpu"] = "%0.1f" % (100 * resources.cpu_sample) + self.vals["stat/rss"] = str(resources.memory_bytes) + self.vals["stat/%mem"] = "%0.1f" % (100 * resources.memory_percent) # determines the cpu time for the arm process (including user and system # time of both the primary and child processes) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 7f8ae02..2e964db 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -19,8 +19,7 @@ Background tasks for gathering informatino about the tor process. +- get_connections - provides our latest connection results ResourceTracker - periodically checks the resource usage of tor - |- get_resource_usage - provides our latest resource usage results - +- last_query_failed - checks if we failed to fetch newer results + +- get_resource_usage - provides our latest resource usage results """ import collections @@ -89,10 +88,10 @@ class Daemon(threading.Thread): """ def __init__(self, rate): - threading.Thread.__init__(self) + super(Daemon, self).__init__() self.setDaemon(True) - self._daemon_lock = threading.RLock() + self._process_lock = threading.RLock() self._process_pid = None self._process_name = None @@ -121,7 +120,7 @@ class Daemon(threading.Thread): continue # done waiting, try again - with self._daemon_lock: + with self._process_lock: if self._process_pid is not None: is_successful = self._task(self._process_pid, self._process_name) else: @@ -193,7 +192,7 @@ class Daemon(threading.Thread): self._pause_condition.notifyAll() def _tor_status_listener(self, controller, event_type, _): - with self._daemon_lock: + with self._process_lock: if not self._halt and event_type in (State.INIT, State.RESET): tor_pid = controller.get_pid(None) tor_cmd = system.get_name_by_pid(tor_pid) if tor_pid else None @@ -208,10 +207,10 @@ class ConnectionTracker(Daemon): """ def __init__(self): - Daemon.__init__(self, CONFIG['queries.connections.rate']) + super(ConnectionTracker, self).__init__(CONFIG['queries.connections.rate']) - self._resolvers = connection.get_system_resolvers() self._connections = [] + self._resolvers = connection.get_system_resolvers() self._custom_resolver = None # Number of times in a row we've either failed with our current resolver or @@ -228,7 +227,7 @@ class ConnectionTracker(Daemon): resolver = self._resolvers[0] is_default_resolver = True else: - return # nothing to resolve with + return False # nothing to resolve with try: start_time = time.time() @@ -289,7 +288,7 @@ class ConnectionTracker(Daemon): Provides the custom resolver the user has selected. This is **None** if we're picking resolvers dynamically. - :returns: :data:`stem.util.connection.Resolver` we're overwritten to use + :returns: :data:`~stem.util.connection.Resolver` we're overwritten to use """ return self._custom_resolver @@ -306,10 +305,10 @@ class ConnectionTracker(Daemon): def get_connections(self): """ - Provides the last queried connection results, an empty list if resolver - has been stopped. + Provides a listing of tor's latest connections. - :returns: **list** of :class:`~stem.util.connection.Connection` we last retrieved + :returns: **list** of :class:`~stem.util.connection.Connection` we last + retrieved, an empty list if our tracker's been stopped """ if self._halt: @@ -317,15 +316,16 @@ class ConnectionTracker(Daemon): else: return list(self._connections) + class ResourceTracker(Daemon): """ Periodically retrieves the resource usage of tor. """ def __init__(self): - Daemon.__init__(self, CONFIG['queries.resources.rate']) + super(ResourceTracker, self).__init__(CONFIG['queries.resources.rate']) - self._last_sample = None + self._resources = None # resolves usage via proc results if true, ps otherwise self._use_proc = proc.is_available() @@ -340,20 +340,13 @@ class ResourceTracker(Daemon): def get_resource_usage(self): """ - Provides the last cached resource usage as a named tuple of the form: - (cpuUsage_sampling, cpuUsage_avg, memUsage_bytes, memUsage_percent) - """ + Provides tor's latest resource usage. - with self._daemon_lock: - return self._last_sample if self._last_sample else Resources(0.0, 0.0, 0, 0.0) - - def last_query_failed(self): - """ - Provides true if, since we fetched the currently cached results, we've - failed to get new results. False otherwise. + :returns: latest :data:`~arm.util.tracker.Resources` we've polled """ - return self._failure_count != 0 + result = self._resources + return result if result else Resources(0.0, 0.0, 0, 0.0) def _task(self, process_pid, process_name): if process_pid is None: @@ -419,7 +412,7 @@ class ResourceTracker(Daemon): log.info("Failed three attempts to get process resource usage from proc, falling back to ps (%s)" % exc) self._use_proc = False - self._failure_count = 1 # prevents last_query_failed() from thinking that we succeeded + self._failure_count = 0 else: # wait a bit and try again log.debug("Unable to query process resource usage from proc (%s)" % exc) @@ -437,7 +430,7 @@ class ResourceTracker(Daemon): new_values["cpuSampling"] = new_values["cpuAvg"] with self.val_lock: - self._last_sample = Resources(new_values["cpuSampling"], new_values["cpuAvg"], new_values["memUsage"], new_values["memUsagePercentage"]) + self._resources = Resources(new_values["cpuSampling"], new_values["cpuAvg"], new_values["memUsage"], new_values["memUsagePercentage"]) self._last_cpu_total = new_values["_lastCpuTotal"] self.last_lookup = time.time() self._failure_count = 0

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits October 2013