commit 4a1b116712c404b8bb48931f2c57e43a9425eeae Author: Damian Johnson atagar@torproject.org Date: Sat Mar 14 20:33:37 2015 -0700

Drop bandwidth prepopulation based on the state file

Not only is this made pointless by #14128, but #13988 just plain old broke this. Users aren't getting errors, but when the state file information is used their graph has horribly inaccurate information. --- seth/config/strings.cfg | 5 +- seth/graph_panel.py | 56 +------------------ seth/util/__init__.py | 94 ------------------------------- sethrc.sample | 1 - test/util/bandwidth_from_state.py | 111 ------------------------------------- 5 files changed, 5 insertions(+), 262 deletions(-)

diff --git a/seth/config/strings.cfg b/seth/config/strings.cfg index 5fc441a..d7a780b 100644 --- a/seth/config/strings.cfg +++ b/seth/config/strings.cfg @@ -20,9 +20,8 @@ msg.config.nothing_loaded No sethrc loaded, using defaults. You can customize se msg.debug.saving_to_path Saving a debug log to {path}, please check it for sensitive information before sharing it. msg.debug.unable_to_write_file Unable to write to our debug log file ({path}): {error}

-msg.panel.graphing.prepopulation_all_successful Read the last day of bandwidth history from the state file -msg.panel.graphing.prepopulation_successful Read the last day of bandwidth history from the state file ({duration} is missing) -msg.panel.graphing.prepopulation_failure Unable to prepopulate bandwidth information ({error}) +msg.panel.graphing.prepopulation_successful Bandwidth graph has information for the last {duration} +msg.panel.graphing.bw_event_cache_malformed Tor's 'GETINFO bw-event-cache' provided malformed output: {response} msg.panel.header.fd_used_at_sixty_percent Tor's file descriptor usage is at {percentage}%. msg.panel.header.fd_used_at_ninety_percent Tor's file descriptor usage is at {percentage}%. If you run out Tor will be unable to continue functioning.

diff --git a/seth/graph_panel.py b/seth/graph_panel.py index b89c667..1e248a3 100644 --- a/seth/graph_panel.py +++ b/seth/graph_panel.py @@ -19,7 +19,7 @@ import seth.controller import seth.popups import seth.util.tracker

-from seth.util import bandwidth_from_state, join, msg, panel, tor_controller +from seth.util import join, msg, panel, tor_controller

from stem.control import EventType, Listener from stem.util import conf, enum, log, str_tools, system @@ -77,7 +77,6 @@ CONFIG = conf.config_dict('seth', { 'features.graph.bound': Bounds.LOCAL_MAX, 'features.graph.max_width': 150, 'features.panels.show.connection': True, - 'features.graph.bw.prepopulate': True, 'features.graph.bw.transferInBytes': False, 'features.graph.bw.accounting.show': True, 'tor.chroot': '', @@ -200,7 +199,7 @@ class BandwidthStats(GraphCategory): entry_comp = entry.split(',')

if len(entry_comp) != 2 or not entry_comp[0].isdigit() or not entry_comp[1].isdigit(): - log.warn("Tor's 'GETINFO bw-event-cache' provided malformed output: %s" % bw_entries) + log.warn(msg('panel.graphing.bw_event_cache_malformed', response = bw_entries)) is_successful = False break

@@ -208,7 +207,7 @@ class BandwidthStats(GraphCategory): self.secondary.update(int(entry_comp[1]))

if is_successful: - log.info('Bandwidth graph has information for the last %s' % str_tools.time_label(len(bw_entries.split()), is_long = True)) + log.info(msg('panel.graphing.prepopulation_successful', duration = str_tools.time_label(len(bw_entries.split()), is_long = True)))

read_total = controller.get_info('traffic/read', None) write_total = controller.get_info('traffic/written', None) @@ -271,39 +270,6 @@ class BandwidthStats(GraphCategory):

self.title_stats = stats

- def prepopulate_from_state(self): - """ - Attempts to use tor's state file to prepopulate values for the 15 minute - interval via the BWHistoryReadValues/BWHistoryWriteValues values. - - :returns: **float** for the number of seconds of data missing - - :raises: **ValueError** if unable to get the bandwidth information from our - state file - """ - - def update_values(stat, entries, latest_time): - # fill missing entries with the last value - - missing_entries = int((time.time() - latest_time) / 900) - entries = entries + [entries[-1]] * missing_entries - - # pad if too short and truncate if too long - - entry_count = CONFIG['features.graph.max_width'] - entries = [0] * (entry_count - len(entries)) + entries[-entry_count:] - - stat.values[Interval.FIFTEEN_MINUTE] = entries - stat.max_value[Interval.FIFTEEN_MINUTE] = max(entries) - stat.latest_value = entries[-1] * 900 - - stats = bandwidth_from_state() - - update_values(self.primary, stats.read_entries, stats.last_read_time) - update_values(self.secondary, stats.write_entries, stats.last_write_time) - - return time.time() - min(stats.last_read_time, stats.last_write_time) - def _size_label(self, byte_count, decimal = 1): """ Alias for str_tools.size_label() that accounts for if the user prefers bits @@ -387,23 +353,7 @@ class GraphPanel(panel.Panel): self.set_pause_attr('_stats') self.set_pause_attr('_accounting_stats')

- # prepopulates bandwidth values from state file - controller = tor_controller() - - if controller.is_alive() and CONFIG['features.graph.bw.prepopulate']: - try: - missing_seconds = self._stats[GraphStat.BANDWIDTH].prepopulate_from_state() - - if missing_seconds: - log.notice(msg('panel.graphing.prepopulation_successful', duration = str_tools.time_label(missing_seconds, 0, True))) - else: - log.notice(msg('panel.graphing.prepopulation_all_successful')) - - self.update_interval = Interval.FIFTEEN_MINUTE - except ValueError as exc: - log.info(msg('panel.graphing.prepopulation_failure', error = exc)) - controller.add_event_listener(self._update_accounting, EventType.BW) controller.add_event_listener(self._update_stats, EventType.BW) controller.add_status_listener(lambda *args: self.redraw(True)) diff --git a/seth/util/__init__.py b/seth/util/__init__.py index 76830cd..51f54bc 100644 --- a/seth/util/__init__.py +++ b/seth/util/__init__.py @@ -12,28 +12,17 @@ __all__ = [ 'ui_tools', ]

-import calendar -import collections import os import sys -import time

import stem.connection import stem.util.conf -import stem.util.system

from seth.util import log

TOR_CONTROLLER = None BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-2])

-StateBandwidth = collections.namedtuple('StateBandwidth', ( - 'read_entries', - 'write_entries', - 'last_read_time', - 'last_write_time', -)) - try: uses_settings = stem.util.conf.uses_settings('seth', os.path.join(BASE_DIR, 'config'), lazy_load = False) except IOError as exc: @@ -118,86 +107,3 @@ def msg(message, config, **attr): except: log.notice('BUG: We attempted to use an undefined string resource (%s)' % message) return '' - - -@uses_settings -def bandwidth_from_state(config): - """ - Read Tor's state file to determine its recent bandwidth usage. These - samplings are at fifteen minute granularity, and can only provide results if - we've been running for at least a day. This provides a named tuple with the - following... - - * read_entries and write_entries - - List of the average bytes read or written during each fifteen minute - period, oldest to newest. - - * last_read_time and last_write_time - - Unix timestamp for when the last entry was recorded. - - :returns: **namedtuple** with the state file's bandwidth informaiton - - :raises: **ValueError** if unable to get the bandwidth information from our - state file - """ - - controller = tor_controller() - - if not controller.is_localhost(): - raise ValueError('we can only prepopulate bandwidth information for a local tor instance') - - start_time = stem.util.system.start_time(controller.get_pid(None)) - uptime = time.time() - start_time if start_time else None - - # Only attempt to prepopulate information if we've been running for a day. - # Reason is that the state file stores a day's worth of data, and we don't - # want to prepopulate with information from a prior tor instance. - - if not uptime: - raise ValueError("unable to determine tor's uptime") - elif uptime < (24 * 60 * 60): - raise ValueError("insufficient uptime, tor must've been running for at least a day") - - # read the user's state file in their data directory (usually '~/.tor') - - data_dir = controller.get_conf('DataDirectory', None) - - if not data_dir: - raise ValueError("unable to determine tor's data directory") - - state_path = os.path.join(config.get('tor.chroot', '') + data_dir, 'state') - - try: - with open(state_path) as state_file: - state_content = state_file.readlines() - except IOError as exc: - raise ValueError('unable to read the state file at %s, %s' % (state_path, exc)) - - # We're interested in two types of entries from our state file... - # - # * BWHistory*Values - Comma separated list of bytes we read or wrote - # during each fifteen minute period. The last value is an incremental - # counter for our current period, so ignoring that. - # - # * BWHistory*Ends - When our last sampling was recorded, in UTC. - - attr = {} - - for line in state_content: - line = line.strip() - - if line.startswith('BWHistoryReadValues '): - attr['read_entries'] = [int(entry) / 900 for entry in line[20:].split(',')[:-1]] - elif line.startswith('BWHistoryWriteValues '): - attr['write_entries'] = [int(entry) / 900 for entry in line[21:].split(',')[:-1]] - elif line.startswith('BWHistoryReadEnds '): - attr['last_read_time'] = calendar.timegm(time.strptime(line[18:], '%Y-%m-%d %H:%M:%S')) - 900 - elif line.startswith('BWHistoryWriteEnds '): - attr['last_write_time'] = calendar.timegm(time.strptime(line[19:], '%Y-%m-%d %H:%M:%S')) - 900 - - if len(attr) != 4: - raise ValueError('bandwidth stats missing from state file') - - return StateBandwidth(**attr) diff --git a/sethrc.sample b/sethrc.sample index 6f120b4..0b64736 100644 --- a/sethrc.sample +++ b/sethrc.sample @@ -178,7 +178,6 @@ features.graph.type bandwidth # accounting.show # provides accounting stats if AccountingMax was set

-features.graph.bw.prepopulate true features.graph.bw.transferInBytes false features.graph.bw.accounting.show true

diff --git a/test/util/bandwidth_from_state.py b/test/util/bandwidth_from_state.py deleted file mode 100644 index 87c0a40..0000000 --- a/test/util/bandwidth_from_state.py +++ /dev/null @@ -1,111 +0,0 @@ -import datetime -import io -import time -import unittest - -from mock import Mock, patch - -from seth.util import bandwidth_from_state - -STATE_FILE = """\ -# Tor state file last generated on 2014-07-20 13:05:10 local time -# Other times below are in UTC -# You *do not* need to edit this file. - -EntryGuard mullbinde7 2546FD2B50165C1567A297B02AD73F62DEA127A0 DirCache -EntryGuardAddedBy 2546FD2B50165C1567A297B02AD73F62DEA127A0 0.2.4.10-alpha-dev 2014-07-11 01:18:47 -EntryGuardPathBias 9.000000 9.000000 9.000000 0.000000 0.000000 1.000000 -TorVersion Tor 0.2.4.10-alpha-dev (git-8be6058d8f31e578) -LastWritten 2014-07-20 20:05:10 -TotalBuildTimes 68 -CircuitBuildTimeBin 525 1 -CircuitBuildTimeBin 575 1 -CircuitBuildTimeBin 675 1 -""" - -STATE_FILE_WITH_ENTRIES = STATE_FILE + """\ -BWHistoryReadValues 921600,1843200,2764800,3686400,4608000 -BWHistoryWriteValues 46080000,46080000,92160000,92160000,92160000 -BWHistoryReadEnds %s -BWHistoryWriteEnds %s -""" - - -class TestBandwidthFromState(unittest.TestCase): - @patch('seth.util.tor_controller') - def test_when_not_localhost(self, tor_controller_mock): - tor_controller_mock().is_localhost.return_value = False - - try: - bandwidth_from_state() - self.fail('expected a ValueError') - except ValueError as exc: - self.assertEqual('we can only prepopulate bandwidth information for a local tor instance', str(exc)) - - @patch('seth.util.tor_controller') - def test_unknown_pid(self, tor_controller_mock): - tor_controller_mock().is_localhost.return_value = True - tor_controller_mock().get_pid.return_value = None - - try: - bandwidth_from_state() - self.fail('expected a ValueError') - except ValueError as exc: - self.assertEqual("unable to determine tor's uptime", str(exc)) - - @patch('seth.util.tor_controller') - @patch('stem.util.system.start_time') - def test_insufficient_uptime(self, start_time_mock, tor_controller_mock): - tor_controller_mock().is_localhost.return_value = True - start_time_mock.return_value = time.time() - 60 # one minute of uptime - - try: - bandwidth_from_state() - self.fail('expected a ValueError') - except ValueError as exc: - self.assertEqual("insufficient uptime, tor must've been running for at least a day", str(exc)) - - @patch('seth.util.tor_controller') - @patch('stem.util.system.start_time', Mock(return_value = 50)) - def test_no_data_dir(self, tor_controller_mock): - tor_controller_mock().is_localhost.return_value = True - tor_controller_mock().get_conf.return_value = None - - try: - bandwidth_from_state() - self.fail('expected a ValueError') - except ValueError as exc: - self.assertEqual("unable to determine tor's data directory", str(exc)) - - @patch('seth.util.tor_controller') - @patch('seth.util.open', create = True) - @patch('stem.util.system.start_time', Mock(return_value = 50)) - def test_no_bandwidth_entries(self, open_mock, tor_controller_mock): - tor_controller_mock().is_localhost.return_value = True - tor_controller_mock().get_conf.return_value = '/home/atagar/.tor' - open_mock.return_value = io.BytesIO(STATE_FILE) - - try: - bandwidth_from_state() - self.fail('expected a ValueError') - except ValueError as exc: - self.assertEqual('bandwidth stats missing from state file', str(exc)) - - open_mock.assert_called_once_with('/home/atagar/.tor/state') - - @patch('seth.util.tor_controller') - @patch('seth.util.open', create = True) - @patch('stem.util.system.start_time', Mock(return_value = 50)) - def test_when_successful(self, open_mock, tor_controller_mock): - tor_controller_mock().is_localhost.return_value = True - tor_controller_mock().get_conf.return_value = '/home/atagar/.tor' - - now = int(time.time()) - timestamp = datetime.datetime.utcfromtimestamp(now + 900).strftime('%Y-%m-%d %H:%M:%S') - open_mock.return_value = io.BytesIO(STATE_FILE_WITH_ENTRIES % (timestamp, timestamp)) - - stats = bandwidth_from_state() - self.assertEqual([1024, 2048, 3072, 4096], stats.read_entries) - self.assertEqual([51200, 51200, 102400, 102400], stats.write_entries) - self.assertEqual(now, stats.last_read_time) - self.assertEqual(now, stats.last_write_time)