commit 0da547cf8119307f436e642148427e8e58328c92 Author: Damian Johnson atagar@torproject.org Date: Fri Apr 3 09:26:29 2015 -0700

Change tail() to be a generator

Switching tail() to be a generator so it can provide constant memory usage, and allow the caller to abort reading content at any time. --- stem/util/system.py | 30 +++++++++++++++++++----------- test/unit/util/system.py | 16 ++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/stem/util/system.py b/stem/util/system.py index 56f3da5..f33d8b7 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -750,19 +750,27 @@ def start_time(pid):

def tail(target, lines = None): """ - Provides the last lines from a file, similar to 'tail -n 50 /tmp/my_log'. + Provides lines of a file starting with the end. For instance, + 'tail -n 50 /tmp/my_log' could be done with... + + :: + + reversed(list(tail('/tmp/my_log', 50)))

:param str,file target: path or file object to read from :param int lines: number of lines to read

- :returns: **list** of lines the file ends with + :returns: **generator** that reads lines, starting with the end

:raises: **IOError** if unable to read the file """

if isinstance(target, str): with open(target) as target_file: - return tail(target_file, lines) + for line in tail(target_file, lines): + yield line + + return

if lines is None: lines = sys.maxint @@ -774,26 +782,26 @@ def tail(target, lines = None): block_end_byte = target.tell() lines_left = lines block_number = -1 - blocks = [] # blocks of size BLOCK_SIZE, in reverse order + content = ''

while lines_left > 0 and block_end_byte > 0: if (block_end_byte - BLOCK_SIZE > 0): # read the last block we haven't yet read target.seek(block_number * BLOCK_SIZE, 2) - blocks.insert(0, target.read(BLOCK_SIZE)) + content, completed_lines = (target.read(BLOCK_SIZE) + content).split('\n', 1) else: # reached the start of the file, just read what's left target.seek(0, 0) - blocks.insert(0, target.read(block_end_byte)) + completed_lines = target.read(block_end_byte) + content + + for line in reversed(completed_lines.splitlines()): + if lines_left > 0: + lines_left -= 1 + yield line

- lines_found = blocks[-1].count('\n') - lines_left -= lines_found block_end_byte -= BLOCK_SIZE block_number -= 1

- text = ''.join(blocks) - return text.splitlines()[-lines:] -

def bsd_jail_id(pid): """ diff --git a/test/unit/util/system.py b/test/unit/util/system.py index e3de4d8..dd1e30f 100644 --- a/test/unit/util/system.py +++ b/test/unit/util/system.py @@ -383,20 +383,20 @@ class TestSystem(unittest.TestCase): # by file handle

with open(path) as riddle_file: - self.assertEqual([' both the wicked and sweet.'], system.tail(riddle_file, 1)) + self.assertEqual([' both the wicked and sweet.'], list(system.tail(riddle_file, 1)))

- self.assertEqual([], system.tail(path, 0)) - self.assertEqual([' both the wicked and sweet.'], system.tail(path, 1)) - self.assertEqual(["but I'm with people you meet", ' both the wicked and sweet.'], system.tail(path, 2)) + self.assertEqual([], list(system.tail(path, 0))) + self.assertEqual([' both the wicked and sweet.'], list(system.tail(path, 1))) + self.assertEqual([' both the wicked and sweet.', "but I'm with people you meet"], list(system.tail(path, 2)))

- self.assertEqual(14, len(system.tail(path))) - self.assertEqual(14, len(system.tail(path, 200))) + self.assertEqual(14, len(list(system.tail(path)))) + self.assertEqual(14, len(list(system.tail(path, 200))))

- self.assertRaises(IOError, system.tail, '/path/doesnt/exist') + self.assertRaises(IOError, list, system.tail('/path/doesnt/exist'))

fd, temp_path = tempfile.mkstemp() os.chmod(temp_path, 0o077) # remove read permissions - self.assertRaises(IOError, system.tail, temp_path) + self.assertRaises(IOError, list, system.tail(temp_path)) os.close(fd) os.remove(temp_path)