commit 0da547cf8119307f436e642148427e8e58328c92
Author: Damian Johnson <atagar(a)torproject.org>
Date: Fri Apr 3 09:26:29 2015 -0700
Change tail() to be a generator
Switching tail() to be a generator so it can provide constant memory usage, and
allow the caller to abort reading content at any time.
---
stem/util/system.py | 30 +++++++++++++++++++-----------
test/unit/util/system.py | 16 ++++++++--------
2 files changed, 27 insertions(+), 19 deletions(-)
diff --git a/stem/util/system.py b/stem/util/system.py
index 56f3da5..f33d8b7 100644
--- a/stem/util/system.py
+++ b/stem/util/system.py
@@ -750,19 +750,27 @@ def start_time(pid):
def tail(target, lines = None):
"""
- Provides the last lines from a file, similar to 'tail -n 50 /tmp/my_log'.
+ Provides lines of a file starting with the end. For instance,
+ 'tail -n 50 /tmp/my_log' could be done with...
+
+ ::
+
+ reversed(list(tail('/tmp/my_log', 50)))
:param str,file target: path or file object to read from
:param int lines: number of lines to read
- :returns: **list** of lines the file ends with
+ :returns: **generator** that reads lines, starting with the end
:raises: **IOError** if unable to read the file
"""
if isinstance(target, str):
with open(target) as target_file:
- return tail(target_file, lines)
+ for line in tail(target_file, lines):
+ yield line
+
+ return
if lines is None:
lines = sys.maxint
@@ -774,26 +782,26 @@ def tail(target, lines = None):
block_end_byte = target.tell()
lines_left = lines
block_number = -1
- blocks = [] # blocks of size BLOCK_SIZE, in reverse order
+ content = ''
while lines_left > 0 and block_end_byte > 0:
if (block_end_byte - BLOCK_SIZE > 0):
# read the last block we haven't yet read
target.seek(block_number * BLOCK_SIZE, 2)
- blocks.insert(0, target.read(BLOCK_SIZE))
+ content, completed_lines = (target.read(BLOCK_SIZE) + content).split('\n', 1)
else:
# reached the start of the file, just read what's left
target.seek(0, 0)
- blocks.insert(0, target.read(block_end_byte))
+ completed_lines = target.read(block_end_byte) + content
+
+ for line in reversed(completed_lines.splitlines()):
+ if lines_left > 0:
+ lines_left -= 1
+ yield line
- lines_found = blocks[-1].count('\n')
- lines_left -= lines_found
block_end_byte -= BLOCK_SIZE
block_number -= 1
- text = ''.join(blocks)
- return text.splitlines()[-lines:]
-
def bsd_jail_id(pid):
"""
diff --git a/test/unit/util/system.py b/test/unit/util/system.py
index e3de4d8..dd1e30f 100644
--- a/test/unit/util/system.py
+++ b/test/unit/util/system.py
@@ -383,20 +383,20 @@ class TestSystem(unittest.TestCase):
# by file handle
with open(path) as riddle_file:
- self.assertEqual([' both the wicked and sweet.'], system.tail(riddle_file, 1))
+ self.assertEqual([' both the wicked and sweet.'], list(system.tail(riddle_file, 1)))
- self.assertEqual([], system.tail(path, 0))
- self.assertEqual([' both the wicked and sweet.'], system.tail(path, 1))
- self.assertEqual(["but I'm with people you meet", ' both the wicked and sweet.'], system.tail(path, 2))
+ self.assertEqual([], list(system.tail(path, 0)))
+ self.assertEqual([' both the wicked and sweet.'], list(system.tail(path, 1)))
+ self.assertEqual([' both the wicked and sweet.', "but I'm with people you meet"], list(system.tail(path, 2)))
- self.assertEqual(14, len(system.tail(path)))
- self.assertEqual(14, len(system.tail(path, 200)))
+ self.assertEqual(14, len(list(system.tail(path))))
+ self.assertEqual(14, len(list(system.tail(path, 200))))
- self.assertRaises(IOError, system.tail, '/path/doesnt/exist')
+ self.assertRaises(IOError, list, system.tail('/path/doesnt/exist'))
fd, temp_path = tempfile.mkstemp()
os.chmod(temp_path, 0o077) # remove read permissions
- self.assertRaises(IOError, system.tail, temp_path)
+ self.assertRaises(IOError, list, system.tail(temp_path))
os.close(fd)
os.remove(temp_path)