commit 34d14afca254eac8f0493f9c9b8aabc7c5400899 Author: Damian Johnson <atagar@torproject.org> Date: Mon Mar 12 09:27:31 2012 -0700 Notifying skip listeners of files that were already read Skip listeners are supposed to be notified of all files that we skip, but wasn't accounting for files that we skipped because their 'last modified' timestamp indicated that they'd already been read. Modifying the integ test for set_processed_files() to test this listener condition (more graceful that the prior test that it had been doing). --- stem/descriptor/reader.py | 9 +++++++++ test/integ/descriptor/reader.py | 29 ++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 27557da..6026408 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -59,6 +59,7 @@ DescriptorReader - Iterator for descriptor data on the local file system. +- __iter__ - iterates over descriptor data in unread files FileSkipped - Base exception for a file that was skipped. + |- AlreadyRead - We've already read a file with this last modified timestamp. |- ParsingFailure - Contents can't be parsed as descriptor data. |- UnrecognizedType - File extension indicates non-descriptor data. +- ReadFailed - Wraps an error that was raised while reading the file. @@ -86,6 +87,13 @@ FINISHED = "DONE" class FileSkipped(Exception): "Base error when we can't provide descriptor data from a file." +class AlreadyRead(FileSkipped): + "Already read a file with this 'last modified' timestamp or later." + + def __init__(self, last_modified, last_modified_when_read): + self.last_modified = last_modified + self.last_modified_when_read = last_modified_when_read + class ParsingFailure(FileSkipped): "File contents could not be parsed as descriptor data." @@ -306,6 +314,7 @@ class DescriptorReader: last_used = self._processed_files.get(target) if last_used and last_used >= last_modified: + self._notify_skip_listeners(target, AlreadyRead(last_modified, last_used)) continue else: self._processed_files[target] = last_modified diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index cb3454a..fbacc36 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -217,28 +217,31 @@ class TestDescriptorReader(unittest.TestCase): self.assertEquals(expected_results, reader.get_processed_files()) - def test_set_processed_files(self): + def test_skip_listener_already_read(self): """ Checks that calling set_processed_files() prior to reading makes us skip - those files. + those files. This also doubles for testing that skip listeners are notified + of files that we've already read. """ - # path and file contents that we want the DescriptorReader to skip - skip_file = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") - - with open(skip_file) as descriptor_file: - skip_contents = descriptor_file.read() - - initial_processed_files = {skip_file: sys.maxint} + # path that we want the DescriptorReader to skip + test_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") + initial_processed_files = {test_path: sys.maxint} + skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader([DESCRIPTOR_TEST_DATA]) + reader.register_skip_listener(skip_listener.listener) reader.set_processed_files(initial_processed_files) + self.assertEquals(initial_processed_files, reader.get_processed_files()) + with reader: list(reader) # iterates over all of the descriptors - with reader: - for descriptor in reader: - if str(descriptor) == skip_contents: - self.fail() # we read the file that we were trying to skip + self.assertTrue(len(skip_listener.results) == 1) + + skipped_path, skip_exception = skip_listener.results[0] + self.assertEqual(test_path, skipped_path) + self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.AlreadyRead)) + self.assertEqual(sys.maxint, skip_exception.last_modified_when_read) def test_skip_listener_unrecognized_type(self): """