commit e625e84b003420908d410ab10f51fdd78a1ccf17 Author: Damian Johnson atagar@torproject.org Date: Sat Jun 23 12:16:53 2012 -0700
Respecting reader stop() calls in mid-directory
If we, say, attempted to read a directory with a million files then the reader's stop() method won't be respected until we've finished with that directory. That's stupid. Checking the _is_stopped Event after processing each file.
Python lacks targeted breaks, so I needed to refactor the code a bit to accomidate this. --- stem/descriptor/reader.py | 16 +++++++++------- 1 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 716b58c..8d4a462 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -354,13 +354,7 @@ class DescriptorReader: else: walker = os.walk(target)
- # adds all of the files that it contains - for root, _, files in walker: - for filename in files: - self._handle_file(os.path.join(root, filename), new_processed_files) - - # this can take a while if, say, we're including the root directory - if self._is_stopped.isSet(): break + self._handle_walker(walker, new_processed_files) else: self._handle_file(target, new_processed_files)
@@ -383,6 +377,14 @@ class DescriptorReader: self._iter_notice.wait() self._iter_notice.clear()
+ def _handle_walker(self, walker, new_processed_files): + for root, _, files in walker: + for filename in files: + self._handle_file(os.path.join(root, filename), new_processed_files) + + # this can take a while if, say, we're including the root directory + if self._is_stopped.isSet(): return + def _handle_file(self, target, new_processed_files): # This is a file. Register its last modified timestamp and check if # it's a file that we should skip.