[tor-commits] [metrics-lib/master] Avoid deleting extraneous local descriptor files.

[karsten at torproject.org]

commit 8d09f56568652bc9104901ca4f27e170f8aa9316
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Tue Jan 31 15:58:05 2017 +0100

    Avoid deleting extraneous local descriptor files.
    
    DescriptorIndexCollector deletes descriptor files from a previous or
    concurrent collect run if it doesn't collect those files itself.  This
    is unexpected behavior and differs from what DescriptorCollectorImpl
    does.
    
    Fixes #20525.
---
 CHANGELOG.md                                          |  2 ++
 .../descriptor/index/DescriptorIndexCollector.java    | 19 ++++++++++++-------
 .../index/DescriptorIndexCollectorTest.java           |  6 ++++--
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22586d2..afaa1e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@
    - Unify the build process by adding git-submodule metrics-base in
      src/build and removing all centralized parts of the build
      process.
+   - Avoid deleting extraneous local descriptor files when collecting
+     descriptors from CollecTor.
 
 
 # Changes in version 1.5.0 - 2016-10-19
diff --git a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
index a6a5fe8..68b6591 100644
--- a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
+++ b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java
@@ -87,7 +87,8 @@ public class DescriptorIndexCollector implements DescriptorCollector {
     if (deleteExtraneousLocalFiles) {
       log.info("Deleting extraneous files from local directory {}.",
           localDirectory);
-      deleteExtraneousLocalFiles(remoteFiles, localDirectory, localFiles);
+      deleteExtraneousLocalFiles(remoteDirectories, remoteFiles, localDirectory,
+          localFiles);
     }
     log.info("Finished descriptor collection.");
   }
@@ -138,15 +139,19 @@ public class DescriptorIndexCollector implements DescriptorCollector {
     return true;
   }
 
-  static void deleteExtraneousLocalFiles(
+  static void deleteExtraneousLocalFiles(String[] remoteDirectories,
       SortedMap<String, FileNode> remoteFiles,
       File localDir, SortedMap<String, Long> locals) {
     for (String localPath : locals.keySet()) {
-      if (!remoteFiles.containsKey(localPath)) {
-        File extraneousLocalFile = new File(localDir, localPath);
-        log.debug("Deleting extraneous local file {}.",
-            extraneousLocalFile.getAbsolutePath());
-        extraneousLocalFile.delete();
+      for (String remoteDirectory : remoteDirectories) {
+        if (localPath.startsWith(remoteDirectory)) {
+          if (!remoteFiles.containsKey(localPath)) {
+            File extraneousLocalFile = new File(localDir, localPath);
+            log.debug("Deleting extraneous local file {}.",
+                extraneousLocalFile.getAbsolutePath());
+            extraneousLocalFile.delete();
+          }
+        }
       }
     }
   }
diff --git a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java
index a40e0e3..e2a31b5 100644
--- a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java
+++ b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java
@@ -139,6 +139,8 @@ public class DescriptorIndexCollectorTest {
   public void testNormalCollectingWithDeletion() throws Exception {
     File localFolder = tmpf.newFolder();
     makeStructure(localFolder, "1");
+    File nonSyncedDir = makeDirs(localFolder.toString(), "d");
+    makeFiles(nonSyncedDir, "p1");
 
     File remoteDirectory = tmpf.newFolder();
     makeStructure(remoteDirectory, "2");
@@ -158,10 +160,10 @@ public class DescriptorIndexCollectorTest {
         .collectDescriptors(indexFile.toURL().toString(),
             new String[]{"a/b", "a/b/c"}, 1451606400_000L, localFolder, true);
 
-    // verify file addition.
+    // verify file addition, including that the non-synced dir is not touched.
     checkContains(true,
         DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
-        "a/b/y2", "a/b/x2", "a/b/c/u2");
+        "a/b/y2", "a/b/x2", "a/b/c/u2", "d/p1");
 
     // verify that invalid files weren't fetched.
     checkContains(false,