commit 8d09f56568652bc9104901ca4f27e170f8aa9316 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Jan 31 15:58:05 2017 +0100

Avoid deleting extraneous local descriptor files.

DescriptorIndexCollector deletes descriptor files from a previous or concurrent collect run if it doesn't collect those files itself. This is unexpected behavior and differs from what DescriptorCollectorImpl does.

Fixes #20525. --- CHANGELOG.md | 2 ++ .../descriptor/index/DescriptorIndexCollector.java | 19 ++++++++++++------- .../index/DescriptorIndexCollectorTest.java | 6 ++++-- 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 22586d2..afaa1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ - Unify the build process by adding git-submodule metrics-base in src/build and removing all centralized parts of the build process. + - Avoid deleting extraneous local descriptor files when collecting + descriptors from CollecTor.

# Changes in version 1.5.0 - 2016-10-19 diff --git a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java index a6a5fe8..68b6591 100644 --- a/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java +++ b/src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java @@ -87,7 +87,8 @@ public class DescriptorIndexCollector implements DescriptorCollector { if (deleteExtraneousLocalFiles) { log.info("Deleting extraneous files from local directory {}.", localDirectory); - deleteExtraneousLocalFiles(remoteFiles, localDirectory, localFiles); + deleteExtraneousLocalFiles(remoteDirectories, remoteFiles, localDirectory, + localFiles); } log.info("Finished descriptor collection."); } @@ -138,15 +139,19 @@ public class DescriptorIndexCollector implements DescriptorCollector { return true; }

- static void deleteExtraneousLocalFiles( + static void deleteExtraneousLocalFiles(String[] remoteDirectories, SortedMap<String, FileNode> remoteFiles, File localDir, SortedMap<String, Long> locals) { for (String localPath : locals.keySet()) { - if (!remoteFiles.containsKey(localPath)) { - File extraneousLocalFile = new File(localDir, localPath); - log.debug("Deleting extraneous local file {}.", - extraneousLocalFile.getAbsolutePath()); - extraneousLocalFile.delete(); + for (String remoteDirectory : remoteDirectories) { + if (localPath.startsWith(remoteDirectory)) { + if (!remoteFiles.containsKey(localPath)) { + File extraneousLocalFile = new File(localDir, localPath); + log.debug("Deleting extraneous local file {}.", + extraneousLocalFile.getAbsolutePath()); + extraneousLocalFile.delete(); + } + } } } } diff --git a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java index a40e0e3..e2a31b5 100644 --- a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java +++ b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java @@ -139,6 +139,8 @@ public class DescriptorIndexCollectorTest { public void testNormalCollectingWithDeletion() throws Exception { File localFolder = tmpf.newFolder(); makeStructure(localFolder, "1"); + File nonSyncedDir = makeDirs(localFolder.toString(), "d"); + makeFiles(nonSyncedDir, "p1");

File remoteDirectory = tmpf.newFolder(); makeStructure(remoteDirectory, "2"); @@ -158,10 +160,10 @@ public class DescriptorIndexCollectorTest { .collectDescriptors(indexFile.toURL().toString(), new String[]{"a/b", "a/b/c"}, 1451606400_000L, localFolder, true);

- // verify file addition. + // verify file addition, including that the non-synced dir is not touched. checkContains(true, DescriptorIndexCollector.statLocalDirectory(localFolder).toString(), - "a/b/y2", "a/b/x2", "a/b/c/u2"); + "a/b/y2", "a/b/x2", "a/b/c/u2", "d/p1");

// verify that invalid files weren't fetched. checkContains(false,