commit e31becd234fb0729b2e89466b2d1327a40df5642 Author: iwakeh <iwakeh@torproject.org> Date: Sat Jun 17 06:55:24 2017 +0000 Adapts CollecTor to metrics-lib 1.9.0, which removed DescriptorFile. Implements #22652. --- build.xml | 2 +- .../collector/exitlists/ExitListDownloader.java | 39 +++--- .../collector/onionperf/OnionperfDownloader.java | 12 +- .../collector/relaydescs/ArchiveWriter.java | 22 ++-- .../collector/relaydescs/ReferenceChecker.java | 52 ++++---- .../collector/sync/ProcessCriterium.java | 12 +- .../org/torproject/collector/sync/SyncManager.java | 39 +++--- .../torproject/collector/sync/SyncPersistence.java | 140 ++++++++++++--------- .../collector/sync/SyncPersistenceTest.java | 22 ++-- 9 files changed, 178 insertions(+), 162 deletions(-) diff --git a/build.xml b/build.xml index 729dbb0..721043e 100644 --- a/build.xml +++ b/build.xml @@ -11,7 +11,7 @@ <property name="release.version" value="1.1.2-dev" /> <property name="project-main-class" value="org.torproject.collector.Main" /> <property name="name" value="collector"/> - <property name="descriptorversion" value="1.8.2" /> + <property name="descriptorversion" value="1.9.0" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 7b8ccc4..daa3c90 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -9,7 +9,6 @@ import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExitList; @@ -28,7 +27,6 @@ import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; -import java.util.List; import java.util.SortedSet; import java.util.Stack; import java.util.TimeZone; @@ -114,27 +112,28 @@ public class ExitListDownloader extends CollecTorMain { File tarballFile = Paths.get(outputPathName, tarballFormat.format(downloadedDate)).toFile(); + DescriptorParser descriptorParser = + DescriptorSourceFactory.createDescriptorParser(); + int parsedExitLists = 0; + int otherDescriptors = 0; long maxScanMillis = 0L; - try { - DescriptorParser descriptorParser = - DescriptorSourceFactory.createDescriptorParser(); - List<Descriptor> parsedDescriptors = - descriptorParser.parseDescriptors(downloadedExitList.getBytes(), - tarballFile.getName()); - if (parsedDescriptors.size() != 1 - || !(parsedDescriptors.get(0) instanceof ExitList)) { - logger.warn("Could not parse downloaded exit list"); - return; - } - ExitList parsedExitList = (ExitList) parsedDescriptors.get(0); - for (ExitList.Entry entry : parsedExitList.getEntries()) { - for (long scanMillis : entry.getExitAddresses().values()) { - maxScanMillis = Math.max(maxScanMillis, scanMillis); + for (Descriptor descriptor : descriptorParser.parseDescriptors( + downloadedExitList.getBytes(), null, tarballFile.getName())) { + if (descriptor instanceof ExitList) { + parsedExitLists++; + ExitList parsedExitList = (ExitList) descriptor; + for (ExitList.Entry entry : parsedExitList.getEntries()) { + for (long scanMillis : entry.getExitAddresses().values()) { + maxScanMillis = Math.max(maxScanMillis, scanMillis); + } } + } else { + otherDescriptors++; } - } catch (DescriptorParseException e) { - logger.warn("Could not parse downloaded exit list", - e); + } + if (parsedExitLists != 1 || otherDescriptors > 0) { + logger.warn("Could not parse downloaded exit list"); + return; } if (maxScanMillis > 0L && maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) { diff --git a/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java b/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java index 7453234..577d16d 100644 --- a/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java +++ b/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java @@ -8,7 +8,6 @@ import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.TorperfResult; @@ -211,16 +210,17 @@ public class OnionperfDownloader extends CollecTorMain { /* Validate contained descriptors. */ DescriptorParser descriptorParser = DescriptorSourceFactory.createDescriptorParser(); - List<Descriptor> descriptors; + byte[] rawDescriptorBytes; try { - descriptors = descriptorParser.parseDescriptors( - Files.readAllBytes(tempFile.toPath()), tpfFileName); - } catch (IOException | DescriptorParseException e) { - logger.warn("OnionPerf file '{}{}' could not be parsed. " + rawDescriptorBytes = Files.readAllBytes(tempFile.toPath()); + } catch (IOException e) { + logger.warn("OnionPerf file '{}{}' could not be read. " + "Skipping.", baseUrl, tpfFileName, e); tempFile.delete(); return; } + Iterable<Descriptor> descriptors = descriptorParser.parseDescriptors( + rawDescriptorBytes, null, tpfFileName); String message = null; for (Descriptor descriptor : descriptors) { if (!(descriptor instanceof TorperfResult)) { diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index bc8244e..bf9e79b 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -9,13 +9,14 @@ import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; import org.torproject.collector.conf.SourceType; import org.torproject.collector.cron.CollecTorMain; -import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.RelayExtraInfoDescriptor; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.RelayNetworkStatusVote; import org.torproject.descriptor.RelayServerDescriptor; +import org.torproject.descriptor.UnparseableDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -824,16 +825,17 @@ public class ArchiveWriter extends CollecTorMain { private boolean store(byte[] typeAnnotation, byte[] data, File[] outputFiles, boolean[] append) { - try { - logger.trace("Storing " + outputFiles[0]); - if (this.descriptorParser.parseDescriptors(data, - outputFiles[0].getName()).size() != 1) { - logger.info("Relay descriptor file " + outputFiles[0] - + " doesn't contain exactly one descriptor. Storing anyway."); + logger.trace("Storing " + outputFiles[0]); + int parseableDescriptors = 0; + for (Descriptor descriptor : this.descriptorParser.parseDescriptors(data, + null, outputFiles[0].getName())) { + if (!(descriptor instanceof UnparseableDescriptor)) { + parseableDescriptors++; } - } catch (DescriptorParseException e) { - this.logger.info("Could not parse relay descriptor " - + outputFiles[0] + " before storing it to disk. Storing anyway.", e); + } + if (parseableDescriptors != 1) { + logger.info("Relay descriptor file " + outputFiles[0] + + " doesn't contain exactly one descriptor. Storing anyway."); } try { for (int i = 0; i < outputFiles.length; i++) { diff --git a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java index 59db27f..843156c 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java +++ b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java @@ -4,7 +4,6 @@ package org.torproject.collector.relaydescs; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.DirSourceEntry; @@ -167,36 +166,33 @@ public class ReferenceChecker { private void readNewDescriptors() { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory(this.descriptorsDir); descriptorReader.setHistoryFile(this.historyFile); - Iterator<DescriptorFile> descriptorFiles = - descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - RelayNetworkStatusConsensus consensus = - (RelayNetworkStatusConsensus) descriptor; - String consensusFlavor = consensus.getConsensusFlavor(); - if (consensusFlavor == null) { - this.readRelayNetworkStatusConsensusUnflavored(consensus); - } else if (consensusFlavor.equals("microdesc")) { - this.readRelayNetworkStatusConsensusMicrodesc(consensus); - } else { - /* Ignore unknown consensus flavors. */ - } - } else if (descriptor instanceof RelayNetworkStatusVote) { - this.readRelayNetworkStatusVote( - (RelayNetworkStatusVote) descriptor); - } else if (descriptor instanceof ServerDescriptor) { - this.readServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof ExtraInfoDescriptor) { - this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); - } else if (descriptor instanceof Microdescriptor) { - readMicrodescriptor((Microdescriptor) descriptor); + Iterator<Descriptor> descriptors + = descriptorReader.readDescriptors(this.descriptorsDir).iterator(); + while (descriptors.hasNext()) { + Descriptor descriptor = descriptors.next(); + if (descriptor instanceof RelayNetworkStatusConsensus) { + RelayNetworkStatusConsensus consensus = + (RelayNetworkStatusConsensus) descriptor; + String consensusFlavor = consensus.getConsensusFlavor(); + if (consensusFlavor == null) { + this.readRelayNetworkStatusConsensusUnflavored(consensus); + } else if (consensusFlavor.equals("microdesc")) { + this.readRelayNetworkStatusConsensusMicrodesc(consensus); } else { - /* Ignore unknown descriptors. */ + /* Ignore unknown consensus flavors. */ } + } else if (descriptor instanceof RelayNetworkStatusVote) { + this.readRelayNetworkStatusVote( + (RelayNetworkStatusVote) descriptor); + } else if (descriptor instanceof ServerDescriptor) { + this.readServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExtraInfoDescriptor) { + this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); + } else if (descriptor instanceof Microdescriptor) { + readMicrodescriptor((Microdescriptor) descriptor); + } else { + /* Ignore unknown descriptors. */ } } descriptorReader.saveHistoryFile(this.historyFile); diff --git a/src/main/java/org/torproject/collector/sync/ProcessCriterium.java b/src/main/java/org/torproject/collector/sync/ProcessCriterium.java index 1fa05aa..c128e14 100644 --- a/src/main/java/org/torproject/collector/sync/ProcessCriterium.java +++ b/src/main/java/org/torproject/collector/sync/ProcessCriterium.java @@ -4,10 +4,9 @@ package org.torproject.collector.sync; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; /** Should a descriptor file be processed during sync. */ -public class ProcessCriterium implements Criterium<DescriptorFile> { +public class ProcessCriterium implements Criterium<Descriptor> { private final Class<? extends Descriptor> wantedType; @@ -17,13 +16,8 @@ public class ProcessCriterium implements Criterium<DescriptorFile> { /** Only process descriptors with the appropriate type. */ @Override - public boolean applies(DescriptorFile file) { - for (Descriptor desc : file.getDescriptors()) { - if (!this.wantedType.isInstance(desc)) { - return false; - } - } - return true; + public boolean applies(Descriptor desc) { + return this.wantedType.isInstance(desc); } } diff --git a/src/main/java/org/torproject/collector/sync/SyncManager.java b/src/main/java/org/torproject/collector/sync/SyncManager.java index d8d2708..0c85a47 100644 --- a/src/main/java/org/torproject/collector/sync/SyncManager.java +++ b/src/main/java/org/torproject/collector/sync/SyncManager.java @@ -9,9 +9,9 @@ import org.torproject.collector.conf.Key; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorCollector; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.UnparseableDescriptor; import org.torproject.descriptor.index.DescriptorIndexCollector; import org.slf4j.Logger; @@ -70,15 +70,17 @@ public class SyncManager { String marker, Configuration conf) throws ConfigurationException { Path basePath = conf.getPath(Key.SyncPath); SyncPersistence persist = new SyncPersistence(conf); + Criterium<Descriptor> unparseable + = new ProcessCriterium(UnparseableDescriptor.class); for (URL source : sources) { File base = new File(basePath.toFile(), marker + "-" + source.getHost()); log.info("Merging {} from {} into storage ...", marker, source.getHost()); for (Map.Entry<String, Class<? extends Descriptor>> entry : mapPathDesc.entrySet()) { + File descFile = new File(base, entry.getKey()); DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory(new File(base, entry.getKey())); String histFileEnding = entry.getValue().getSimpleName() + (entry.getKey().contains("consensus-microdesc") ? "-micro" : ""); @@ -87,28 +89,27 @@ public class SyncManager { + histFileEnding); descriptorReader.setHistoryFile(historyFile); log.info("Reading {} of type {} ... ", marker, histFileEnding); - Iterator<DescriptorFile> descriptorFiles - = descriptorReader.readDescriptors(); + Iterator<Descriptor> descriptors + = descriptorReader.readDescriptors(descFile).iterator(); log.info("Done reading {} of type {}.", marker, histFileEnding); - Criterium crit = new ProcessCriterium(entry.getValue()); - while (descriptorFiles.hasNext()) { - DescriptorFile descFile = descriptorFiles.next(); - log.debug("Operating on desc-file containing {} descs.", - descFile.getDescriptors().size()); - if (!crit.applies(descFile)) { - log.warn("Not processing {} in {}.", descFile.getFileName(), - descFile.getDirectory()); + Criterium<Descriptor> crit = new ProcessCriterium(entry.getValue()); + while (descriptors.hasNext()) { + Descriptor desc = descriptors.next(); + if (unparseable.applies(desc)) { + Exception ex + = ((UnparseableDescriptor)desc).getDescriptorParseException(); + log.warn("Parsing of {} caused Exception(s). Processing anyway.", + desc.getDescriptorFile(), ex); + } + if (!crit.applies(desc)) { + log.warn("Not processing {} in {}.", desc.getClass().getName(), + desc.getDescriptorFile()); continue; } - Exception ex = descFile.getException(); - if (null != ex) { - log.warn("Parsing of {} caused Exception(s). Processing anyway.", - descFile.getDirectory() + "/" + descFile.getFileName(), ex); - } - persist.storeDescs(descFile.getDescriptors(), - descFile.getFile().getName(), collectionDate.getTime()); + persist.storeDesc(desc, collectionDate.getTime()); } + persist.cleanDirectory(); descriptorReader.saveHistoryFile(historyFile); } log.info("Done merging {} from {}.", marker, source.getHost()); diff --git a/src/main/java/org/torproject/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/collector/sync/SyncPersistence.java index 5f18f63..5fd6dc6 100644 --- a/src/main/java/org/torproject/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/collector/sync/SyncPersistence.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; -import java.util.List; /** Provides persistence for descriptors based on the descriptor type. */ public class SyncPersistence { @@ -55,74 +54,93 @@ public class SyncPersistence { } /** - * Stores the list of descriptors in main storage and recent. + * Cleans the directory in <code>RecentPath</code> after storing descriptors. + */ + public void cleanDirectory() { + try { + PersistenceUtils.cleanDirectory(recentPath); + } catch (IOException ioe) { + log.error("Cleaning of {} failed.", recentPath.toString(), ioe); + } + } + + /** + * Stores descriptors in main storage and recent. * The storage locations are taken from <code>collector.properties</code>' * options <code>OutputPath</code> and <code>RecentPath</code>. */ - public void storeDescs(List<Descriptor> descs, String filename, - long received) { + public void storeDescs(Iterable<Descriptor> descs, long received) { for (Descriptor desc : descs) { - boolean recognizedAndWritten = false; - for (Class clazz : desc.getClass().getInterfaces()) { - DescriptorPersistence descPersist = null; - switch (clazz.getSimpleName()) { - case "RelayNetworkStatusVote": - descPersist - = new VotePersistence((RelayNetworkStatusVote) desc, received); - break; - case "RelayNetworkStatusConsensus": - RelayNetworkStatusConsensus cons = - (RelayNetworkStatusConsensus) desc; - if (null == cons.getConsensusFlavor()) { - descPersist = new ConsensusPersistence(cons, received); - } else if ("microdesc".equals(cons.getConsensusFlavor())) { - descPersist = new MicroConsensusPersistence(cons, received); - } - break; - case "RelayServerDescriptor": - descPersist = new ServerDescriptorPersistence( - (RelayServerDescriptor) desc, received); - break; - case "BridgeExtraInfoDescriptor": - descPersist = new BridgeExtraInfoPersistence( - (BridgeExtraInfoDescriptor) desc, received); - break; - case "RelayExtraInfoDescriptor": - descPersist = new ExtraInfoPersistence( - (RelayExtraInfoDescriptor) desc, received); - break; - case "BridgeNetworkStatus": // need to infer authId from filename - descPersist = new StatusPersistence( - (BridgeNetworkStatus) desc, filename.split(DASH)[2], received); - break; - case "BridgeServerDescriptor": - descPersist = new BridgeServerDescriptorPersistence( - (BridgeServerDescriptor) desc, received); - break; - case "ExitList": // downloaded is part of desc, which to use? - descPersist = new ExitlistPersistence((ExitList) desc, received); + storeDesc(desc, received); + } + } + + /** + * Stores a descriptor in main storage and recent. + * The storage locations are taken from <code>collector.properties</code>' + * options <code>OutputPath</code> and <code>RecentPath</code>. + */ + public void storeDesc(Descriptor desc, long received) { + String filename = desc.getDescriptorFile().getName(); + boolean recognizedAndWritten = false; + for (Class clazz : desc.getClass().getInterfaces()) { + DescriptorPersistence descPersist = null; + switch (clazz.getSimpleName()) { + case "RelayNetworkStatusVote": + descPersist + = new VotePersistence((RelayNetworkStatusVote) desc, received); + break; + case "RelayNetworkStatusConsensus": + RelayNetworkStatusConsensus cons = + (RelayNetworkStatusConsensus) desc; + if (null == cons.getConsensusFlavor()) { + descPersist = new ConsensusPersistence(cons, received); + } else if ("microdesc".equals(cons.getConsensusFlavor())) { + descPersist = new MicroConsensusPersistence(cons, received); + } + break; + case "RelayServerDescriptor": + descPersist = new ServerDescriptorPersistence( + (RelayServerDescriptor) desc, received); + break; + case "BridgeExtraInfoDescriptor": + descPersist = new BridgeExtraInfoPersistence( + (BridgeExtraInfoDescriptor) desc, received); + break; + case "RelayExtraInfoDescriptor": + descPersist = new ExtraInfoPersistence( + (RelayExtraInfoDescriptor) desc, received); + break; + case "BridgeNetworkStatus": // need to infer authId from filename + String[] filenameParts = filename.split(DASH); + if (filenameParts.length < 3) { + log.error("Invalid BridgeNetworkStatus; skipping: {}.", filename); break; - default: - log.trace("Invalid descriptor type {} for sync-merge.", - clazz.getName()); - continue; - } - if (null != descPersist) { - descPersist.storeAll(recentPathName, outputPathName); - recognizedAndWritten = true; - } - break; + } + descPersist = new StatusPersistence( + (BridgeNetworkStatus) desc, filenameParts[2], received); + break; + case "BridgeServerDescriptor": + descPersist = new BridgeServerDescriptorPersistence( + (BridgeServerDescriptor) desc, received); + break; + case "ExitList": // downloaded is part of desc, which to use? + descPersist = new ExitlistPersistence((ExitList) desc, received); + break; + default: + log.trace("Invalid descriptor type {} for sync-merge.", + clazz.getName()); + continue; } - if (!recognizedAndWritten) { - log.error("Unknown descriptor type {} implementing {}.", - desc.getClass().getSimpleName(), desc.getClass().getInterfaces()); + if (null != descPersist) { + descPersist.storeAll(recentPathName, outputPathName); + recognizedAndWritten = true; } + break; } - try { - PersistenceUtils.cleanDirectory(recentPath); - } catch (IOException ioe) { - log.error("Cleaning of {} failed.", recentPath.toString(), ioe); + if (!recognizedAndWritten) { + log.error("Unknown descriptor type {} implementing {}.", + desc.getClass().getSimpleName(), desc.getClass().getInterfaces()); } } - } diff --git a/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java b/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java index f5aee87..fe47f26 100644 --- a/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java +++ b/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java @@ -193,6 +193,7 @@ public class SyncPersistenceTest { private int outCount; private int recentCount; private String filename; + private File fileForName; private File recent; private File output; private String recentName; @@ -205,6 +206,7 @@ public class SyncPersistenceTest { this.expectedRecentDir = subRecent; this.expectedOutputDir = subOutput; this.filename = fn; + this.fileForName = new File(fn); this.outCount = outCount; this.recentCount = recentCount; } @@ -220,8 +222,9 @@ public class SyncPersistenceTest { makeTemporaryFolders(); DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser(); SyncPersistence persist = new SyncPersistence(conf); - persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename), - filename, 1475694377000L); + persist.storeDescs(dp.parseDescriptors(bytesFromResource(), + fileForName, filename), 1475694377000L); + persist.cleanDirectory(); List<File> recentList = new ArrayList<>(); Files.walkFileTree(recent.toPath(), new FileCollector(recentList)); String dataUsed = "data used: " + expectedRecentDir + ", " + filename @@ -237,8 +240,9 @@ public class SyncPersistenceTest { makeTemporaryFolders(); DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser(); SyncPersistence persist = new SyncPersistence(conf); - persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename), - filename, 1475694377000L); + persist.storeDescs(dp.parseDescriptors(bytesFromResource(), + fileForName, filename), 1475694377000L); + persist.cleanDirectory(); List<File> recentList = new ArrayList<>(); List<File> outputList = new ArrayList<>(); Files.walkFileTree(output.toPath(), new FileCollector(outputList)); @@ -260,8 +264,9 @@ public class SyncPersistenceTest { makeTemporaryFolders(); DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser(); SyncPersistence persist = new SyncPersistence(conf); - persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename), - filename, 1475694377000L); + persist.storeDescs(dp.parseDescriptors(bytesFromResource(), + fileForName, filename), 1475694377000L); + persist.cleanDirectory(); List<File> recentList = new ArrayList<>(); Files.walkFileTree(recent.toPath(), new FileCollector(recentList)); assertEquals(recentCount, recentList.size()); @@ -284,8 +289,9 @@ public class SyncPersistenceTest { makeTemporaryFolders(); DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser(); SyncPersistence persist = new SyncPersistence(conf); - persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename), - filename, 1475694377000L); + persist.storeDescs(dp.parseDescriptors(bytesFromResource(), + fileForName, filename), 1475694377000L); + persist.cleanDirectory(); List<File> outputList = new ArrayList<>(); Files.walkFileTree(output.toPath(), new FileCollector(outputList)); assertEquals(outCount, outputList.size());