[tor-commits] [collector/master] Adapts CollecTor to metrics-lib 1.9.0, which removed DescriptorFile.
karsten at torproject.org
karsten at torproject.org
Wed Jul 5 19:40:50 UTC 2017
commit e31becd234fb0729b2e89466b2d1327a40df5642
Author: iwakeh <iwakeh at torproject.org>
Date: Sat Jun 17 06:55:24 2017 +0000
Adapts CollecTor to metrics-lib 1.9.0, which removed DescriptorFile.
Implements #22652.
---
build.xml | 2 +-
.../collector/exitlists/ExitListDownloader.java | 39 +++---
.../collector/onionperf/OnionperfDownloader.java | 12 +-
.../collector/relaydescs/ArchiveWriter.java | 22 ++--
.../collector/relaydescs/ReferenceChecker.java | 52 ++++----
.../collector/sync/ProcessCriterium.java | 12 +-
.../org/torproject/collector/sync/SyncManager.java | 39 +++---
.../torproject/collector/sync/SyncPersistence.java | 140 ++++++++++++---------
.../collector/sync/SyncPersistenceTest.java | 22 ++--
9 files changed, 178 insertions(+), 162 deletions(-)
diff --git a/build.xml b/build.xml
index 729dbb0..721043e 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
<property name="release.version" value="1.1.2-dev" />
<property name="project-main-class" value="org.torproject.collector.Main" />
<property name="name" value="collector"/>
- <property name="descriptorversion" value="1.8.2" />
+ <property name="descriptorversion" value="1.9.0" />
<property name="jarincludes" value="collector.properties logback.xml" />
<patternset id="runtime" >
diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
index 7b8ccc4..daa3c90 100644
--- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -9,7 +9,6 @@ import org.torproject.collector.conf.ConfigurationException;
import org.torproject.collector.conf.Key;
import org.torproject.collector.cron.CollecTorMain;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.ExitList;
@@ -28,7 +27,6 @@ import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
-import java.util.List;
import java.util.SortedSet;
import java.util.Stack;
import java.util.TimeZone;
@@ -114,27 +112,28 @@ public class ExitListDownloader extends CollecTorMain {
File tarballFile = Paths.get(outputPathName,
tarballFormat.format(downloadedDate)).toFile();
+ DescriptorParser descriptorParser =
+ DescriptorSourceFactory.createDescriptorParser();
+ int parsedExitLists = 0;
+ int otherDescriptors = 0;
long maxScanMillis = 0L;
- try {
- DescriptorParser descriptorParser =
- DescriptorSourceFactory.createDescriptorParser();
- List<Descriptor> parsedDescriptors =
- descriptorParser.parseDescriptors(downloadedExitList.getBytes(),
- tarballFile.getName());
- if (parsedDescriptors.size() != 1
- || !(parsedDescriptors.get(0) instanceof ExitList)) {
- logger.warn("Could not parse downloaded exit list");
- return;
- }
- ExitList parsedExitList = (ExitList) parsedDescriptors.get(0);
- for (ExitList.Entry entry : parsedExitList.getEntries()) {
- for (long scanMillis : entry.getExitAddresses().values()) {
- maxScanMillis = Math.max(maxScanMillis, scanMillis);
+ for (Descriptor descriptor : descriptorParser.parseDescriptors(
+ downloadedExitList.getBytes(), null, tarballFile.getName())) {
+ if (descriptor instanceof ExitList) {
+ parsedExitLists++;
+ ExitList parsedExitList = (ExitList) descriptor;
+ for (ExitList.Entry entry : parsedExitList.getEntries()) {
+ for (long scanMillis : entry.getExitAddresses().values()) {
+ maxScanMillis = Math.max(maxScanMillis, scanMillis);
+ }
}
+ } else {
+ otherDescriptors++;
}
- } catch (DescriptorParseException e) {
- logger.warn("Could not parse downloaded exit list",
- e);
+ }
+ if (parsedExitLists != 1 || otherDescriptors > 0) {
+ logger.warn("Could not parse downloaded exit list");
+ return;
}
if (maxScanMillis > 0L
&& maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) {
diff --git a/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java b/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java
index 7453234..577d16d 100644
--- a/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java
+++ b/src/main/java/org/torproject/collector/onionperf/OnionperfDownloader.java
@@ -8,7 +8,6 @@ import org.torproject.collector.conf.ConfigurationException;
import org.torproject.collector.conf.Key;
import org.torproject.collector.cron.CollecTorMain;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.TorperfResult;
@@ -211,16 +210,17 @@ public class OnionperfDownloader extends CollecTorMain {
/* Validate contained descriptors. */
DescriptorParser descriptorParser =
DescriptorSourceFactory.createDescriptorParser();
- List<Descriptor> descriptors;
+ byte[] rawDescriptorBytes;
try {
- descriptors = descriptorParser.parseDescriptors(
- Files.readAllBytes(tempFile.toPath()), tpfFileName);
- } catch (IOException | DescriptorParseException e) {
- logger.warn("OnionPerf file '{}{}' could not be parsed. "
+ rawDescriptorBytes = Files.readAllBytes(tempFile.toPath());
+ } catch (IOException e) {
+ logger.warn("OnionPerf file '{}{}' could not be read. "
+ "Skipping.", baseUrl, tpfFileName, e);
tempFile.delete();
return;
}
+ Iterable<Descriptor> descriptors = descriptorParser.parseDescriptors(
+ rawDescriptorBytes, null, tpfFileName);
String message = null;
for (Descriptor descriptor : descriptors) {
if (!(descriptor instanceof TorperfResult)) {
diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
index bc8244e..bf9e79b 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -9,13 +9,14 @@ import org.torproject.collector.conf.ConfigurationException;
import org.torproject.collector.conf.Key;
import org.torproject.collector.conf.SourceType;
import org.torproject.collector.cron.CollecTorMain;
-import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.RelayExtraInfoDescriptor;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.RelayNetworkStatusVote;
import org.torproject.descriptor.RelayServerDescriptor;
+import org.torproject.descriptor.UnparseableDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -824,16 +825,17 @@ public class ArchiveWriter extends CollecTorMain {
private boolean store(byte[] typeAnnotation, byte[] data,
File[] outputFiles, boolean[] append) {
- try {
- logger.trace("Storing " + outputFiles[0]);
- if (this.descriptorParser.parseDescriptors(data,
- outputFiles[0].getName()).size() != 1) {
- logger.info("Relay descriptor file " + outputFiles[0]
- + " doesn't contain exactly one descriptor. Storing anyway.");
+ logger.trace("Storing " + outputFiles[0]);
+ int parseableDescriptors = 0;
+ for (Descriptor descriptor : this.descriptorParser.parseDescriptors(data,
+ null, outputFiles[0].getName())) {
+ if (!(descriptor instanceof UnparseableDescriptor)) {
+ parseableDescriptors++;
}
- } catch (DescriptorParseException e) {
- this.logger.info("Could not parse relay descriptor "
- + outputFiles[0] + " before storing it to disk. Storing anyway.", e);
+ }
+ if (parseableDescriptors != 1) {
+ logger.info("Relay descriptor file " + outputFiles[0]
+ + " doesn't contain exactly one descriptor. Storing anyway.");
}
try {
for (int i = 0; i < outputFiles.length; i++) {
diff --git a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java
index 59db27f..843156c 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java
@@ -4,7 +4,6 @@
package org.torproject.collector.relaydescs;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.DirSourceEntry;
@@ -167,36 +166,33 @@ public class ReferenceChecker {
private void readNewDescriptors() {
DescriptorReader descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.addDirectory(this.descriptorsDir);
descriptorReader.setHistoryFile(this.historyFile);
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof RelayNetworkStatusConsensus) {
- RelayNetworkStatusConsensus consensus =
- (RelayNetworkStatusConsensus) descriptor;
- String consensusFlavor = consensus.getConsensusFlavor();
- if (consensusFlavor == null) {
- this.readRelayNetworkStatusConsensusUnflavored(consensus);
- } else if (consensusFlavor.equals("microdesc")) {
- this.readRelayNetworkStatusConsensusMicrodesc(consensus);
- } else {
- /* Ignore unknown consensus flavors. */
- }
- } else if (descriptor instanceof RelayNetworkStatusVote) {
- this.readRelayNetworkStatusVote(
- (RelayNetworkStatusVote) descriptor);
- } else if (descriptor instanceof ServerDescriptor) {
- this.readServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof ExtraInfoDescriptor) {
- this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
- } else if (descriptor instanceof Microdescriptor) {
- readMicrodescriptor((Microdescriptor) descriptor);
+ Iterator<Descriptor> descriptors
+ = descriptorReader.readDescriptors(this.descriptorsDir).iterator();
+ while (descriptors.hasNext()) {
+ Descriptor descriptor = descriptors.next();
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ RelayNetworkStatusConsensus consensus =
+ (RelayNetworkStatusConsensus) descriptor;
+ String consensusFlavor = consensus.getConsensusFlavor();
+ if (consensusFlavor == null) {
+ this.readRelayNetworkStatusConsensusUnflavored(consensus);
+ } else if (consensusFlavor.equals("microdesc")) {
+ this.readRelayNetworkStatusConsensusMicrodesc(consensus);
} else {
- /* Ignore unknown descriptors. */
+ /* Ignore unknown consensus flavors. */
}
+ } else if (descriptor instanceof RelayNetworkStatusVote) {
+ this.readRelayNetworkStatusVote(
+ (RelayNetworkStatusVote) descriptor);
+ } else if (descriptor instanceof ServerDescriptor) {
+ this.readServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof ExtraInfoDescriptor) {
+ this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
+ } else if (descriptor instanceof Microdescriptor) {
+ readMicrodescriptor((Microdescriptor) descriptor);
+ } else {
+ /* Ignore unknown descriptors. */
}
}
descriptorReader.saveHistoryFile(this.historyFile);
diff --git a/src/main/java/org/torproject/collector/sync/ProcessCriterium.java b/src/main/java/org/torproject/collector/sync/ProcessCriterium.java
index 1fa05aa..c128e14 100644
--- a/src/main/java/org/torproject/collector/sync/ProcessCriterium.java
+++ b/src/main/java/org/torproject/collector/sync/ProcessCriterium.java
@@ -4,10 +4,9 @@
package org.torproject.collector.sync;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
/** Should a descriptor file be processed during sync. */
-public class ProcessCriterium implements Criterium<DescriptorFile> {
+public class ProcessCriterium implements Criterium<Descriptor> {
private final Class<? extends Descriptor> wantedType;
@@ -17,13 +16,8 @@ public class ProcessCriterium implements Criterium<DescriptorFile> {
/** Only process descriptors with the appropriate type. */
@Override
- public boolean applies(DescriptorFile file) {
- for (Descriptor desc : file.getDescriptors()) {
- if (!this.wantedType.isInstance(desc)) {
- return false;
- }
- }
- return true;
+ public boolean applies(Descriptor desc) {
+ return this.wantedType.isInstance(desc);
}
}
diff --git a/src/main/java/org/torproject/collector/sync/SyncManager.java b/src/main/java/org/torproject/collector/sync/SyncManager.java
index d8d2708..0c85a47 100644
--- a/src/main/java/org/torproject/collector/sync/SyncManager.java
+++ b/src/main/java/org/torproject/collector/sync/SyncManager.java
@@ -9,9 +9,9 @@ import org.torproject.collector.conf.Key;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorCollector;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.UnparseableDescriptor;
import org.torproject.descriptor.index.DescriptorIndexCollector;
import org.slf4j.Logger;
@@ -70,15 +70,17 @@ public class SyncManager {
String marker, Configuration conf) throws ConfigurationException {
Path basePath = conf.getPath(Key.SyncPath);
SyncPersistence persist = new SyncPersistence(conf);
+ Criterium<Descriptor> unparseable
+ = new ProcessCriterium(UnparseableDescriptor.class);
for (URL source : sources) {
File base = new File(basePath.toFile(), marker + "-" + source.getHost());
log.info("Merging {} from {} into storage ...", marker,
source.getHost());
for (Map.Entry<String, Class<? extends Descriptor>> entry
: mapPathDesc.entrySet()) {
+ File descFile = new File(base, entry.getKey());
DescriptorReader descriptorReader
= DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.addDirectory(new File(base, entry.getKey()));
String histFileEnding = entry.getValue().getSimpleName()
+ (entry.getKey().contains("consensus-microdesc")
? "-micro" : "");
@@ -87,28 +89,27 @@ public class SyncManager {
+ histFileEnding);
descriptorReader.setHistoryFile(historyFile);
log.info("Reading {} of type {} ... ", marker, histFileEnding);
- Iterator<DescriptorFile> descriptorFiles
- = descriptorReader.readDescriptors();
+ Iterator<Descriptor> descriptors
+ = descriptorReader.readDescriptors(descFile).iterator();
log.info("Done reading {} of type {}.", marker, histFileEnding);
- Criterium crit = new ProcessCriterium(entry.getValue());
- while (descriptorFiles.hasNext()) {
- DescriptorFile descFile = descriptorFiles.next();
- log.debug("Operating on desc-file containing {} descs.",
- descFile.getDescriptors().size());
- if (!crit.applies(descFile)) {
- log.warn("Not processing {} in {}.", descFile.getFileName(),
- descFile.getDirectory());
+ Criterium<Descriptor> crit = new ProcessCriterium(entry.getValue());
+ while (descriptors.hasNext()) {
+ Descriptor desc = descriptors.next();
+ if (unparseable.applies(desc)) {
+ Exception ex
+ = ((UnparseableDescriptor)desc).getDescriptorParseException();
+ log.warn("Parsing of {} caused Exception(s). Processing anyway.",
+ desc.getDescriptorFile(), ex);
+ }
+ if (!crit.applies(desc)) {
+ log.warn("Not processing {} in {}.", desc.getClass().getName(),
+ desc.getDescriptorFile());
continue;
}
- Exception ex = descFile.getException();
- if (null != ex) {
- log.warn("Parsing of {} caused Exception(s). Processing anyway.",
- descFile.getDirectory() + "/" + descFile.getFileName(), ex);
- }
- persist.storeDescs(descFile.getDescriptors(),
- descFile.getFile().getName(), collectionDate.getTime());
+ persist.storeDesc(desc, collectionDate.getTime());
}
+ persist.cleanDirectory();
descriptorReader.saveHistoryFile(historyFile);
}
log.info("Done merging {} from {}.", marker, source.getHost());
diff --git a/src/main/java/org/torproject/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/collector/sync/SyncPersistence.java
index 5f18f63..5fd6dc6 100644
--- a/src/main/java/org/torproject/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/collector/sync/SyncPersistence.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Path;
-import java.util.List;
/** Provides persistence for descriptors based on the descriptor type. */
public class SyncPersistence {
@@ -55,74 +54,93 @@ public class SyncPersistence {
}
/**
- * Stores the list of descriptors in main storage and recent.
+ * Cleans the directory in <code>RecentPath</code> after storing descriptors.
+ */
+ public void cleanDirectory() {
+ try {
+ PersistenceUtils.cleanDirectory(recentPath);
+ } catch (IOException ioe) {
+ log.error("Cleaning of {} failed.", recentPath.toString(), ioe);
+ }
+ }
+
+ /**
+ * Stores descriptors in main storage and recent.
* The storage locations are taken from <code>collector.properties</code>'
* options <code>OutputPath</code> and <code>RecentPath</code>.
*/
- public void storeDescs(List<Descriptor> descs, String filename,
- long received) {
+ public void storeDescs(Iterable<Descriptor> descs, long received) {
for (Descriptor desc : descs) {
- boolean recognizedAndWritten = false;
- for (Class clazz : desc.getClass().getInterfaces()) {
- DescriptorPersistence descPersist = null;
- switch (clazz.getSimpleName()) {
- case "RelayNetworkStatusVote":
- descPersist
- = new VotePersistence((RelayNetworkStatusVote) desc, received);
- break;
- case "RelayNetworkStatusConsensus":
- RelayNetworkStatusConsensus cons =
- (RelayNetworkStatusConsensus) desc;
- if (null == cons.getConsensusFlavor()) {
- descPersist = new ConsensusPersistence(cons, received);
- } else if ("microdesc".equals(cons.getConsensusFlavor())) {
- descPersist = new MicroConsensusPersistence(cons, received);
- }
- break;
- case "RelayServerDescriptor":
- descPersist = new ServerDescriptorPersistence(
- (RelayServerDescriptor) desc, received);
- break;
- case "BridgeExtraInfoDescriptor":
- descPersist = new BridgeExtraInfoPersistence(
- (BridgeExtraInfoDescriptor) desc, received);
- break;
- case "RelayExtraInfoDescriptor":
- descPersist = new ExtraInfoPersistence(
- (RelayExtraInfoDescriptor) desc, received);
- break;
- case "BridgeNetworkStatus": // need to infer authId from filename
- descPersist = new StatusPersistence(
- (BridgeNetworkStatus) desc, filename.split(DASH)[2], received);
- break;
- case "BridgeServerDescriptor":
- descPersist = new BridgeServerDescriptorPersistence(
- (BridgeServerDescriptor) desc, received);
- break;
- case "ExitList": // downloaded is part of desc, which to use?
- descPersist = new ExitlistPersistence((ExitList) desc, received);
+ storeDesc(desc, received);
+ }
+ }
+
+ /**
+ * Stores a descriptor in main storage and recent.
+ * The storage locations are taken from <code>collector.properties</code>'
+ * options <code>OutputPath</code> and <code>RecentPath</code>.
+ */
+ public void storeDesc(Descriptor desc, long received) {
+ String filename = desc.getDescriptorFile().getName();
+ boolean recognizedAndWritten = false;
+ for (Class clazz : desc.getClass().getInterfaces()) {
+ DescriptorPersistence descPersist = null;
+ switch (clazz.getSimpleName()) {
+ case "RelayNetworkStatusVote":
+ descPersist
+ = new VotePersistence((RelayNetworkStatusVote) desc, received);
+ break;
+ case "RelayNetworkStatusConsensus":
+ RelayNetworkStatusConsensus cons =
+ (RelayNetworkStatusConsensus) desc;
+ if (null == cons.getConsensusFlavor()) {
+ descPersist = new ConsensusPersistence(cons, received);
+ } else if ("microdesc".equals(cons.getConsensusFlavor())) {
+ descPersist = new MicroConsensusPersistence(cons, received);
+ }
+ break;
+ case "RelayServerDescriptor":
+ descPersist = new ServerDescriptorPersistence(
+ (RelayServerDescriptor) desc, received);
+ break;
+ case "BridgeExtraInfoDescriptor":
+ descPersist = new BridgeExtraInfoPersistence(
+ (BridgeExtraInfoDescriptor) desc, received);
+ break;
+ case "RelayExtraInfoDescriptor":
+ descPersist = new ExtraInfoPersistence(
+ (RelayExtraInfoDescriptor) desc, received);
+ break;
+ case "BridgeNetworkStatus": // need to infer authId from filename
+ String[] filenameParts = filename.split(DASH);
+ if (filenameParts.length < 3) {
+ log.error("Invalid BridgeNetworkStatus; skipping: {}.", filename);
break;
- default:
- log.trace("Invalid descriptor type {} for sync-merge.",
- clazz.getName());
- continue;
- }
- if (null != descPersist) {
- descPersist.storeAll(recentPathName, outputPathName);
- recognizedAndWritten = true;
- }
- break;
+ }
+ descPersist = new StatusPersistence(
+ (BridgeNetworkStatus) desc, filenameParts[2], received);
+ break;
+ case "BridgeServerDescriptor":
+ descPersist = new BridgeServerDescriptorPersistence(
+ (BridgeServerDescriptor) desc, received);
+ break;
+ case "ExitList": // downloaded is part of desc, which to use?
+ descPersist = new ExitlistPersistence((ExitList) desc, received);
+ break;
+ default:
+ log.trace("Invalid descriptor type {} for sync-merge.",
+ clazz.getName());
+ continue;
}
- if (!recognizedAndWritten) {
- log.error("Unknown descriptor type {} implementing {}.",
- desc.getClass().getSimpleName(), desc.getClass().getInterfaces());
+ if (null != descPersist) {
+ descPersist.storeAll(recentPathName, outputPathName);
+ recognizedAndWritten = true;
}
+ break;
}
- try {
- PersistenceUtils.cleanDirectory(recentPath);
- } catch (IOException ioe) {
- log.error("Cleaning of {} failed.", recentPath.toString(), ioe);
+ if (!recognizedAndWritten) {
+ log.error("Unknown descriptor type {} implementing {}.",
+ desc.getClass().getSimpleName(), desc.getClass().getInterfaces());
}
}
-
}
diff --git a/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java b/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java
index f5aee87..fe47f26 100644
--- a/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java
+++ b/src/test/java/org/torproject/collector/sync/SyncPersistenceTest.java
@@ -193,6 +193,7 @@ public class SyncPersistenceTest {
private int outCount;
private int recentCount;
private String filename;
+ private File fileForName;
private File recent;
private File output;
private String recentName;
@@ -205,6 +206,7 @@ public class SyncPersistenceTest {
this.expectedRecentDir = subRecent;
this.expectedOutputDir = subOutput;
this.filename = fn;
+ this.fileForName = new File(fn);
this.outCount = outCount;
this.recentCount = recentCount;
}
@@ -220,8 +222,9 @@ public class SyncPersistenceTest {
makeTemporaryFolders();
DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser();
SyncPersistence persist = new SyncPersistence(conf);
- persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename),
- filename, 1475694377000L);
+ persist.storeDescs(dp.parseDescriptors(bytesFromResource(),
+ fileForName, filename), 1475694377000L);
+ persist.cleanDirectory();
List<File> recentList = new ArrayList<>();
Files.walkFileTree(recent.toPath(), new FileCollector(recentList));
String dataUsed = "data used: " + expectedRecentDir + ", " + filename
@@ -237,8 +240,9 @@ public class SyncPersistenceTest {
makeTemporaryFolders();
DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser();
SyncPersistence persist = new SyncPersistence(conf);
- persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename),
- filename, 1475694377000L);
+ persist.storeDescs(dp.parseDescriptors(bytesFromResource(),
+ fileForName, filename), 1475694377000L);
+ persist.cleanDirectory();
List<File> recentList = new ArrayList<>();
List<File> outputList = new ArrayList<>();
Files.walkFileTree(output.toPath(), new FileCollector(outputList));
@@ -260,8 +264,9 @@ public class SyncPersistenceTest {
makeTemporaryFolders();
DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser();
SyncPersistence persist = new SyncPersistence(conf);
- persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename),
- filename, 1475694377000L);
+ persist.storeDescs(dp.parseDescriptors(bytesFromResource(),
+ fileForName, filename), 1475694377000L);
+ persist.cleanDirectory();
List<File> recentList = new ArrayList<>();
Files.walkFileTree(recent.toPath(), new FileCollector(recentList));
assertEquals(recentCount, recentList.size());
@@ -284,8 +289,9 @@ public class SyncPersistenceTest {
makeTemporaryFolders();
DescriptorParser dp = DescriptorSourceFactory.createDescriptorParser();
SyncPersistence persist = new SyncPersistence(conf);
- persist.storeDescs(dp.parseDescriptors(bytesFromResource(), filename),
- filename, 1475694377000L);
+ persist.storeDescs(dp.parseDescriptors(bytesFromResource(),
+ fileForName, filename), 1475694377000L);
+ persist.cleanDirectory();
List<File> outputList = new ArrayList<>();
Files.walkFileTree(output.toPath(), new FileCollector(outputList));
assertEquals(outCount, outputList.size());
More information about the tor-commits
mailing list