commit 06c0d78e4a73042c0e7fb6052f079237b8d7ff01 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Dec 14 23:23:54 2020 +0100 Include OnionPerf analysis files when syncing. --- CHANGELOG.md | 4 +- .../collector/onionperf/OnionPerfDownloader.java | 1 + .../collector/persist/OnionPerfPersistence.java | 60 +++++++++++++++++++--- src/main/resources/collector.properties | 2 +- 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f56f74d..7956e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ - Clean up descriptors written to the `out/` directory by deleting files that are older than seven weeks. - Correctly index files that are moved away and back. - - Include microdescriptors and certs when syncing from another - CollecTor instance. + - Include microdescriptors, certs, and OnionPerf analysis files when + syncing from another CollecTor instance. - Update to metrics-lib 2.15.0. diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index f90bdfe..352d24a 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -57,6 +57,7 @@ public class OnionPerfDownloader extends CollecTorMain { public OnionPerfDownloader(Configuration config) { super(config); this.mapPathDescriptors.put("recent/torperf", TorperfResult.class); + this.mapPathDescriptors.put("recent/onionperf", TorperfResult.class); } /** File containing the download history, which is necessary, because diff --git a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java index 7ed16a2..8975d80 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java @@ -6,12 +6,21 @@ package org.torproject.metrics.collector.persist; import org.torproject.descriptor.TorperfResult; import org.torproject.metrics.collector.conf.Annotation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; public class OnionPerfPersistence extends DescriptorPersistence<TorperfResult> { + private static final Logger logger + = LoggerFactory.getLogger(OnionPerfPersistence.class); + private static final String ONIONPERF = "torperf"; public OnionPerfPersistence(TorperfResult desc) { @@ -32,18 +41,55 @@ public class OnionPerfPersistence name).toString(); } - /** OnionPerf default storage appends. */ + /** If the original descriptor file was a .tpf file, append the parsed Torperf + * result to the destination .tpf file, but if it was a .json.xz file, just + * copy over the entire file, unless it already exists. */ @Override - public boolean storeOut(String outRoot) { - return super.storeOut(outRoot, StandardOpenOption.APPEND); + public boolean storeOut(String outRoot, StandardOpenOption option) { + if (desc.getDescriptorFile().getName().endsWith(".tpf")) { + return super.storeOut(outRoot, StandardOpenOption.APPEND); + } else { + String fileName = desc.getDescriptorFile().getName(); + String[] dateParts = fileName.split("\\.")[0].split("-"); + return this.copyIfNotExists( + Paths.get(outRoot, + "onionperf", + dateParts[0], // year + dateParts[1], // month + dateParts[2], // day + fileName)); + } } - /** OnionPerf default storage appends. */ + /** If the original descriptor file was a .tpf file, append the parsed Torperf + * result to the destination .tpf file, but if it was a .json.xz file, just + * copy over the entire file, unless it already exists. */ @Override - public boolean storeAll(String recentRoot, String outRoot) { - return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND, - StandardOpenOption.APPEND); + public boolean storeRecent(String recentRoot, StandardOpenOption option) { + if (desc.getDescriptorFile().getName().endsWith(".tpf")) { + return super.storeRecent(recentRoot, StandardOpenOption.APPEND); + } else { + String fileName = desc.getDescriptorFile().getName(); + return this.copyIfNotExists( + Paths.get(recentRoot, + "onionperf", + fileName)); + } } + private boolean copyIfNotExists(Path destinationFile) { + if (Files.exists(destinationFile)) { + return false; + } + Path originalFile = this.desc.getDescriptorFile().toPath(); + try { + Files.createDirectories(destinationFile.getParent()); + Files.copy(originalFile, destinationFile); + } catch (IOException e) { + logger.warn("Unable to copy file.", e); + return false; + } + return true; + } } diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 2347021..99a7cfa 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -175,7 +175,7 @@ ExitlistUrl = https://check.torproject.org/exit-addresses ######## OnionPerf downloader ######## # ## Define descriptor sources -# possible values: Remote,Sync (.tpf files only!) +# possible values: Remote, Sync OnionPerfSources = Remote # Retrieve files from the following CollecTor instances. # List of URLs separated by comma.