[tor-commits] [collector/master] Include OnionPerf analysis files when syncing.

[karsten at torproject.org]

commit 06c0d78e4a73042c0e7fb6052f079237b8d7ff01
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon Dec 14 23:23:54 2020 +0100

    Include OnionPerf analysis files when syncing.
---
 CHANGELOG.md                                       |  4 +-
 .../collector/onionperf/OnionPerfDownloader.java   |  1 +
 .../collector/persist/OnionPerfPersistence.java    | 60 +++++++++++++++++++---
 src/main/resources/collector.properties            |  2 +-
 4 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f56f74d..7956e55 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,8 +4,8 @@
   - Clean up descriptors written to the `out/` directory by deleting
     files that are older than seven weeks.
   - Correctly index files that are moved away and back.
-  - Include microdescriptors and certs when syncing from another
-    CollecTor instance.
+  - Include microdescriptors, certs, and OnionPerf analysis files when
+    syncing from another CollecTor instance.
   - Update to metrics-lib 2.15.0.
 
 
diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
index f90bdfe..352d24a 100644
--- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
@@ -57,6 +57,7 @@ public class OnionPerfDownloader extends CollecTorMain {
   public OnionPerfDownloader(Configuration config) {
     super(config);
     this.mapPathDescriptors.put("recent/torperf", TorperfResult.class);
+    this.mapPathDescriptors.put("recent/onionperf", TorperfResult.class);
   }
 
   /** File containing the download history, which is necessary, because
diff --git a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
index 7ed16a2..8975d80 100644
--- a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java
@@ -6,12 +6,21 @@ package org.torproject.metrics.collector.persist;
 import org.torproject.descriptor.TorperfResult;
 import org.torproject.metrics.collector.conf.Annotation;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 
 public class OnionPerfPersistence
     extends DescriptorPersistence<TorperfResult> {
 
+  private static final Logger logger
+      = LoggerFactory.getLogger(OnionPerfPersistence.class);
+
   private static final String ONIONPERF = "torperf";
 
   public OnionPerfPersistence(TorperfResult desc) {
@@ -32,18 +41,55 @@ public class OnionPerfPersistence
         name).toString();
   }
 
-  /** OnionPerf default storage appends. */
+  /** If the original descriptor file was a .tpf file, append the parsed Torperf
+   * result to the destination .tpf file, but if it was a .json.xz file, just
+   * copy over the entire file, unless it already exists. */
   @Override
-  public boolean storeOut(String outRoot) {
-    return super.storeOut(outRoot, StandardOpenOption.APPEND);
+  public boolean storeOut(String outRoot, StandardOpenOption option) {
+    if (desc.getDescriptorFile().getName().endsWith(".tpf")) {
+      return super.storeOut(outRoot, StandardOpenOption.APPEND);
+    } else {
+      String fileName = desc.getDescriptorFile().getName();
+      String[] dateParts = fileName.split("\\.")[0].split("-");
+      return this.copyIfNotExists(
+          Paths.get(outRoot,
+              "onionperf",
+              dateParts[0], // year
+              dateParts[1], // month
+              dateParts[2], // day
+              fileName));
+    }
   }
 
-  /** OnionPerf default storage appends. */
+  /** If the original descriptor file was a .tpf file, append the parsed Torperf
+   * result to the destination .tpf file, but if it was a .json.xz file, just
+   * copy over the entire file, unless it already exists. */
   @Override
-  public boolean storeAll(String recentRoot, String outRoot) {
-    return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND,
-        StandardOpenOption.APPEND);
+  public boolean storeRecent(String recentRoot, StandardOpenOption option) {
+    if (desc.getDescriptorFile().getName().endsWith(".tpf")) {
+      return super.storeRecent(recentRoot, StandardOpenOption.APPEND);
+    } else {
+      String fileName = desc.getDescriptorFile().getName();
+      return this.copyIfNotExists(
+          Paths.get(recentRoot,
+          "onionperf",
+          fileName));
+    }
   }
 
+  private boolean copyIfNotExists(Path destinationFile) {
+    if (Files.exists(destinationFile)) {
+      return false;
+    }
+    Path originalFile = this.desc.getDescriptorFile().toPath();
+    try {
+      Files.createDirectories(destinationFile.getParent());
+      Files.copy(originalFile, destinationFile);
+    } catch (IOException e) {
+      logger.warn("Unable to copy file.", e);
+      return false;
+    }
+    return true;
+  }
 }
 
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index 2347021..99a7cfa 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -175,7 +175,7 @@ ExitlistUrl = https://check.torproject.org/exit-addresses
 ######## OnionPerf downloader ########
 #
 ## Define descriptor sources
-#  possible values: Remote,Sync (.tpf files only!)
+#  possible values: Remote, Sync
 OnionPerfSources = Remote
 #  Retrieve files from the following CollecTor instances.
 #  List of URLs separated by comma.