[tor-commits] [collector/master] Enable OnionPerf 'Sync' runs.

[karsten at torproject.org]

commit 8ffdfd6c2cd4a1d6a6f0660681e9a788b88dc342
Author: iwakeh <iwakeh at torproject.org>
Date:   Wed Jul 12 18:47:45 2017 +0000

    Enable OnionPerf 'Sync' runs.
---
 .../org/torproject/collector/conf/Annotation.java  |  2 +-
 .../java/org/torproject/collector/conf/Key.java    |  1 +
 .../collector/onionperf/OnionPerfDownloader.java   |  1 +
 .../collector/persist/OnionPerfPersistence.java    | 49 ++++++++++++++++++++++
 .../torproject/collector/sync/SyncPersistence.java |  5 +++
 src/main/resources/collector.properties            |  5 ++-
 6 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/torproject/collector/conf/Annotation.java b/src/main/java/org/torproject/collector/conf/Annotation.java
index 0dab546..23f485c 100644
--- a/src/main/java/org/torproject/collector/conf/Annotation.java
+++ b/src/main/java/org/torproject/collector/conf/Annotation.java
@@ -16,7 +16,7 @@ public enum Annotation {
   Microdescriptor("@type microdescriptor 1.0\n"),
   Server("@type server-descriptor 1.0\n"),
   Status("@type bridge-network-status 1.2\n"),
-  OnionPerf("@type torperf 1.0\n"),
+  OnionPerf("@type torperf 1.1\n"),
   Vote("@type network-status-vote-3 1.0\n");
 
   private final String annotation;
diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java
index 4341432..72af4bb 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -34,6 +34,7 @@ public enum Key {
   BridgeSyncOrigins(URL[].class),
   BridgeLocalOrigins(Path.class),
   ExitlistSyncOrigins(URL[].class),
+  OnionPerfSyncOrigins(URL[].class),
   BridgedescsActivated(Boolean.class),
   BridgedescsOffsetMinutes(Integer.class),
   BridgedescsPeriodMinutes(Integer.class),
diff --git a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java
index b2a27d4..36b006a 100644
--- a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java
+++ b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java
@@ -51,6 +51,7 @@ public class OnionPerfDownloader extends CollecTorMain {
   /** Instantiate the OnionPerf module using the given configuration. */
   public OnionPerfDownloader(Configuration config) {
     super(config);
+    this.mapPathDescriptors.put("recent/torperf", TorperfResult.class);
   }
 
   /** File containing the download history, which is necessary, because
diff --git a/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java
new file mode 100644
index 0000000..22093eb
--- /dev/null
+++ b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java
@@ -0,0 +1,49 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.persist;
+
+import org.torproject.collector.conf.Annotation;
+import org.torproject.descriptor.TorperfResult;
+
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+
+public class OnionPerfPersistence
+    extends DescriptorPersistence<TorperfResult> {
+
+  private static final String ONIONPERF = "torperf";
+
+  public OnionPerfPersistence(TorperfResult desc) {
+    super(desc, Annotation.OnionPerf.bytes());
+    calculatePaths();
+  }
+
+  private void calculatePaths() {
+    String[] parts = PersistenceUtils.dateTimeParts(desc.getStartMillis());
+    String name =  desc.getSource() + DASH + desc.getFileSize() + DASH
+        + parts[0] + DASH + parts[1] + DASH + parts[2] + ".tpf";
+    this.recentPath = Paths.get(ONIONPERF, name).toString();
+    this.storagePath = Paths.get(
+        ONIONPERF,
+        parts[0], // year
+        parts[1], // month
+        parts[2], // day
+        name).toString();
+  }
+
+  /** OnionPerf default storage appends. */
+  @Override
+  public boolean storeOut(String outRoot) {
+    return super.storeOut(outRoot, StandardOpenOption.APPEND);
+  }
+
+  /** OnionPerf default storage appends. */
+  @Override
+  public boolean storeAll(String recentRoot, String outRoot) {
+    return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND,
+        StandardOpenOption.APPEND);
+  }
+
+}
+
diff --git a/src/main/java/org/torproject/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/collector/sync/SyncPersistence.java
index 5fd6dc6..baca83d 100644
--- a/src/main/java/org/torproject/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/collector/sync/SyncPersistence.java
@@ -13,6 +13,7 @@ import org.torproject.collector.persist.DescriptorPersistence;
 import org.torproject.collector.persist.ExitlistPersistence;
 import org.torproject.collector.persist.ExtraInfoPersistence;
 import org.torproject.collector.persist.MicroConsensusPersistence;
+import org.torproject.collector.persist.OnionPerfPersistence;
 import org.torproject.collector.persist.PersistenceUtils;
 import org.torproject.collector.persist.ServerDescriptorPersistence;
 import org.torproject.collector.persist.StatusPersistence;
@@ -26,6 +27,7 @@ import org.torproject.descriptor.RelayExtraInfoDescriptor;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.descriptor.RelayNetworkStatusVote;
 import org.torproject.descriptor.RelayServerDescriptor;
+import org.torproject.descriptor.TorperfResult;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -127,6 +129,9 @@ public class SyncPersistence {
         case "ExitList": // downloaded is part of desc, which to use?
           descPersist = new ExitlistPersistence((ExitList) desc, received);
           break;
+        case "TorperfResult":
+          descPersist = new OnionPerfPersistence((TorperfResult) desc);
+          break;
         default:
           log.trace("Invalid descriptor type {} for sync-merge.",
               clazz.getName());
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index d49bd58..0a9f932 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -142,8 +142,11 @@ ExitlistUrl = https://check.torproject.org/exit-addresses
 ######## OnionPerf downloader ########
 #
 ## Define descriptor sources
-#  possible values: Remote
+#  possible values: Remote,Sync
 OnionPerfSources = Remote
+#  Retrieve files from the following CollecTor instances.
+#  List of URLs separated by comma.
+OnionPerfSyncOrigins = https://collector.torproject.org
 #
 ## OnionPerf base URLs
 ## Hosts must be configured to use the first subdomain part of the given URL as