[tor-commits] [collector/master] Fix synchronizing bandwidth files from other CollecTors.

[karsten at torproject.org]

commit c19be5b35490a1da33ecf57201e37409599fea48
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Thu May 16 15:55:57 2019 +0200

    Fix synchronizing bandwidth files from other CollecTors.
    
    Fixes #30525.
---
 build.xml                                          |  2 +-
 .../persist/BandwidthFilePersistence.java          | 65 ++++++++++++++++++++++
 .../metrics/collector/sync/SyncPersistence.java    |  5 ++
 3 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/build.xml b/build.xml
index 5be33ff..05cba36 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
   <property name="release.version" value="1.9.0-dev" />
   <property name="project-main-class" value="org.torproject.metrics.collector.Main" />
   <property name="name" value="collector"/>
-  <property name="metricslibversion" value="2.6.1" />
+  <property name="metricslibversion" value="2.6.2" />
   <property name="jarincludes" value="collector.properties logback.xml" />
 
   <patternset id="runtime" >
diff --git a/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java
new file mode 100644
index 0000000..aeaf0ee
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java
@@ -0,0 +1,65 @@
+/* Copyright 2016--2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.persist;
+
+import org.torproject.descriptor.BandwidthFile;
+import org.torproject.metrics.collector.conf.Annotation;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+
+public class BandwidthFilePersistence
+    extends DescriptorPersistence<BandwidthFile> {
+
+  private static final String BANDWIDTH = "bandwidth";
+  private static final String BANDWIDTHS = "bandwidths";
+
+  public BandwidthFilePersistence(BandwidthFile desc) {
+    super(desc, Annotation.BandwidthFile.bytes());
+    calculatePaths();
+  }
+
+  private void calculatePaths() {
+    LocalDateTime fileCreatedOrTimestamp = this.desc.fileCreated()
+        .orElse(this.desc.timestamp());
+    DateTimeFormatter directoriesFormatter = DateTimeFormatter
+        .ofPattern("uuuu/MM/dd").withZone(ZoneOffset.UTC);
+    String[] directories = fileCreatedOrTimestamp.format(directoriesFormatter)
+        .split("/");
+    DateTimeFormatter fileFormatter = DateTimeFormatter
+        .ofPattern("uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);
+    String bandwidthFileDigest = calcDigestFromBytes(
+        this.desc.getRawDescriptorBytes());
+    String fileOut = fileCreatedOrTimestamp.format(fileFormatter)
+        + "-bandwidth-" + bandwidthFileDigest;
+    this.recentPath = Paths.get(RELAYDESCS, BANDWIDTHS, fileOut).toString();
+    this.storagePath = Paths.get(RELAYDESCS, BANDWIDTH, directories[0],
+        directories[1], directories[2], fileOut).toString();
+  }
+
+  /** Calculate a digest for bandwidth files. */
+  private static String calcDigestFromBytes(byte[] bytes) {
+    String digest = "";
+    int start = 0;
+    while (start < bytes.length && bytes[start] == (byte) '@') {
+      do {
+        start++;
+      } while (start < bytes.length && bytes[start] != (byte) '\n');
+      start++;
+    }
+    if (start < bytes.length) {
+      byte[] forDigest = new byte[bytes.length - start];
+      System.arraycopy(bytes, start, forDigest, 0, forDigest.length);
+      digest = DigestUtils.sha256Hex(forDigest).toUpperCase();
+    } else {
+      log.error("No digest calculation possible.  Returning empty string.");
+    }
+    return digest;
+  }
+}
+
diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
index f2d3e55..0d344bf 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
@@ -3,6 +3,7 @@
 
 package org.torproject.metrics.collector.sync;
 
+import org.torproject.descriptor.BandwidthFile;
 import org.torproject.descriptor.BridgeExtraInfoDescriptor;
 import org.torproject.descriptor.BridgeNetworkStatus;
 import org.torproject.descriptor.BridgeServerDescriptor;
@@ -17,6 +18,7 @@ import org.torproject.descriptor.WebServerAccessLog;
 import org.torproject.metrics.collector.conf.Configuration;
 import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
+import org.torproject.metrics.collector.persist.BandwidthFilePersistence;
 import org.torproject.metrics.collector.persist.BridgeExtraInfoPersistence;
 import org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistence;
 import org.torproject.metrics.collector.persist.ConsensusPersistence;
@@ -138,6 +140,9 @@ public class SyncPersistence {
           descPersist = new WebServerAccessLogPersistence(
               (WebServerAccessLog) desc);
           break;
+        case "BandwidthFile":
+          descPersist = new BandwidthFilePersistence((BandwidthFile) desc);
+          break;
         default:
           log.trace("Invalid descriptor type {} for sync-merge.",
               clazz.getName());