commit c19be5b35490a1da33ecf57201e37409599fea48 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu May 16 15:55:57 2019 +0200 Fix synchronizing bandwidth files from other CollecTors. Fixes #30525. --- build.xml | 2 +- .../persist/BandwidthFilePersistence.java | 65 ++++++++++++++++++++++ .../metrics/collector/sync/SyncPersistence.java | 5 ++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 5be33ff..05cba36 100644 --- a/build.xml +++ b/build.xml @@ -11,7 +11,7 @@ <property name="release.version" value="1.9.0-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.6.1" /> + <property name="metricslibversion" value="2.6.2" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java new file mode 100644 index 0000000..aeaf0ee --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java @@ -0,0 +1,65 @@ +/* Copyright 2016--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.persist; + +import org.torproject.descriptor.BandwidthFile; +import org.torproject.metrics.collector.conf.Annotation; + +import org.apache.commons.codec.digest.DigestUtils; + +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +public class BandwidthFilePersistence + extends DescriptorPersistence<BandwidthFile> { + + private static final String BANDWIDTH = "bandwidth"; + private static final String BANDWIDTHS = "bandwidths"; + + public BandwidthFilePersistence(BandwidthFile desc) { + super(desc, Annotation.BandwidthFile.bytes()); + calculatePaths(); + } + + private void calculatePaths() { + LocalDateTime fileCreatedOrTimestamp = this.desc.fileCreated() + .orElse(this.desc.timestamp()); + DateTimeFormatter directoriesFormatter = DateTimeFormatter + .ofPattern("uuuu/MM/dd").withZone(ZoneOffset.UTC); + String[] directories = fileCreatedOrTimestamp.format(directoriesFormatter) + .split("/"); + DateTimeFormatter fileFormatter = DateTimeFormatter + .ofPattern("uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC); + String bandwidthFileDigest = calcDigestFromBytes( + this.desc.getRawDescriptorBytes()); + String fileOut = fileCreatedOrTimestamp.format(fileFormatter) + + "-bandwidth-" + bandwidthFileDigest; + this.recentPath = Paths.get(RELAYDESCS, BANDWIDTHS, fileOut).toString(); + this.storagePath = Paths.get(RELAYDESCS, BANDWIDTH, directories[0], + directories[1], directories[2], fileOut).toString(); + } + + /** Calculate a digest for bandwidth files. */ + private static String calcDigestFromBytes(byte[] bytes) { + String digest = ""; + int start = 0; + while (start < bytes.length && bytes[start] == (byte) '@') { + do { + start++; + } while (start < bytes.length && bytes[start] != (byte) '\n'); + start++; + } + if (start < bytes.length) { + byte[] forDigest = new byte[bytes.length - start]; + System.arraycopy(bytes, start, forDigest, 0, forDigest.length); + digest = DigestUtils.sha256Hex(forDigest).toUpperCase(); + } else { + log.error("No digest calculation possible. Returning empty string."); + } + return digest; + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java index f2d3e55..0d344bf 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java @@ -3,6 +3,7 @@ package org.torproject.metrics.collector.sync; +import org.torproject.descriptor.BandwidthFile; import org.torproject.descriptor.BridgeExtraInfoDescriptor; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.BridgeServerDescriptor; @@ -17,6 +18,7 @@ import org.torproject.descriptor.WebServerAccessLog; import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; +import org.torproject.metrics.collector.persist.BandwidthFilePersistence; import org.torproject.metrics.collector.persist.BridgeExtraInfoPersistence; import org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistence; import org.torproject.metrics.collector.persist.ConsensusPersistence; @@ -138,6 +140,9 @@ public class SyncPersistence { descPersist = new WebServerAccessLogPersistence( (WebServerAccessLog) desc); break; + case "BandwidthFile": + descPersist = new BandwidthFilePersistence((BandwidthFile) desc); + break; default: log.trace("Invalid descriptor type {} for sync-merge.", clazz.getName());