[tor-commits] [collector/master] Add source name to locally imported bandwidth files.
karsten at torproject.org
karsten at torproject.org
Tue Jul 30 09:31:53 UTC 2019
commit abde3f7591117060ccf1afc9b26531d79619301f
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed Jul 10 10:34:09 2019 +0200
Add source name to locally imported bandwidth files.
Implements #30219.
---
CHANGELOG.md | 7 +++++++
.../metrics/collector/relaydescs/ArchiveReader.java | 2 +-
.../metrics/collector/relaydescs/ArchiveWriter.java | 12 +++++++++---
.../collector/relaydescs/CachedRelayDescriptorReader.java | 6 +++---
.../collector/relaydescs/RelayDescriptorDownloader.java | 4 ++--
.../metrics/collector/relaydescs/RelayDescriptorParser.java | 7 +++++--
src/main/resources/collector.properties | 5 ++++-
7 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4682fef..613ef71 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.10.0 - 2019-??-??
+
+ * Medium changes
+ - Changed local import of bandwidth files to include the parent
+ directory name as @source annotation and to the filename.
+
+
# Changes in version 1.9.1 - 2019-05-29
* Medium changes
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
index 27010a9..fee46c9 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
@@ -141,7 +141,7 @@ public class ArchiveReader {
}
bis.close();
byte[] allData = baos.toByteArray();
- boolean stored = this.rdp.parse(allData);
+ boolean stored = this.rdp.parse(allData, pop);
if (!stored) {
filesToRetry.add(pop);
continue;
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index e1279ee..911ac2a 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -745,18 +745,24 @@ public class ArchiveWriter extends CollecTorMain {
/** Stores a bandwidth file to disk. */
void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp,
- String bandwidthFileDigest) {
+ String sourceName, String bandwidthFileDigest) {
DateTimeFormatter printFormat = DateTimeFormatter
.ofPattern("uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);
File tarballFile = Paths.get(this.outputDirectory, "bandwidth",
fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-"
+ + (null == sourceName ? "" : (sourceName + "-"))
+ bandwidthFileDigest).toFile();
+ StringBuilder sb = new StringBuilder();
+ sb.append(Annotation.BandwidthFile.toString());
+ if (null != sourceName) {
+ sb.append("@source ").append(sourceName).append('\n');
+ }
boolean tarballFileExistedBefore = tarballFile.exists();
File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths",
tarballFile.getName()).toFile();
File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null)) {
- this.storedVotesCounter++;
+ if (this.store(sb.toString().getBytes(), data, outputFiles, null)) {
+ this.storedBandwidthsCounter++;
}
if (!tarballFileExistedBefore
&& this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L))) {
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
index 3e1bba4..0308f05 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -158,7 +158,7 @@ public class CachedRelayDescriptorReader {
allData));
if (!this.lastImportHistory.contains(digest)
&& !this.currentImportHistory.contains(digest)) {
- this.rdp.parse(allData);
+ this.rdp.parse(allData, null);
} else {
this.dumpStats.append(" (skipped)");
}
@@ -183,7 +183,7 @@ public class CachedRelayDescriptorReader {
rawNetworkStatusBytes));
if (!this.lastImportHistory.contains(digest)
&& !this.currentImportHistory.contains(digest)) {
- this.rdp.parse(rawNetworkStatusBytes);
+ this.rdp.parse(rawNetworkStatusBytes, null);
parsedNum++;
} else {
skippedNum++;
@@ -229,7 +229,7 @@ public class CachedRelayDescriptorReader {
descBytes));
if (!this.lastImportHistory.contains(digest)
&& !this.currentImportHistory.contains(digest)) {
- this.rdp.parse(descBytes);
+ this.rdp.parse(descBytes, null);
parsedNum++;
} else {
skippedNum++;
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
index 5a241f4..f1179d8 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
@@ -899,7 +899,7 @@ public class RelayDescriptorDownloader {
int receivedDescriptors = 0;
if (allData != null) {
if (resource.startsWith("/tor/status-vote/")) {
- this.rdp.parse(allData);
+ this.rdp.parse(allData, null);
receivedDescriptors = 1;
} else if (resource.startsWith("/tor/server/")
|| resource.startsWith("/tor/extra/")) {
@@ -933,7 +933,7 @@ public class RelayDescriptorDownloader {
end += endToken.length();
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
- this.rdp.parse(descBytes);
+ this.rdp.parse(descBytes, null);
receivedDescriptors++;
}
} else if (resource.startsWith("/tor/micro/")) {
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
index 113ac77..a610ed9 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
@@ -10,6 +10,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.text.ParseException;
@@ -75,7 +76,7 @@ public class RelayDescriptorParser {
* forwards them to the archive writer to store them to disk, and tells
* the relay descriptor downloader and archive reader about the
* contained descriptor and all referenced descriptors. */
- public boolean parse(byte[] data) {
+ public boolean parse(byte[] data, File containingFile) {
boolean stored = false;
try {
/* Convert descriptor to ASCII for parsing. This means we'll lose
@@ -357,7 +358,9 @@ public class RelayDescriptorParser {
fileCreatedOrTimestamp = LocalDateTime.of(2000, 1, 1, 0, 0, 0);
}
}
- this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp,
+ String sourceName = containingFile == null ? null
+ : containingFile.getParentFile().getName();
+ this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp, sourceName,
DigestUtils.sha256Hex(data).toUpperCase());
stored = true;
}
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index 787f188..292e876 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -86,7 +86,10 @@ RelaySyncOrigins = https://collector.torproject.org
## Path to Tor data directory to read cached-* files from
## the listed path(s). If there is more that one separated by comma.
RelayCacheOrigins = in/relay-descriptors/cacheddesc/
-## Relative path to directory to import directory archives from
+## Relative path to directory to import directory archives from.
+## Note that when importing bandwidth files, the parent directory name is
+## included in the @source annotation and in the file name. Recommended
+## source names are nicknames of directory authorities using these files.
RelayLocalOrigins = in/relay-descriptors/archives/
#
## Keep a history of imported directory archive files to know which files
More information about the tor-commits
mailing list