commit abde3f7591117060ccf1afc9b26531d79619301f Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Jul 10 10:34:09 2019 +0200 Add source name to locally imported bandwidth files. Implements #30219. --- CHANGELOG.md | 7 +++++++ .../metrics/collector/relaydescs/ArchiveReader.java | 2 +- .../metrics/collector/relaydescs/ArchiveWriter.java | 12 +++++++++--- .../collector/relaydescs/CachedRelayDescriptorReader.java | 6 +++--- .../collector/relaydescs/RelayDescriptorDownloader.java | 4 ++-- .../metrics/collector/relaydescs/RelayDescriptorParser.java | 7 +++++-- src/main/resources/collector.properties | 5 ++++- 7 files changed, 31 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4682fef..613ef71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 1.10.0 - 2019-??-?? + + * Medium changes + - Changed local import of bandwidth files to include the parent + directory name as @source annotation and to the filename. + + # Changes in version 1.9.1 - 2019-05-29 * Medium changes diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 27010a9..fee46c9 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -141,7 +141,7 @@ public class ArchiveReader { } bis.close(); byte[] allData = baos.toByteArray(); - boolean stored = this.rdp.parse(allData); + boolean stored = this.rdp.parse(allData, pop); if (!stored) { filesToRetry.add(pop); continue; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index e1279ee..911ac2a 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -745,18 +745,24 @@ public class ArchiveWriter extends CollecTorMain { /** Stores a bandwidth file to disk. */ void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp, - String bandwidthFileDigest) { + String sourceName, String bandwidthFileDigest) { DateTimeFormatter printFormat = DateTimeFormatter .ofPattern("uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC); File tarballFile = Paths.get(this.outputDirectory, "bandwidth", fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-" + + (null == sourceName ? "" : (sourceName + "-")) + bandwidthFileDigest).toFile(); + StringBuilder sb = new StringBuilder(); + sb.append(Annotation.BandwidthFile.toString()); + if (null != sourceName) { + sb.append("@source ").append(sourceName).append('\n'); + } boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths", tarballFile.getName()).toFile(); File[] outputFiles = new File[] { tarballFile, rsyncFile }; - if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null)) { - this.storedVotesCounter++; + if (this.store(sb.toString().getBytes(), data, outputFiles, null)) { + this.storedBandwidthsCounter++; } if (!tarballFileExistedBefore && this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L))) { diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java index 3e1bba4..0308f05 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java @@ -158,7 +158,7 @@ public class CachedRelayDescriptorReader { allData)); if (!this.lastImportHistory.contains(digest) && !this.currentImportHistory.contains(digest)) { - this.rdp.parse(allData); + this.rdp.parse(allData, null); } else { this.dumpStats.append(" (skipped)"); } @@ -183,7 +183,7 @@ public class CachedRelayDescriptorReader { rawNetworkStatusBytes)); if (!this.lastImportHistory.contains(digest) && !this.currentImportHistory.contains(digest)) { - this.rdp.parse(rawNetworkStatusBytes); + this.rdp.parse(rawNetworkStatusBytes, null); parsedNum++; } else { skippedNum++; @@ -229,7 +229,7 @@ public class CachedRelayDescriptorReader { descBytes)); if (!this.lastImportHistory.contains(digest) && !this.currentImportHistory.contains(digest)) { - this.rdp.parse(descBytes); + this.rdp.parse(descBytes, null); parsedNum++; } else { skippedNum++; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index 5a241f4..f1179d8 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -899,7 +899,7 @@ public class RelayDescriptorDownloader { int receivedDescriptors = 0; if (allData != null) { if (resource.startsWith("/tor/status-vote/")) { - this.rdp.parse(allData); + this.rdp.parse(allData, null); receivedDescriptors = 1; } else if (resource.startsWith("/tor/server/") || resource.startsWith("/tor/extra/")) { @@ -933,7 +933,7 @@ public class RelayDescriptorDownloader { end += endToken.length(); byte[] descBytes = new byte[end - start]; System.arraycopy(allData, start, descBytes, 0, end - start); - this.rdp.parse(descBytes); + this.rdp.parse(descBytes, null); receivedDescriptors++; } } else if (resource.startsWith("/tor/micro/")) { diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java index 113ac77..a610ed9 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java @@ -10,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.StringReader; import java.text.ParseException; @@ -75,7 +76,7 @@ public class RelayDescriptorParser { * forwards them to the archive writer to store them to disk, and tells * the relay descriptor downloader and archive reader about the * contained descriptor and all referenced descriptors. */ - public boolean parse(byte[] data) { + public boolean parse(byte[] data, File containingFile) { boolean stored = false; try { /* Convert descriptor to ASCII for parsing. This means we'll lose @@ -357,7 +358,9 @@ public class RelayDescriptorParser { fileCreatedOrTimestamp = LocalDateTime.of(2000, 1, 1, 0, 0, 0); } } - this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp, + String sourceName = containingFile == null ? null + : containingFile.getParentFile().getName(); + this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp, sourceName, DigestUtils.sha256Hex(data).toUpperCase()); stored = true; } diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 787f188..292e876 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -86,7 +86,10 @@ RelaySyncOrigins = https://collector.torproject.org ## Path to Tor data directory to read cached-* files from ## the listed path(s). If there is more that one separated by comma. RelayCacheOrigins = in/relay-descriptors/cacheddesc/ -## Relative path to directory to import directory archives from +## Relative path to directory to import directory archives from. +## Note that when importing bandwidth files, the parent directory name is +## included in the @source annotation and in the file name. Recommended +## source names are nicknames of directory authorities using these files. RelayLocalOrigins = in/relay-descriptors/archives/ # ## Keep a history of imported directory archive files to know which files