commit 4b7b13d37d2423f9b1dee865e203a524b8df488b Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Dec 5 17:03:02 2019 +0100
Only write changed status and document files.
Fixes #32660. --- CHANGELOG.md | 4 ++++ .../metrics/onionoo/docs/DocumentStore.java | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md index 6060551..cd27f30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes in version 7.0-1.2?.? - 2019-1?-??
+ * Medium changes + - Only write status files and document files if their content has + changed. +
# Changes in version 7.0-1.22.0 - 2019-11-28
diff --git a/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java index 81b428a..ea7786e 100644 --- a/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java @@ -8,6 +8,7 @@ import org.torproject.metrics.onionoo.util.FormattingUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory;
@@ -20,7 +21,9 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -59,6 +62,10 @@ public class DocumentStore {
private long storedBytes = 0L;
+ private long unchangedFiles = 0L; + + private long unchangedBytes = 0L; + private long retrievedFiles = 0L;
private long retrievedBytes = 0L; @@ -351,6 +358,17 @@ public class DocumentStore { documentString.length()); } documentFile.getParentFile().mkdirs(); + if (documentFile.exists()) { + try (InputStream stream = Files.newInputStream(documentFile.toPath())) { + String existingFileDigest = DigestUtils.sha256Hex(stream); + String newFileDigest = DigestUtils.sha256Hex(documentString); + if (existingFileDigest.equals(newFileDigest)) { + this.unchangedFiles++; + this.unchangedBytes += documentString.length(); + return true; + } + } + } File documentTempFile = new File( documentFile.getAbsolutePath() + ".tmp"); writeToFile(documentTempFile, documentString); @@ -820,6 +838,8 @@ public class DocumentStore { + " %s files listed\n" + " %s files stored\n" + " %s stored\n" + + " %s files not rewritten\n" + + " %s not rewritten\n" + " %s files retrieved\n" + " %s retrieved\n" + " %s files removed\n", @@ -827,6 +847,8 @@ public class DocumentStore { FormattingUtils.formatDecimalNumber(listedFiles), FormattingUtils.formatDecimalNumber(storedFiles), FormattingUtils.formatBytes(storedBytes), + FormattingUtils.formatDecimalNumber(unchangedFiles), + FormattingUtils.formatBytes(unchangedBytes), FormattingUtils.formatDecimalNumber(retrievedFiles), FormattingUtils.formatBytes(retrievedBytes), FormattingUtils.formatDecimalNumber(removedFiles));