commit 8b14cb159b7ee5b4f4e2c11a9a4b03269bda6974 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Jan 11 12:45:09 2018 +0100

Don't rely on system time for compressing histories.

Related to not relying on system time for writing histories, we also shouldn't rely on it for compressing histories anymore. Otherwise we might be compressing the history of a relay or bridge that recently went offline while expecting an uncompressed history of that relay for writing history documents. In short, let's do the same when updating *Status objects as we do when writing *Document objects.

Related to #16513, but not strictly part of it. --- .../torproject/onionoo/docs/BandwidthStatus.java | 23 ++++++++++------------ .../org/torproject/onionoo/docs/ClientsStatus.java | 7 +++---- .../org/torproject/onionoo/docs/WeightsStatus.java | 11 +++++------ .../onionoo/updater/BandwidthStatusUpdater.java | 7 ++++++- .../onionoo/updater/ClientsStatusUpdater.java | 7 ++++++- .../onionoo/updater/WeightsStatusUpdater.java | 7 ++++++- .../torproject/onionoo/docs/WeightsStatusTest.java | 14 ++++++------- 7 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java index 1a81d20..ba22dd4 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java @@ -108,16 +108,13 @@ public class BandwidthStatus extends Document { } }

- public void compressHistory() { - this.compressHistory(System.currentTimeMillis()); + public void compressHistory(long lastSeenMillis) { + this.compressHistory(this.writeHistory, lastSeenMillis); + this.compressHistory(this.readHistory, lastSeenMillis); }

- public void compressHistory(long now) { - this.compressHistory(this.writeHistory, now); - this.compressHistory(this.readHistory, now); - } - - private void compressHistory(SortedMap<Long, long[]> history, long now) { + private void compressHistory(SortedMap<Long, long[]> history, + long lastSeenMillis) { SortedMap<Long, long[]> uncompressedHistory = new TreeMap<>(history); history.clear(); long lastStartMillis = 0L; @@ -129,17 +126,17 @@ public class BandwidthStatus extends Document { long endMillis = v[1]; long bandwidth = v[2]; long intervalLengthMillis; - if (now - endMillis <= DateTimeHelper.THREE_DAYS) { + if (lastSeenMillis - endMillis <= DateTimeHelper.THREE_DAYS) { intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES; - } else if (now - endMillis <= DateTimeHelper.ONE_WEEK) { + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) { intervalLengthMillis = DateTimeHelper.ONE_HOUR; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_MONTH) { intervalLengthMillis = DateTimeHelper.FOUR_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_THREE_MONTHS) { intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_YEAR) { intervalLengthMillis = DateTimeHelper.TWO_DAYS; } else { diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java index 5310abf..a5bc8e8 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java @@ -69,18 +69,17 @@ public class ClientsStatus extends Document {

/** Compresses the history of clients objects by merging adjacent * intervals, depending on how far back in the past they lie. */ - public void compressHistory() { + public void compressHistory(long lastSeenMillis) { SortedSet<ClientsHistory> uncompressedHistory = new TreeSet<>(this.history); history.clear(); ClientsHistory lastResponses = null; String lastMonthString = "1970-01"; - long now = System.currentTimeMillis(); for (ClientsHistory responses : uncompressedHistory) { long intervalLengthMillis; - if (now - responses.getEndMillis() + if (lastSeenMillis - responses.getEndMillis() <= DateTimeHelper.ROUGHLY_THREE_MONTHS) { intervalLengthMillis = DateTimeHelper.ONE_DAY; - } else if (now - responses.getEndMillis() + } else if (lastSeenMillis - responses.getEndMillis() <= DateTimeHelper.ROUGHLY_ONE_YEAR) { intervalLengthMillis = DateTimeHelper.TWO_DAYS; } else { diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index 0dfd4f1..94efda6 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -125,7 +125,7 @@ public class WeightsStatus extends Document {

/** Compresses the history of weights objects by merging adjacent * intervals, depending on how far back in the past they lie. */ - public void compressHistory() { + public void compressHistory(long lastSeenMillis) { SortedMap<long[], double[]> uncompressedHistory = new TreeMap<>(histComparator); uncompressedHistory.putAll(this.history); @@ -135,21 +135,20 @@ public class WeightsStatus extends Document { double[] lastWeights = null; String lastMonthString = "1970-01"; int lastMissingValues = -1; - long now = System.currentTimeMillis(); for (Map.Entry<long[], double[]> e : uncompressedHistory.entrySet()) { long startMillis = e.getKey()[0]; long endMillis = e.getKey()[1]; double[] weights = e.getValue(); long intervalLengthMillis; - if (now - endMillis <= DateTimeHelper.ONE_WEEK) { + if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) { intervalLengthMillis = DateTimeHelper.ONE_HOUR; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_MONTH) { intervalLengthMillis = DateTimeHelper.FOUR_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_THREE_MONTHS) { intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_YEAR) { intervalLengthMillis = DateTimeHelper.TWO_DAYS; } else { diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java index d8288f8..b5e5ddb 100644 --- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.ExtraInfoDescriptor; import org.torproject.onionoo.docs.BandwidthStatus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus;

public class BandwidthStatusUpdater implements DescriptorListener, StatusUpdater { @@ -58,7 +59,11 @@ public class BandwidthStatusUpdater implements DescriptorListener, bandwidthStatus.addToReadHistory(descriptor.getReadHistory()); } if (bandwidthStatus.isDirty()) { - bandwidthStatus.compressHistory(); + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null != nodeStatus) { + bandwidthStatus.compressHistory(nodeStatus.getLastSeenMillis()); + } this.documentStore.store(bandwidthStatus, fingerprint); bandwidthStatus.clearDirty(); } diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java index 0021068..0529dcf 100644 --- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java @@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.ClientsStatus; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.util.FormattingUtils;

import java.util.Map; @@ -158,7 +159,11 @@ public class ClientsStatusUpdater implements DescriptorListener, } clientsStatus.addToHistory(e.getValue()); if (clientsStatus.isDirty()) { - clientsStatus.compressHistory(); + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, hashedFingerprint); + if (null != nodeStatus) { + clientsStatus.compressHistory(nodeStatus.getLastSeenMillis()); + } this.documentStore.store(clientsStatus, hashedFingerprint); clientsStatus.clearDirty(); } diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java index 6c5ec0d..886f2f8 100644 --- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.WeightsStatus;

import java.util.Arrays; @@ -76,7 +77,11 @@ public class WeightsStatusUpdater implements DescriptorListener, weightsStatus.addToHistory(validAfterMillis, freshUntilMillis, weights); if (weightsStatus.isDirty()) { - weightsStatus.compressHistory(); + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null != nodeStatus) { + weightsStatus.compressHistory(nodeStatus.getLastSeenMillis()); + } this.documentStore.store(weightsStatus, fingerprint); weightsStatus.clearDirty(); } diff --git a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java index c0457e7..e28f989 100644 --- a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java @@ -38,16 +38,16 @@ public class WeightsStatusTest { WeightsStatus ws = new WeightsStatus(); SortedMap<long[], double[]> history = ws.getHistory(); assertTrue("actually: " + mapToString(history), history.isEmpty()); - long now = System.currentTimeMillis(); + long lastSeenMillis = 1515670935476L; long hourMillis = 60L * 60L * 1000L; for (long j = 1L; j < 10L; j++) { // add arbitrary data - ws.addToHistory(now - j * hourMillis - hourMillis, - now - j * hourMillis, + ws.addToHistory(lastSeenMillis - j * hourMillis - hourMillis, + lastSeenMillis - j * hourMillis, new double[]{ 1.0 * j, 2.0 * j, 1.0 * j, 2.0 * j, 1.0 * j, 2.0 * j, Double.NaN}); assertEquals("have: " + mapToString(history), (int) j, history.size()); } - ws.compressHistory(); + ws.compressHistory(lastSeenMillis); assertEquals("history map: " + mapToString(history), 9, history.size()); assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(), ws.toDocumentString().contains("NaN")); @@ -67,7 +67,7 @@ public class WeightsStatusTest { 2.0 * j, j * 0.5}); assertEquals("have: " + mapToString(history), (int) j, history.size()); } - ws.compressHistory(); + ws.compressHistory(1515670935476L); assertEquals("history map: " + mapToString(history), 1, history.size()); assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(), ws.toDocumentString().contains("NaN")); @@ -101,7 +101,7 @@ public class WeightsStatusTest { } assertEquals("history: ", correctHistory, mapToString(ws.getHistory())); assertEquals(correctLines.length, ws.getHistory().size()); - ws.compressHistory(); + ws.compressHistory(1515670935476L); assertEquals("found: " + mapToString(ws.getHistory()), 1, ws.getHistory().size()); assertEquals("[1431032400000, 1431043200000] : [-1.0, 1.78279826E-4, " @@ -128,7 +128,7 @@ public class WeightsStatusTest { assertEquals("document string: ", exp, ws.toDocumentString()); } assertEquals(correctLines.length, ws.getHistory().size()); - ws.compressHistory(); + ws.compressHistory(1515670935476L); assertEquals("found: " + mapToString(ws.getHistory()), 3, ws.getHistory().size()); }