tor-commits
Threads by month
- ----- 2025 -----
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
February 2018
- 19 participants
- 1579 discussions

[onionoo/master] Don't rely on system time for compressing histories.
by karsten@torproject.org 07 Feb '18
by karsten@torproject.org 07 Feb '18
07 Feb '18
commit 8b14cb159b7ee5b4f4e2c11a9a4b03269bda6974
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Thu Jan 11 12:45:09 2018 +0100
Don't rely on system time for compressing histories.
Related to not relying on system time for writing histories, we also
shouldn't rely on it for compressing histories anymore. Otherwise we
might be compressing the history of a relay or bridge that recently
went offline while expecting an uncompressed history of that relay for
writing history documents. In short, let's do the same when updating
*Status objects as we do when writing *Document objects.
Related to #16513, but not strictly part of it.
---
.../torproject/onionoo/docs/BandwidthStatus.java | 23 ++++++++++------------
.../org/torproject/onionoo/docs/ClientsStatus.java | 7 +++----
.../org/torproject/onionoo/docs/WeightsStatus.java | 11 +++++------
.../onionoo/updater/BandwidthStatusUpdater.java | 7 ++++++-
.../onionoo/updater/ClientsStatusUpdater.java | 7 ++++++-
.../onionoo/updater/WeightsStatusUpdater.java | 7 ++++++-
.../torproject/onionoo/docs/WeightsStatusTest.java | 14 ++++++-------
7 files changed, 43 insertions(+), 33 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
index 1a81d20..ba22dd4 100644
--- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
@@ -108,16 +108,13 @@ public class BandwidthStatus extends Document {
}
}
- public void compressHistory() {
- this.compressHistory(System.currentTimeMillis());
+ public void compressHistory(long lastSeenMillis) {
+ this.compressHistory(this.writeHistory, lastSeenMillis);
+ this.compressHistory(this.readHistory, lastSeenMillis);
}
- public void compressHistory(long now) {
- this.compressHistory(this.writeHistory, now);
- this.compressHistory(this.readHistory, now);
- }
-
- private void compressHistory(SortedMap<Long, long[]> history, long now) {
+ private void compressHistory(SortedMap<Long, long[]> history,
+ long lastSeenMillis) {
SortedMap<Long, long[]> uncompressedHistory = new TreeMap<>(history);
history.clear();
long lastStartMillis = 0L;
@@ -129,17 +126,17 @@ public class BandwidthStatus extends Document {
long endMillis = v[1];
long bandwidth = v[2];
long intervalLengthMillis;
- if (now - endMillis <= DateTimeHelper.THREE_DAYS) {
+ if (lastSeenMillis - endMillis <= DateTimeHelper.THREE_DAYS) {
intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES;
- } else if (now - endMillis <= DateTimeHelper.ONE_WEEK) {
+ } else if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) {
intervalLengthMillis = DateTimeHelper.ONE_HOUR;
- } else if (now - endMillis
+ } else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_ONE_MONTH) {
intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
- } else if (now - endMillis
+ } else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
- } else if (now - endMillis
+ } else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_ONE_YEAR) {
intervalLengthMillis = DateTimeHelper.TWO_DAYS;
} else {
diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
index 5310abf..a5bc8e8 100644
--- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
@@ -69,18 +69,17 @@ public class ClientsStatus extends Document {
/** Compresses the history of clients objects by merging adjacent
* intervals, depending on how far back in the past they lie. */
- public void compressHistory() {
+ public void compressHistory(long lastSeenMillis) {
SortedSet<ClientsHistory> uncompressedHistory = new TreeSet<>(this.history);
history.clear();
ClientsHistory lastResponses = null;
String lastMonthString = "1970-01";
- long now = System.currentTimeMillis();
for (ClientsHistory responses : uncompressedHistory) {
long intervalLengthMillis;
- if (now - responses.getEndMillis()
+ if (lastSeenMillis - responses.getEndMillis()
<= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
intervalLengthMillis = DateTimeHelper.ONE_DAY;
- } else if (now - responses.getEndMillis()
+ } else if (lastSeenMillis - responses.getEndMillis()
<= DateTimeHelper.ROUGHLY_ONE_YEAR) {
intervalLengthMillis = DateTimeHelper.TWO_DAYS;
} else {
diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
index 0dfd4f1..94efda6 100644
--- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
@@ -125,7 +125,7 @@ public class WeightsStatus extends Document {
/** Compresses the history of weights objects by merging adjacent
* intervals, depending on how far back in the past they lie. */
- public void compressHistory() {
+ public void compressHistory(long lastSeenMillis) {
SortedMap<long[], double[]> uncompressedHistory =
new TreeMap<>(histComparator);
uncompressedHistory.putAll(this.history);
@@ -135,21 +135,20 @@ public class WeightsStatus extends Document {
double[] lastWeights = null;
String lastMonthString = "1970-01";
int lastMissingValues = -1;
- long now = System.currentTimeMillis();
for (Map.Entry<long[], double[]> e : uncompressedHistory.entrySet()) {
long startMillis = e.getKey()[0];
long endMillis = e.getKey()[1];
double[] weights = e.getValue();
long intervalLengthMillis;
- if (now - endMillis <= DateTimeHelper.ONE_WEEK) {
+ if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) {
intervalLengthMillis = DateTimeHelper.ONE_HOUR;
- } else if (now - endMillis
+ } else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_ONE_MONTH) {
intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
- } else if (now - endMillis
+ } else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
- } else if (now - endMillis
+ } else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_ONE_YEAR) {
intervalLengthMillis = DateTimeHelper.TWO_DAYS;
} else {
diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
index d8288f8..b5e5ddb 100644
--- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.ExtraInfoDescriptor;
import org.torproject.onionoo.docs.BandwidthStatus;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
public class BandwidthStatusUpdater implements DescriptorListener,
StatusUpdater {
@@ -58,7 +59,11 @@ public class BandwidthStatusUpdater implements DescriptorListener,
bandwidthStatus.addToReadHistory(descriptor.getReadHistory());
}
if (bandwidthStatus.isDirty()) {
- bandwidthStatus.compressHistory();
+ NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
+ true, fingerprint);
+ if (null != nodeStatus) {
+ bandwidthStatus.compressHistory(nodeStatus.getLastSeenMillis());
+ }
this.documentStore.store(bandwidthStatus, fingerprint);
bandwidthStatus.clearDirty();
}
diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java
index 0021068..0529dcf 100644
--- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java
@@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.ClientsStatus;
import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.util.FormattingUtils;
import java.util.Map;
@@ -158,7 +159,11 @@ public class ClientsStatusUpdater implements DescriptorListener,
}
clientsStatus.addToHistory(e.getValue());
if (clientsStatus.isDirty()) {
- clientsStatus.compressHistory();
+ NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
+ true, hashedFingerprint);
+ if (null != nodeStatus) {
+ clientsStatus.compressHistory(nodeStatus.getLastSeenMillis());
+ }
this.documentStore.store(clientsStatus, hashedFingerprint);
clientsStatus.clearDirty();
}
diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java
index 6c5ec0d..886f2f8 100644
--- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.NetworkStatusEntry;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.WeightsStatus;
import java.util.Arrays;
@@ -76,7 +77,11 @@ public class WeightsStatusUpdater implements DescriptorListener,
weightsStatus.addToHistory(validAfterMillis, freshUntilMillis,
weights);
if (weightsStatus.isDirty()) {
- weightsStatus.compressHistory();
+ NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
+ true, fingerprint);
+ if (null != nodeStatus) {
+ weightsStatus.compressHistory(nodeStatus.getLastSeenMillis());
+ }
this.documentStore.store(weightsStatus, fingerprint);
weightsStatus.clearDirty();
}
diff --git a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java
index c0457e7..e28f989 100644
--- a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java
@@ -38,16 +38,16 @@ public class WeightsStatusTest {
WeightsStatus ws = new WeightsStatus();
SortedMap<long[], double[]> history = ws.getHistory();
assertTrue("actually: " + mapToString(history), history.isEmpty());
- long now = System.currentTimeMillis();
+ long lastSeenMillis = 1515670935476L;
long hourMillis = 60L * 60L * 1000L;
for (long j = 1L; j < 10L; j++) { // add arbitrary data
- ws.addToHistory(now - j * hourMillis - hourMillis,
- now - j * hourMillis,
+ ws.addToHistory(lastSeenMillis - j * hourMillis - hourMillis,
+ lastSeenMillis - j * hourMillis,
new double[]{ 1.0 * j, 2.0 * j, 1.0 * j, 2.0 * j, 1.0 * j,
2.0 * j, Double.NaN});
assertEquals("have: " + mapToString(history), (int) j, history.size());
}
- ws.compressHistory();
+ ws.compressHistory(lastSeenMillis);
assertEquals("history map: " + mapToString(history), 9, history.size());
assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(),
ws.toDocumentString().contains("NaN"));
@@ -67,7 +67,7 @@ public class WeightsStatusTest {
2.0 * j, j * 0.5});
assertEquals("have: " + mapToString(history), (int) j, history.size());
}
- ws.compressHistory();
+ ws.compressHistory(1515670935476L);
assertEquals("history map: " + mapToString(history), 1, history.size());
assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(),
ws.toDocumentString().contains("NaN"));
@@ -101,7 +101,7 @@ public class WeightsStatusTest {
}
assertEquals("history: ", correctHistory, mapToString(ws.getHistory()));
assertEquals(correctLines.length, ws.getHistory().size());
- ws.compressHistory();
+ ws.compressHistory(1515670935476L);
assertEquals("found: " + mapToString(ws.getHistory()), 1,
ws.getHistory().size());
assertEquals("[1431032400000, 1431043200000] : [-1.0, 1.78279826E-4, "
@@ -128,7 +128,7 @@ public class WeightsStatusTest {
assertEquals("document string: ", exp, ws.toDocumentString());
}
assertEquals(correctLines.length, ws.getHistory().size());
- ws.compressHistory();
+ ws.compressHistory(1515670935476L);
assertEquals("found: " + mapToString(ws.getHistory()), 3,
ws.getHistory().size());
}
1
0

[onionoo/master] Fix off-by-one bug in writing history documents.
by karsten@torproject.org 07 Feb '18
by karsten@torproject.org 07 Feb '18
07 Feb '18
commit bd2c7af771a64dd393970e2e483c59778b32dbfc
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Jan 15 11:12:51 2018 +0100
Fix off-by-one bug in writing history documents.
When we consider a history interval from start to end, we really mean
the interval [start, end[. Fixing a few off-by-one bugs in the code.
---
.../java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java | 4 ++--
.../java/org/torproject/onionoo/writer/ClientsDocumentWriter.java | 4 ++--
src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java | 4 ++--
.../java/org/torproject/onionoo/writer/WeightsDocumentWriter.java | 4 ++--
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
index 99a5a00..f1c5041 100644
--- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
@@ -111,7 +111,7 @@ public class BandwidthDocumentWriter implements DocumentWriter {
long totalBandwidth = 0L;
for (long[] v : history.values()) {
long endMillis = v[1];
- if (endMillis < intervalStartMillis) {
+ if (endMillis <= intervalStartMillis) {
continue;
} else if (endMillis > graphEndMillis) {
break;
@@ -124,7 +124,7 @@ public class BandwidthDocumentWriter implements DocumentWriter {
continue;
}
while ((intervalStartMillis / dataPointInterval)
- != (endMillis / dataPointInterval)) {
+ != ((endMillis - 1L) / dataPointInterval)) {
dataPoints.add(totalMillis * 5L < dataPointInterval
? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
/ totalMillis);
diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
index 834df9c..81168f5 100644
--- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
@@ -140,13 +140,13 @@ public class ClientsDocumentWriter implements DocumentWriter {
long millis = 0L;
double responses = 0.0;
for (ClientsHistory hist : history) {
- if (hist.getEndMillis() < intervalStartMillis) {
+ if (hist.getEndMillis() <= intervalStartMillis) {
continue;
} else if (hist.getEndMillis() > graphEndMillis) {
break;
}
while ((intervalStartMillis / dataPointInterval)
- != (hist.getEndMillis() / dataPointInterval)) {
+ != ((hist.getEndMillis() - 1L) / dataPointInterval)) {
dataPoints.add(millis * 2L < dataPointInterval
? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
/ (((double) millis) * 10.0));
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index f739b9e..2aee7a1 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -175,7 +175,7 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
* hist.getUptimeHours();
- if (histEndMillis < intervalStartMillis) {
+ if (histEndMillis <= intervalStartMillis) {
continue;
} else if (histEndMillis > graphEndMillis) {
histEndMillis = graphEndMillis;
@@ -218,7 +218,7 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
* hist.getUptimeHours();
- if (histEndMillis < intervalStartMillis) {
+ if (histEndMillis <= intervalStartMillis) {
continue;
} else if (histEndMillis > graphEndMillis) {
histEndMillis = graphEndMillis;
diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
index ad54fea..10d8a94 100644
--- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
@@ -132,13 +132,13 @@ public class WeightsDocumentWriter implements DocumentWriter {
long startMillis = e.getKey()[0];
long endMillis = e.getKey()[1];
double weight = e.getValue()[graphTypeIndex];
- if (endMillis < intervalStartMillis) {
+ if (endMillis <= intervalStartMillis) {
continue;
} else if (endMillis > graphEndMillis) {
break;
}
while ((intervalStartMillis / dataPointInterval)
- != (endMillis / dataPointInterval)) {
+ != ((endMillis - 1L) / dataPointInterval)) {
dataPoints.add(totalMillis * 5L < dataPointInterval
? -1.0 : totalWeightTimesMillis / (double) totalMillis);
totalWeightTimesMillis = 0.0;
1
0

07 Feb '18
commit 314feacaf18dae7e1f29861b39a74fc3df2d1330
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Wed Jan 17 20:29:11 2018 +0100
Compile graph histories in a single place.
---
.../org/torproject/onionoo/docs/UptimeHistory.java | 4 +
.../onionoo/writer/BandwidthDocumentWriter.java | 120 +-------
.../onionoo/writer/ClientsDocumentWriter.java | 130 +--------
.../onionoo/writer/GraphHistoryCompiler.java | 254 +++++++++++++++++
.../onionoo/writer/UptimeDocumentWriter.java | 312 +++++++--------------
.../onionoo/writer/WeightsDocumentWriter.java | 124 +-------
.../writer/BandwidthDocumentWriterTest.java | 4 +-
.../onionoo/writer/GraphHistoryCompilerTest.java | 203 ++++++++++++++
.../onionoo/writer/UptimeDocumentWriterTest.java | 14 +-
9 files changed, 606 insertions(+), 559 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
index 60e283f..f8cc116 100644
--- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
+++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
@@ -32,6 +32,10 @@ public class UptimeHistory implements Comparable<UptimeHistory> {
return this.uptimeHours;
}
+ public long getEndMillis() {
+ return this.startMillis + DateTimeHelper.ONE_HOUR * this.uptimeHours;
+ }
+
private SortedSet<String> flags;
public SortedSet<String> getFlags() {
diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
index 2f27271..71595e2 100644
--- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
@@ -15,12 +15,7 @@ import org.torproject.onionoo.docs.UpdateStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -100,114 +95,17 @@ public class BandwidthDocumentWriter implements DocumentWriter {
private Map<String, GraphHistory> compileGraphType(long lastSeenMillis,
SortedMap<Long, long[]> history) {
- Map<String, GraphHistory> graphs = new LinkedHashMap<>();
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ lastSeenMillis + DateTimeHelper.ONE_HOUR);
for (int i = 0; i < this.graphIntervals.length; i++) {
- String graphName = this.graphNames[i];
- Period graphInterval = this.graphIntervals[i];
- long dataPointInterval = this.dataPointIntervals[i];
- List<Long> dataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- long totalMillis = 0L;
- long totalBandwidth = 0L;
- for (long[] v : history.values()) {
- long endMillis = v[1];
- if (endMillis <= intervalStartMillis) {
- continue;
- } else if (endMillis > graphEndMillis) {
- break;
- }
- long startMillis = v[0];
- if (endMillis - startMillis > dataPointInterval) {
- /* This history interval is too long for this graph's data point
- * interval. Maybe the next graph will contain it, but not this
- * one. */
- continue;
- }
- while ((intervalStartMillis / dataPointInterval)
- != ((endMillis - 1L) / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
- / totalMillis);
- totalBandwidth = 0L;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- long bandwidth = v[2];
- totalBandwidth += bandwidth;
- totalMillis += (endMillis - startMillis);
- }
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
- / totalMillis);
- long maxValue = 1L;
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int j = 0; j < dataPoints.size(); j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = j;
- }
- lastNonNullIndex = j;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- continue;
- }
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[i - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
-
- /* Skip bandwidth history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * bandwidth history object(s). Unless we did not include any of
- * the previous bandwidth history objects for other reasons, in
- * which case we should include this one. */
- continue;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (j - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = j;
- }
- values.add(dataPoint < 0L ? null
- : (int) ((dataPoint * 999L) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- graphs.put(graphName, graphHistory);
- }
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
+ }
+ for (long[] v : history.values()) {
+ ghc.addHistoryEntry(v[0], v[1],
+ (double) (v[2] * DateTimeHelper.ONE_SECOND));
}
- return graphs;
+ return ghc.compileGraphHistories();
}
@Override
diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
index 4eca33a..aba45cf 100644
--- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
@@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.ClientsStatus;
import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
-import org.torproject.onionoo.docs.GraphHistory;
import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UpdateStatus;
import org.torproject.onionoo.util.FormattingUtils;
@@ -17,13 +16,7 @@ import org.torproject.onionoo.util.FormattingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
import java.util.SortedSet;
/*
@@ -115,122 +108,19 @@ public class ClientsDocumentWriter implements DocumentWriter {
NodeStatus nodeStatus, SortedSet<ClientsHistory> history) {
ClientsDocument clientsDocument = new ClientsDocument();
clientsDocument.setFingerprint(hashedFingerprint);
- Map<String, GraphHistory> averageClients = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileClientsHistory(
- graphIntervalIndex, history, nodeStatus.getLastSeenMillis());
- if (graphHistory != null) {
- averageClients.put(graphName, graphHistory);
- }
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ nodeStatus.getLastSeenMillis() + DateTimeHelper.ONE_HOUR);
+ ghc.setThreshold(2L);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
}
- clientsDocument.setAverageClients(averageClients);
- return clientsDocument;
- }
-
- private GraphHistory compileClientsHistory(
- int graphIntervalIndex, SortedSet<ClientsHistory> history,
- long lastSeenMillis) {
- Period graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- List<Double> dataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- long millis = 0L;
- double responses = 0.0;
for (ClientsHistory hist : history) {
- if (hist.getEndMillis() <= intervalStartMillis) {
- continue;
- } else if (hist.getEndMillis() > graphEndMillis) {
- break;
- }
- while ((intervalStartMillis / dataPointInterval)
- != ((hist.getEndMillis() - 1L) / dataPointInterval)) {
- dataPoints.add(millis * 2L < dataPointInterval
- ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
- / (((double) millis) * 10.0));
- responses = 0.0;
- millis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- responses += hist.getTotalResponses();
- millis += (hist.getEndMillis() - hist.getStartMillis());
- }
- dataPoints.add(millis * 2L < dataPointInterval
- ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
- / (((double) millis) * 10.0));
- double maxValue = 0.0;
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- /* Not a single non-negative value in the data points. */
- return null;
- }
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[graphIntervalIndex - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
- /* Skip clients history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * clients history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex
- <= lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < 0.0 ? null :
- (int) ((dataPoint * 999.0) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- /* There are no two adjacent values in the data points that are
- * required to draw a line graph. */
- return null;
+ ghc.addHistoryEntry(hist.getStartMillis(), hist.getEndMillis(),
+ hist.getTotalResponses() * ((double) DateTimeHelper.ONE_DAY) / 10.0);
}
+ clientsDocument.setAverageClients(ghc.compileGraphHistories());
+ return clientsDocument;
}
@Override
diff --git a/src/main/java/org/torproject/onionoo/writer/GraphHistoryCompiler.java b/src/main/java/org/torproject/onionoo/writer/GraphHistoryCompiler.java
new file mode 100644
index 0000000..853ab32
--- /dev/null
+++ b/src/main/java/org/torproject/onionoo/writer/GraphHistoryCompiler.java
@@ -0,0 +1,254 @@
+/* Copyright 2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.onionoo.writer;
+
+import org.torproject.onionoo.docs.DateTimeHelper;
+import org.torproject.onionoo.docs.GraphHistory;
+
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Helper class to compile graph histories. */
+public class GraphHistoryCompiler {
+
+ private long graphsEndMillis;
+
+ /**
+ * Instantiates a new graph history compiler with the provided end time for
+ * all compiled graphs.
+ *
+ * @param graphsEndMillis End time for all compiled graphs.
+ */
+ GraphHistoryCompiler(long graphsEndMillis) {
+ this.graphsEndMillis = graphsEndMillis;
+ }
+
+ private boolean divisible = false;
+
+ /**
+ * Set whether history elements are divisible in the sense that they may be
+ * longer than one data point; this is the case for uptime intervals where
+ * uptime is equally distributed over potentially many data point intervals,
+ * but it's not the case for bandwidth/weights/clients intervals where
+ * observations are given for fixed-size reporting intervals. */
+ void setDivisible(boolean divisible) {
+ this.divisible = divisible;
+ }
+
+ private long threshold = 5;
+
+ /**
+ * Set the threshold (given as reciprocal value) of available history entries
+ * for any given data points below which the data point will be counted as
+ * null (missing); default is 5 for 1/5 = 20%.
+ */
+ void setThreshold(long threshold) {
+ this.threshold = threshold;
+ }
+
+ private List<String> graphNames = new ArrayList<>();
+
+ private List<Period> graphIntervals = new ArrayList<>();
+
+ private List<Long> dataPointIntervals = new ArrayList<>();
+
+ /**
+ * Add a graph type with the given graph name, graph interval, and data point
+ * interval.
+ *
+ * @param graphName Graph name, like "1_week".
+ * @param graphInterval Graph interval, like Period.ofWeeks(1).
+ * @param dataPointInterval Data point interval, like 1 hour in milliseconds.
+ */
+ void addGraphType(String graphName, Period graphInterval,
+ Long dataPointInterval) {
+ this.graphNames.add(graphName);
+ this.graphIntervals.add(graphInterval);
+ this.dataPointIntervals.add(dataPointInterval);
+ }
+
+ private Map<long[], Double> history = new LinkedHashMap<>();
+
+ /**
+ * Add a history entry with given start and end time and value.
+ *
+ * @param startMillis Start time in milliseconds.
+ * @param endMillis End time in milliseconds.
+ * @param value History entry value.
+ */
+ void addHistoryEntry(long startMillis, long endMillis, double value) {
+ this.history.put(new long[] { startMillis, endMillis }, value);
+ }
+
+ /**
+ * Compile graph histories from the history entries provided earlier.
+ *
+ * @return Map with graph names as keys and GraphHistory instances as values.
+ */
+ Map<String, GraphHistory> compileGraphHistories() {
+ Map<String, GraphHistory> graphs = new LinkedHashMap<>();
+ for (int graphIntervalIndex = 0;
+ graphIntervalIndex < this.graphIntervals.size();
+ graphIntervalIndex++) {
+
+ /* Look up graph name, graph interval, and data point interval from the
+ * graph type details provided earlier. */
+ final String graphName = this.graphNames.get(graphIntervalIndex);
+ Period graphInterval = this.graphIntervals.get(graphIntervalIndex);
+ long dataPointInterval = this.dataPointIntervals.get(graphIntervalIndex);
+
+ /* Determine graph end time as the end time for all graphs, rounded down
+ * to the last full data point interval. */
+ long graphEndMillis = (this.graphsEndMillis / dataPointInterval)
+ * dataPointInterval;
+
+ /* Determine graph start time as graph end time minus graph interval,
+ * rounded down to the last full data point interval. */
+ long graphStartMillis = ((LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphInterval)
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) / dataPointInterval)
+ * dataPointInterval;
+
+ /* Keep input for graph values in two arrays, one for values * millis,
+ * another one for millis. */
+ int dataPoints = (int) ((graphEndMillis - graphStartMillis)
+ / dataPointInterval);
+ double[] totalValues = new double[dataPoints];
+ long[] totalMillis = new long[dataPoints];
+
+ /* Iterate over all history entries and see which ones we need for this
+ * graph. */
+ for (Map.Entry<long[], Double> h : this.history.entrySet()) {
+ long startMillis = h.getKey()[0];
+ long endMillis = h.getKey()[1];
+ double value = h.getValue();
+
+ /* If a history entry ends before this graph starts or starts before
+ * this graph ends, skip it. */
+ if (endMillis <= graphStartMillis || startMillis >= graphEndMillis) {
+ continue;
+ }
+
+ /* If history entries are not divisible and this entry is longer than
+ * the data point interval, skip it. Maybe the next graph will contain
+ * it, but not this one. */
+ if (!this.divisible && endMillis - startMillis > dataPointInterval) {
+ continue;
+ }
+
+ /* Iterate over all data points that this history element falls into.
+ * Even if history entries are not divisible, we may have to split it
+ * over two data points, because reported statistics rarely align with
+ * our data point intervals. And if history entries are divisible, we
+ * may have to split them over many data points. */
+ for (long intervalStartMillis = startMillis;
+ intervalStartMillis < endMillis;
+ intervalStartMillis = ((intervalStartMillis + dataPointInterval)
+ / dataPointInterval) * dataPointInterval) {
+
+ /* Determine the data point that this (partial) history entry falls
+ * into. And if it's out of bounds, skip it. */
+ int dataPointIndex = (int) ((intervalStartMillis - graphStartMillis)
+ / dataPointInterval);
+ if (dataPointIndex < 0 || dataPointIndex >= dataPoints) {
+ continue;
+ }
+
+ /* Determine the interval end, which may be the end of the data point
+ * or the end of the history entry, whichever comes first. Then add
+ * values and millis to the data point. */
+ long intervalEndMillis = Math.min(endMillis, ((intervalStartMillis
+ + dataPointInterval) / dataPointInterval) * dataPointInterval);
+ long millis = intervalEndMillis - intervalStartMillis;
+ totalValues[dataPointIndex] += (value * (double) millis)
+ / (double) (endMillis - startMillis);
+ totalMillis[dataPointIndex] += millis;
+ }
+ }
+
+ /* Go through the previously compiled data points and extract some pieces
+ * that will be relevant for deciding whether to include this graph and
+ * for adding meta data to the GraphHistory object. */
+ double maxValue = 0.0;
+ int firstNonNullIndex = -1;
+ int lastNonNullIndex = -1;
+ boolean foundTwoAdjacentDataPoints = false;
+ for (int dataPointIndex = 0, previousNonNullIndex = -2;
+ dataPointIndex < dataPoints; dataPointIndex++) {
+
+ /* Only consider data points containing values for at least the given
+ * threshold of time (20% by default). If so, record first and last
+ * data point containing data, whether there exist two adjacent data
+ * points containing data, and determine the maximum value. */
+ if (totalMillis[dataPointIndex] * this.threshold >= dataPointInterval) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ maxValue = Math.max(maxValue, totalValues[dataPointIndex]
+ / totalMillis[dataPointIndex]);
+ }
+ }
+
+ /* If there are not at least two adjacent data points containing data,
+ * skip the graph. */
+ if (!foundTwoAdjacentDataPoints) {
+ continue;
+ }
+
+ /* Calculate the timestamp of the first data point containing data. */
+ long firstDataPointMillis = graphStartMillis + firstNonNullIndex
+ * dataPointInterval + dataPointInterval / 2L;
+
+ /* If the graph doesn't contain anything new that wasn't already contained
+ * in previously compiled graphs, skip this graph. */
+ if (graphIntervalIndex > 0 && !graphs.isEmpty()
+ && firstDataPointMillis >= LocalDateTime.ofEpochSecond(
+ graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(this.graphIntervals.get(graphIntervalIndex - 1))
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) {
+ continue;
+ }
+
+ /* Put together the list of values that will go into the graph. */
+ List<Integer> values = new ArrayList<>();
+ for (int dataPointIndex = firstNonNullIndex;
+ dataPointIndex <= lastNonNullIndex; dataPointIndex++) {
+ if (totalMillis[dataPointIndex] * this.threshold >= dataPointInterval) {
+ values.add((int) ((totalValues[dataPointIndex] * 999.0)
+ / (maxValue * totalMillis[dataPointIndex])));
+ } else {
+ values.add(null);
+ }
+ }
+
+ /* Put together a GraphHistory object and add it to the map under the
+ * given graph name. */
+ GraphHistory graphHistory = new GraphHistory();
+ graphHistory.setFirst(firstDataPointMillis);
+ graphHistory.setLast(firstDataPointMillis + (lastNonNullIndex
+ - firstNonNullIndex) * dataPointInterval);
+ graphHistory.setInterval((int) (dataPointInterval
+ / DateTimeHelper.ONE_SECOND));
+ graphHistory.setFactor(maxValue / 999.0);
+ graphHistory.setCount(lastNonNullIndex - firstNonNullIndex + 1);
+ graphHistory.setValues(values);
+ graphs.put(graphName, graphHistory);
+ }
+
+ /* We're done. Return the map of compiled graphs. */
+ return graphs;
+ }
+}
+
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index d1e2003..12ba8fa 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -17,12 +17,8 @@ import org.torproject.onionoo.util.FormattingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
+import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -110,18 +106,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis) {
UptimeDocument uptimeDocument = new UptimeDocument();
uptimeDocument.setFingerprint(fingerprint);
- Map<String, GraphHistory> uptime = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileUptimeHistory(
- graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis,
- null);
- if (graphHistory != null) {
- uptime.put(graphName, graphHistory);
- }
- }
- uptimeDocument.setUptime(uptime);
+ uptimeDocument.setUptime(this.compileUptimeHistory(relay, history,
+ knownStatuses, lastSeenMillis, null));
SortedMap<String, Map<String, GraphHistory>> flags = new TreeMap<>();
SortedSet<String> allFlags = new TreeSet<>();
for (UptimeHistory hist : history) {
@@ -130,17 +116,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
}
for (String flag : allFlags) {
- Map<String, GraphHistory> graphsForFlags = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileUptimeHistory(
- graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis,
- flag);
- if (graphHistory != null) {
- graphsForFlags.put(graphName, graphHistory);
- }
- }
+ Map<String, GraphHistory> graphsForFlags = this.compileUptimeHistory(
+ relay, history, knownStatuses, lastSeenMillis, flag);
if (!graphsForFlags.isEmpty()) {
flags.put(flag, graphsForFlags);
}
@@ -151,187 +128,116 @@ public class UptimeDocumentWriter implements DocumentWriter {
return uptimeDocument;
}
- private GraphHistory compileUptimeHistory(int graphIntervalIndex,
- boolean relay, SortedSet<UptimeHistory> history,
- SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis,
- String flag) {
- Period graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- int dataPointIntervalHours = (int) (dataPointInterval
- / DateTimeHelper.ONE_HOUR);
- List<Integer> uptimeDataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- int uptimeHours = 0;
- long firstStatusStartMillis = -1L;
- for (UptimeHistory hist : history) {
- if (hist.isRelay() != relay
- || (flag != null && (hist.getFlags() == null
- || !hist.getFlags().contains(flag)))) {
- continue;
- }
- if (firstStatusStartMillis < 0L) {
- firstStatusStartMillis = hist.getStartMillis();
- }
- long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
- * hist.getUptimeHours();
- if (histEndMillis <= intervalStartMillis) {
- continue;
- } else if (histEndMillis > graphEndMillis) {
- histEndMillis = graphEndMillis;
- }
- while (hist.getStartMillis() >= intervalStartMillis
- + dataPointInterval) {
- if (firstStatusStartMillis < intervalStartMillis
- + dataPointInterval) {
- uptimeDataPoints.add(uptimeHours);
- } else {
- uptimeDataPoints.add(-1);
- }
- uptimeHours = 0;
- intervalStartMillis += dataPointInterval;
- }
- while (histEndMillis >= intervalStartMillis + dataPointInterval) {
- uptimeHours += (int) ((intervalStartMillis + dataPointInterval
- - Math.max(hist.getStartMillis(), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- uptimeDataPoints.add(uptimeHours);
- uptimeHours = 0;
- intervalStartMillis += dataPointInterval;
- }
- uptimeHours += (int) ((histEndMillis - Math.max(
- hist.getStartMillis(), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- }
- uptimeDataPoints.add(uptimeHours);
- List<Integer> statusDataPoints = new ArrayList<>();
- intervalStartMillis = graphStartMillis;
- int statusHours = -1;
- for (UptimeHistory hist : knownStatuses) {
- if (hist.getStartMillis() >= graphEndMillis) {
- break;
- }
- if (hist.isRelay() != relay
- || (flag != null && (hist.getFlags() == null
- || !hist.getFlags().contains(flag)))) {
- continue;
- }
- long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
- * hist.getUptimeHours();
- if (histEndMillis <= intervalStartMillis) {
- continue;
- } else if (histEndMillis > graphEndMillis) {
- histEndMillis = graphEndMillis;
- }
- while (hist.getStartMillis() >= intervalStartMillis
- + dataPointInterval) {
- statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
- ? -1 : statusHours);
- statusHours = -1;
- intervalStartMillis += dataPointInterval;
- }
- while (histEndMillis >= intervalStartMillis + dataPointInterval) {
- if (statusHours < 0) {
- statusHours = 0;
- }
- statusHours += (int) ((intervalStartMillis + dataPointInterval
- - Math.max(Math.max(hist.getStartMillis(),
- firstStatusStartMillis), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
- ? -1 : statusHours);
- statusHours = -1;
- intervalStartMillis += dataPointInterval;
- }
- if (statusHours < 0) {
- statusHours = 0;
- }
- statusHours += (int) ((histEndMillis - Math.max(Math.max(
- hist.getStartMillis(), firstStatusStartMillis),
- intervalStartMillis)) / DateTimeHelper.ONE_HOUR);
- }
- if (statusHours > 0) {
- statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
- ? -1 : statusHours);
- }
- List<Double> dataPoints = new ArrayList<>();
- for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size();
- dataPointIndex++) {
- if (dataPointIndex >= uptimeDataPoints.size()) {
- dataPoints.add(0.0);
- } else if (uptimeDataPoints.get(dataPointIndex) >= 0
- && statusDataPoints.get(dataPointIndex) > 0) {
- dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex))
- / ((double) statusDataPoints.get(dataPointIndex)));
- } else {
- dataPoints.add(-1.0);
- }
- }
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- }
- }
- if (firstNonNullIndex < 0) {
- /* Not a single non-negative value in the data points. */
+ private Map<String, GraphHistory> compileUptimeHistory(boolean relay,
+ SortedSet<UptimeHistory> history, SortedSet<UptimeHistory> knownStatuses,
+ long lastSeenMillis, String flag) {
+
+ /* Extracting history entries for compiling GraphHistory objects is a bit
+ * harder than for the other document types. The reason is that we have to
+ * combine (A) uptime history of all relays/bridges and (B) uptime history
+ * of the relay/bridge that we're writing the document for. We're going to
+ * refer to A and B below, to simplify descriptions a bit. */
+
+ /* If there are either no A entries or no B entries, we can't compile
+ * graphs. */
+ if (history.isEmpty() || knownStatuses.isEmpty()) {
return null;
}
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[graphIntervalIndex - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
- /* Skip uptime history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * uptime history object(s). */
+
+ /* Initialize the graph history compiler, and tell it that history entries
+ * are divisible. This is different from the other history writers. */
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ lastSeenMillis + DateTimeHelper.ONE_HOUR);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
+ }
+ ghc.setDivisible(true);
+
+ /* The general idea for extracting history entries and passing them to the
+ * graph history compiler is to iterate over A entries one by one and keep
+ * an Iterator for B entries to move forward as "time" proceeds. */
+ Iterator<UptimeHistory> historyIterator = history.iterator();
+ UptimeHistory hist;
+ do {
+ hist = historyIterator.hasNext() ? historyIterator.next() : null;
+ } while (null != hist && (hist.isRelay() != relay
+ || (null != flag && (null == hist.getFlags()
+ || !hist.getFlags().contains(flag)))));
+
+ /* If there is not at least one B entry, we can't compile graphs. */
+ if (null == hist) {
return null;
}
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(1.0 / 999.0);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex
- <= lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
+
+ for (UptimeHistory statuses : knownStatuses) {
+
+ /* If this A entry contains uptime information that we're not interested
+ * in, skip it. */
+ if (statuses.isRelay() != relay
+ || (null != flag && (null == statuses.getFlags()
+ || !statuses.getFlags().contains(flag)))) {
+ continue;
}
- values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0)));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- /* There are no two adjacent values in the data points that are
- * required to draw a line graph. */
- return null;
+
+ /* The "current" time is the time that we're currently considering as part
+ * of the A entry. It starts out as the interval start, but as we may
+ * consider multiple B entries, it may proceed. The loop ends when
+ * "current" time has reached the end of the considered A entry. */
+ long currentTimeMillis = statuses.getStartMillis();
+ do {
+ if (null == hist) {
+
+ /* There is no B entry left, which means that the relay/bridge was
+ * offline from "current" time to the end of the A entry. */
+ ghc.addHistoryEntry(currentTimeMillis, statuses.getEndMillis(),0.0);
+ currentTimeMillis = statuses.getEndMillis();
+ } else if (statuses.getEndMillis() <= hist.getStartMillis()) {
+
+ /* This A entry ends before the B entry starts. If there was an
+ * earlier B entry, count this time as offline time. */
+ if (history.first().getStartMillis() <= currentTimeMillis) {
+ ghc.addHistoryEntry(currentTimeMillis, statuses.getEndMillis(),
+ 0.0);
+ }
+ currentTimeMillis = statuses.getEndMillis();
+ } else {
+
+ /* A and B entries overlap. First, if there's time between "current"
+ * time and the time when B starts, possibly count that as offline
+ * time, but only if the relay was around earlier. */
+ if (currentTimeMillis < hist.getStartMillis()) {
+ if (history.first().getStartMillis() <= currentTimeMillis) {
+ ghc.addHistoryEntry(currentTimeMillis, hist.getStartMillis(),
+ 0.0);
+ }
+ currentTimeMillis = hist.getStartMillis();
+ }
+
+ /* Now handle the actually overlapping part. First determine when the
+ * overlap ends, then add a history entry with the number of uptime
+ * milliseconds as value. */
+ long overlapEndMillis = Math.min(statuses.getEndMillis(),
+ hist.getEndMillis());
+ ghc.addHistoryEntry(currentTimeMillis, overlapEndMillis,
+ overlapEndMillis - currentTimeMillis);
+ currentTimeMillis = overlapEndMillis;
+
+ /* If A ends after B, move on to the next B entry. */
+ if (statuses.getEndMillis() >= hist.getEndMillis()) {
+ do {
+ hist = historyIterator.hasNext() ? historyIterator.next() : null;
+ } while (null != hist && (hist.isRelay() != relay
+ || (null != flag && (null == hist.getFlags()
+ || !hist.getFlags().contains(flag)))));
+ }
+ }
+ } while (currentTimeMillis < statuses.getEndMillis());
}
+
+ /* Now that the graph history compiler knows all relevant history, ask it to
+ * compile graphs for us, and return them. */
+ return ghc.compileGraphHistories();
}
@Override
diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
index f4e2c3a..b34a9e6 100644
--- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
@@ -15,12 +15,7 @@ import org.torproject.onionoo.docs.WeightsStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -105,125 +100,22 @@ public class WeightsDocumentWriter implements DocumentWriter {
private Map<String, GraphHistory> compileGraphType(
SortedMap<long[], double[]> history, long lastSeenMillis,
int graphTypeIndex) {
- Map<String, GraphHistory> graphs = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileWeightsHistory(
- graphTypeIndex, graphIntervalIndex, history, lastSeenMillis);
- if (graphHistory != null) {
- graphs.put(graphName, graphHistory);
- }
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ lastSeenMillis + DateTimeHelper.ONE_HOUR);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
}
- return graphs;
- }
-
- private GraphHistory compileWeightsHistory(int graphTypeIndex,
- int graphIntervalIndex, SortedMap<long[], double[]> history,
- long lastSeenMillis) {
- Period graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- List<Double> dataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- long totalMillis = 0L;
- double totalWeightTimesMillis = 0.0;
for (Map.Entry<long[], double[]> e : history.entrySet()) {
long startMillis = e.getKey()[0];
long endMillis = e.getKey()[1];
double weight = e.getValue()[graphTypeIndex];
- if (endMillis <= intervalStartMillis) {
- continue;
- } else if (endMillis > graphEndMillis) {
- break;
- }
- while ((intervalStartMillis / dataPointInterval)
- != ((endMillis - 1L) / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
- totalWeightTimesMillis = 0.0;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
if (weight >= 0.0) {
- totalWeightTimesMillis += weight
- * ((double) (endMillis - startMillis));
- totalMillis += (endMillis - startMillis);
+ ghc.addHistoryEntry(startMillis, endMillis,
+ weight * ((double) (endMillis - startMillis)));
}
}
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
- double maxValue = 0.0;
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- /* Not a single non-negative value in the data points. */
- return null;
- }
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[graphIntervalIndex - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
- /* Skip weights history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * weights history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex
- <= lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < 0.0 ? null :
- (int) ((dataPoint * 999.0) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- /* There are no two adjacent values in the data points that are
- * required to draw a line graph. */
- return null;
- }
+ return ghc.compileGraphHistories();
}
@Override
diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
index 324122c..fa76699 100644
--- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
@@ -74,9 +74,9 @@ public class BandwidthDocumentWriterTest {
assertEquals(1, document.getReadHistory().size());
assertTrue(document.getReadHistory().containsKey("1_month"));
GraphHistory history = document.getReadHistory().get("1_month");
- assertEquals(DateTimeHelper.parse(dayBeforeYesterday + " 14:00:00"),
+ assertEquals(DateTimeHelper.parse(dayBeforeYesterday + " 10:00:00"),
history.getFirst());
- assertEquals(DateTimeHelper.parse(yesterday + " 02:00:00"),
+ assertEquals(DateTimeHelper.parse(dayBeforeYesterday + " 22:00:00"),
history.getLast());
assertEquals(DateTimeHelper.FOUR_HOURS / DateTimeHelper.ONE_SECOND,
(int) history.getInterval());
diff --git a/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java b/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
new file mode 100644
index 0000000..6d9c461
--- /dev/null
+++ b/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
@@ -0,0 +1,203 @@
+/* Copyright 2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.onionoo.writer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.torproject.onionoo.docs.DateTimeHelper;
+import org.torproject.onionoo.docs.GraphHistory;
+
+import com.google.gson.Gson;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.time.Period;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+(a)RunWith(Parameterized.class)
+public class GraphHistoryCompilerTest {
+
+ /** Provide test data. */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { "Empty history",
+ false, new String[0][], 0, null, null, null, null, null, null,
+ null },
+ { "Single entry right before graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 23:00", "2018-01-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive entries right before graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 22:00", "2017-12-31 23:00", "1" },
+ new String[] { "2017-12-31 23:00", "2018-01-01 00:00", "1" }},
+ 1, "1_week", "2017-12-31 22:30", "2017-12-31 23:30", 3600, 0.001, 2,
+ new Integer[] { 999, 999 } },
+ { "Two non-consecutive entries towards graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 21:00", "2017-12-31 22:00", "1" },
+ new String[] { "2017-12-31 23:00", "2018-01-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive entries passing 1 week threshold",
+ false, new String[][] {
+ new String[] { "2017-12-24 23:00", "2017-12-25 00:00", "1" },
+ new String[] { "2017-12-25 00:00", "2017-12-25 01:00", "1" }},
+ 1, "1_month", "2017-12-24 22:00", "2017-12-25 02:00", 14400,
+ 2.7805583361138913E-10, 2, new Integer[] { 999, 999 } },
+ { "Two consecutive 1-hour entries over 1 week from graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-21 22:00", "2017-12-21 23:00", "1" },
+ new String[] { "2017-12-21 23:00", "2017-12-22 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive 4-hour entries over 1 week from graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-21 16:00", "2017-12-21 20:00", "1" },
+ new String[] { "2017-12-21 20:00", "2017-12-22 00:00", "1" }},
+ 1, "1_month", "2017-12-21 18:00", "2017-12-21 22:00", 14400, 0.001,
+ 2, new Integer[] { 999, 999 } },
+ { "Two consecutive 4-hour entries right before graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 16:00", "2017-12-31 20:00", "1" },
+ new String[] { "2017-12-31 20:00", "2018-01-01 00:00", "1" }},
+ 1, "1_month", "2017-12-31 18:00", "2017-12-31 22:00", 14400, 0.001,
+ 2, new Integer[] { 999, 999 } },
+ { "Single 1-week divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-25 00:00", "2018-01-01 00:00", "1" }},
+ 1, "1_week", "2017-12-25 00:30", "2017-12-31 23:30", 3600, 0.001,
+ 168, null },
+ { "Single 1-week-and-1-hour divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-24 23:00", "2018-01-01 00:00", "1" }},
+ 2, "1_month", "2017-12-24 22:00", "2017-12-31 22:00", 14400, 0.001,
+ 43, null },
+ { "Single 66-minute divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-31 22:54", "2018-01-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Single 72-minute divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-31 22:48", "2018-01-01 00:00", "1" }},
+ 1, "1_week", "2017-12-31 22:30", "2017-12-31 23:30", 3600, 0.001,
+ 2, null },
+ { "Single 6-month divisible entry 6 years before graphs end",
+ true, new String[][] {
+ new String[] { "2012-01-01 00:00", "2012-07-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive 1-hour entries right after graphs end",
+ false, new String[][] {
+ new String[] { "2018-01-01 00:00", "2018-01-01 01:00", "1" },
+ new String[] { "2018-01-01 01:00", "2018-01-01 02:00", "1" }},
+ 0, null, null, null, null, null, null, null }
+ });
+ }
+
+ @Parameter
+ public String testDescription;
+
+ @Parameter(1)
+ public boolean divisible;
+
+ @Parameter(2)
+ public String[][] historyEntries;
+
+ @Parameter(3)
+ public int expectedGraphs;
+
+ @Parameter(4)
+ public String expectedGraphName;
+
+ @Parameter(5)
+ public String expectedFirst;
+
+ @Parameter(6)
+ public String expectedLast;
+
+ @Parameter(7)
+ public Integer expectedInterval;
+
+ @Parameter(8)
+ public Double expectedFactor;
+
+ @Parameter(9)
+ public Integer expectedCount;
+
+ @Parameter(10)
+ public Integer[] expectedValues;
+
+ private final String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private final Period[] graphIntervals = new Period[] {
+ Period.ofWeeks(1),
+ Period.ofMonths(1),
+ Period.ofMonths(3),
+ Period.ofYears(1),
+ Period.ofYears(5) };
+
+ private final long[] dataPointIntervals = new long[] {
+ DateTimeHelper.ONE_HOUR,
+ DateTimeHelper.FOUR_HOURS,
+ DateTimeHelper.TWELVE_HOURS,
+ DateTimeHelper.TWO_DAYS,
+ DateTimeHelper.TEN_DAYS };
+
+ @Test
+ public void test() {
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(DateTimeHelper.parse(
+ "2018-01-01 00:00:00"));
+ ghc.setDivisible(this.divisible);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
+ }
+ for (String[] historyEntry : this.historyEntries) {
+ ghc.addHistoryEntry(DateTimeHelper.parse(historyEntry[0] + ":00"),
+ DateTimeHelper.parse(historyEntry[1] + ":00"),
+ Double.parseDouble(historyEntry[2]));
+ }
+ Map<String, GraphHistory> compiledGraphHistories =
+ ghc.compileGraphHistories();
+ String message = this.testDescription + "; "
+ + new Gson().toJson(compiledGraphHistories);
+ assertEquals(message, this.expectedGraphs, compiledGraphHistories.size());
+ if (null != this.expectedGraphName) {
+ GraphHistory gh = compiledGraphHistories.get(this.expectedGraphName);
+ assertNotNull(message, gh);
+ if (null != this.expectedFirst) {
+ assertEquals(message, DateTimeHelper.parse(this.expectedFirst + ":00"),
+ gh.getFirst());
+ }
+ if (null != this.expectedLast) {
+ assertEquals(message, DateTimeHelper.parse(this.expectedLast + ":00"),
+ gh.getLast());
+ }
+ if (null != this.expectedInterval) {
+ assertEquals(message, this.expectedInterval, gh.getInterval());
+ }
+ if (null != this.expectedFactor) {
+ assertEquals(message, this.expectedFactor, gh.getFactor(), 0.01);
+ }
+ if (null != this.expectedCount) {
+ assertEquals(message, this.expectedCount, gh.getCount());
+ }
+ if (null != this.expectedValues) {
+ assertEquals(message, Arrays.asList(this.expectedValues),
+ gh.getValues());
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
index b1ba2ed..030d100 100644
--- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
@@ -248,7 +248,7 @@ public class UptimeDocumentWriterTest {
UptimeDocument.class, GABELMOO_FINGERPRINT);
this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00",
"2014-03-16 14:00:00", 2,
- Arrays.asList(new Integer[] { 499, 249 }));
+ Arrays.asList(new Integer[] { 999, 499 }));
}
@Test
@@ -256,8 +256,8 @@ public class UptimeDocumentWriterTest {
/* This relay was running for exactly 11 days and 23 hours over 2 years ago.
* This time period exactly matches 100% of a data point interval of 10 days
* plus a tiny bit less than 20% of the next data point interval. */
- this.addStatusOneWeekSample("r 2012-03-05-00 287\n",
- "r 2012-03-05-00 287\n");
+ this.addStatusOneWeekSample("r 2012-03-01-00 287\n",
+ "r 2012-03-01-00 287\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
writer.writeDocuments();
@@ -274,8 +274,8 @@ public class UptimeDocumentWriterTest {
/* This relay was running for exactly 12 days over 2 years ago. This time
* period exactly matches 100% of a data point interval of 10 days plus 20%
* of the next data point interval. */
- this.addStatusOneWeekSample("r 2012-03-05-00 288\n",
- "r 2012-03-05-00 288\n");
+ this.addStatusOneWeekSample("r 2012-03-01-00 288\n",
+ "r 2012-03-01-00 288\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
writer.writeDocuments();
@@ -283,8 +283,8 @@ public class UptimeDocumentWriterTest {
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
- this.assertFiveYearGraph(document, 1, "2012-03-10 00:00:00",
- "2012-03-20 00:00:00", 2, Arrays.asList(new Integer[] { 999, 999 }));
+ this.assertFiveYearGraph(document, 1, "2012-03-06 00:00:00",
+ "2012-03-16 00:00:00", 2, Arrays.asList(new Integer[] { 999, 999 }));
}
}
1
0

[onionoo/master] Use exact periods for graphs covering months/years.
by karsten@torproject.org 07 Feb '18
by karsten@torproject.org 07 Feb '18
07 Feb '18
commit 5cefee7b9de2a746ccaff42c6a0adb27cac8578a
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Jan 15 15:20:02 2018 +0100
Use exact periods for graphs covering months/years.
---
.../onionoo/writer/BandwidthDocumentWriter.java | 32 ++++++++++++++--------
.../onionoo/writer/ClientsDocumentWriter.java | 29 ++++++++++++--------
.../onionoo/writer/UptimeDocumentWriter.java | 29 ++++++++++++--------
.../onionoo/writer/WeightsDocumentWriter.java | 29 ++++++++++++--------
4 files changed, 74 insertions(+), 45 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
index f1c5041..2f27271 100644
--- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
@@ -15,6 +15,9 @@ import org.torproject.onionoo.docs.UpdateStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -79,13 +82,13 @@ public class BandwidthDocumentWriter implements DocumentWriter {
"1_year",
"5_years" };
- private long[] graphIntervals = new long[] {
- DateTimeHelper.THREE_DAYS,
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
+ private Period[] graphIntervals = new Period[] {
+ Period.ofDays(3),
+ Period.ofWeeks(1),
+ Period.ofMonths(1),
+ Period.ofMonths(3),
+ Period.ofYears(1),
+ Period.ofYears(5) };
private long[] dataPointIntervals = new long[] {
DateTimeHelper.FIFTEEN_MINUTES,
@@ -100,12 +103,15 @@ public class BandwidthDocumentWriter implements DocumentWriter {
Map<String, GraphHistory> graphs = new LinkedHashMap<>();
for (int i = 0; i < this.graphIntervals.length; i++) {
String graphName = this.graphNames[i];
- long graphInterval = this.graphIntervals[i];
+ Period graphInterval = this.graphIntervals[i];
long dataPointInterval = this.dataPointIntervals[i];
List<Long> dataPoints = new ArrayList<>();
long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
/ dataPointInterval) * dataPointInterval;
- long graphStartMillis = graphEndMillis - graphInterval;
+ long graphStartMillis = LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphInterval)
+ .toEpochSecond(ZoneOffset.UTC) * 1000L;
long intervalStartMillis = graphStartMillis;
long totalMillis = 0L;
long totalBandwidth = 0L;
@@ -159,9 +165,11 @@ public class BandwidthDocumentWriter implements DocumentWriter {
}
long firstDataPointMillis = graphStartMillis + firstNonNullIndex
* dataPointInterval + dataPointInterval / 2L;
- if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >=
- ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval)
- * dataPointInterval - graphIntervals[i - 1]) {
+ if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphIntervals[i - 1])
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) {
+
/* Skip bandwidth history object, because it doesn't contain
* anything new that wasn't already contained in the last
* bandwidth history object(s). Unless we did not include any of
diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
index 81168f5..4eca33a 100644
--- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
@@ -17,6 +17,9 @@ import org.torproject.onionoo.util.FormattingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -94,12 +97,12 @@ public class ClientsDocumentWriter implements DocumentWriter {
"1_year",
"5_years" };
- private long[] graphIntervals = new long[] {
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
+ private Period[] graphIntervals = new Period[] {
+ Period.ofWeeks(1),
+ Period.ofMonths(1),
+ Period.ofMonths(3),
+ Period.ofYears(1),
+ Period.ofYears(5) };
private long[] dataPointIntervals = new long[] {
DateTimeHelper.ONE_DAY,
@@ -129,13 +132,16 @@ public class ClientsDocumentWriter implements DocumentWriter {
private GraphHistory compileClientsHistory(
int graphIntervalIndex, SortedSet<ClientsHistory> history,
long lastSeenMillis) {
- long graphInterval = this.graphIntervals[graphIntervalIndex];
+ Period graphInterval = this.graphIntervals[graphIntervalIndex];
long dataPointInterval =
this.dataPointIntervals[graphIntervalIndex];
List<Double> dataPoints = new ArrayList<>();
long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
/ dataPointInterval) * dataPointInterval;
- long graphStartMillis = graphEndMillis - graphInterval;
+ long graphStartMillis = LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphInterval)
+ .toEpochSecond(ZoneOffset.UTC) * 1000L;
long intervalStartMillis = graphStartMillis;
long millis = 0L;
double responses = 0.0;
@@ -182,9 +188,10 @@ public class ClientsDocumentWriter implements DocumentWriter {
}
long firstDataPointMillis = graphStartMillis + firstNonNullIndex
* dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >=
- ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval)
- * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) {
+ if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphIntervals[graphIntervalIndex - 1])
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) {
/* Skip clients history object, because it doesn't contain
* anything new that wasn't already contained in the last
* clients history object(s). */
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index 2aee7a1..d1e2003 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -17,6 +17,9 @@ import org.torproject.onionoo.util.FormattingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -88,12 +91,12 @@ public class UptimeDocumentWriter implements DocumentWriter {
"1_year",
"5_years" };
- private long[] graphIntervals = new long[] {
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
+ private Period[] graphIntervals = new Period[] {
+ Period.ofWeeks(1),
+ Period.ofMonths(1),
+ Period.ofMonths(3),
+ Period.ofYears(1),
+ Period.ofYears(5) };
private long[] dataPointIntervals = new long[] {
DateTimeHelper.ONE_HOUR,
@@ -152,7 +155,7 @@ public class UptimeDocumentWriter implements DocumentWriter {
boolean relay, SortedSet<UptimeHistory> history,
SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis,
String flag) {
- long graphInterval = this.graphIntervals[graphIntervalIndex];
+ Period graphInterval = this.graphIntervals[graphIntervalIndex];
long dataPointInterval =
this.dataPointIntervals[graphIntervalIndex];
int dataPointIntervalHours = (int) (dataPointInterval
@@ -160,7 +163,10 @@ public class UptimeDocumentWriter implements DocumentWriter {
List<Integer> uptimeDataPoints = new ArrayList<>();
long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
/ dataPointInterval) * dataPointInterval;
- long graphStartMillis = graphEndMillis - graphInterval;
+ long graphStartMillis = LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphInterval)
+ .toEpochSecond(ZoneOffset.UTC) * 1000L;
long intervalStartMillis = graphStartMillis;
int uptimeHours = 0;
long firstStatusStartMillis = -1L;
@@ -285,9 +291,10 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
long firstDataPointMillis = graphStartMillis + firstNonNullIndex
* dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >=
- ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval)
- * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) {
+ if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphIntervals[graphIntervalIndex - 1])
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) {
/* Skip uptime history object, because it doesn't contain
* anything new that wasn't already contained in the last
* uptime history object(s). */
diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
index 10d8a94..f4e2c3a 100644
--- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
@@ -15,6 +15,9 @@ import org.torproject.onionoo.docs.WeightsStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -68,12 +71,12 @@ public class WeightsDocumentWriter implements DocumentWriter {
"1_year",
"5_years" };
- private long[] graphIntervals = new long[] {
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
+ private Period[] graphIntervals = new Period[] {
+ Period.ofWeeks(1),
+ Period.ofMonths(1),
+ Period.ofMonths(3),
+ Period.ofYears(1),
+ Period.ofYears(5) };
private long[] dataPointIntervals = new long[] {
DateTimeHelper.ONE_HOUR,
@@ -118,13 +121,16 @@ public class WeightsDocumentWriter implements DocumentWriter {
private GraphHistory compileWeightsHistory(int graphTypeIndex,
int graphIntervalIndex, SortedMap<long[], double[]> history,
long lastSeenMillis) {
- long graphInterval = this.graphIntervals[graphIntervalIndex];
+ Period graphInterval = this.graphIntervals[graphIntervalIndex];
long dataPointInterval =
this.dataPointIntervals[graphIntervalIndex];
List<Double> dataPoints = new ArrayList<>();
long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
/ dataPointInterval) * dataPointInterval;
- long graphStartMillis = graphEndMillis - graphInterval;
+ long graphStartMillis = LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphInterval)
+ .toEpochSecond(ZoneOffset.UTC) * 1000L;
long intervalStartMillis = graphStartMillis;
long totalMillis = 0L;
double totalWeightTimesMillis = 0.0;
@@ -175,9 +181,10 @@ public class WeightsDocumentWriter implements DocumentWriter {
}
long firstDataPointMillis = graphStartMillis + firstNonNullIndex
* dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >=
- ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval)
- * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) {
+ if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphIntervals[graphIntervalIndex - 1])
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) {
/* Skip weights history object, because it doesn't contain
* anything new that wasn't already contained in the last
* weights history object(s). */
1
0

07 Feb '18
commit 203d5d885d4155ed719c5168430ba237bad734aa
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Jan 15 11:11:20 2018 +0100
Fix comparison in writing uptime documents.
In the spec we write: "Contained graph history objects may contain
null values if less than 20% of network statuses have been processed
for a given time period."
However, in the code we checked less than or equal. We should fix
that.
---
.../onionoo/writer/UptimeDocumentWriter.java | 12 +++---
.../onionoo/writer/UptimeDocumentWriterTest.java | 44 +++++++++++++++++++++-
2 files changed, 49 insertions(+), 7 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index 96537ef..f739b9e 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -225,8 +225,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
while (hist.getStartMillis() >= intervalStartMillis
+ dataPointInterval) {
- statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
- ? statusHours : -1);
+ statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
+ ? -1 : statusHours);
statusHours = -1;
intervalStartMillis += dataPointInterval;
}
@@ -238,8 +238,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
- Math.max(Math.max(hist.getStartMillis(),
firstStatusStartMillis), intervalStartMillis))
/ DateTimeHelper.ONE_HOUR);
- statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
- ? statusHours : -1);
+ statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
+ ? -1 : statusHours);
statusHours = -1;
intervalStartMillis += dataPointInterval;
}
@@ -251,8 +251,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
intervalStartMillis)) / DateTimeHelper.ONE_HOUR);
}
if (statusHours > 0) {
- statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
- ? statusHours : -1);
+ statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
+ ? -1 : statusHours);
}
List<Double> dataPoints = new ArrayList<>();
for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size();
diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
index 3504509..b1ba2ed 100644
--- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
@@ -77,6 +77,12 @@ public class UptimeDocumentWriterTest {
(int) (FOUR_HOURS / ONE_SECOND), count, values);
}
+ private void assertFiveYearGraph(UptimeDocument document, int graphs,
+ String first, String last, int count, List<Integer> values) {
+ this.assertGraph(document, graphs, "5_years", first, last,
+ (int) (DateTimeHelper.TEN_DAYS / ONE_SECOND), count, values);
+ }
+
private void assertGraph(UptimeDocument document, int graphs,
String graphName, String first, String last, int interval,
int count, List<Integer> values) {
@@ -93,7 +99,7 @@ public class UptimeDocumentWriterTest {
(int) history.getInterval());
assertEquals("Factor should be 1.0 / 999.0.", 1.0 / 999.0,
(double) history.getFactor(), 0.01);
- assertEquals("There should be one data point per hour.", count,
+ assertEquals("There should be " + count + " data points.", count,
(int) history.getCount());
assertEquals("Count should be the same as the number of values.",
count, history.getValues().size());
@@ -244,5 +250,41 @@ public class UptimeDocumentWriterTest {
"2014-03-16 14:00:00", 2,
Arrays.asList(new Integer[] { 499, 249 }));
}
+
+ @Test
+ public void testFiveYearsLessThan20Percent() {
+ /* This relay was running for exactly 11 days and 23 hours over 2 years ago.
+ * This time period exactly matches 100% of a data point interval of 10 days
+ * plus a tiny bit less than 20% of the next data point interval. */
+ this.addStatusOneWeekSample("r 2012-03-05-00 287\n",
+ "r 2012-03-05-00 287\n");
+ UptimeDocumentWriter writer = new UptimeDocumentWriter();
+ DescriptorSourceFactory.getDescriptorSource().readDescriptors();
+ writer.writeDocuments();
+ assertEquals("Should write exactly one document.", 1,
+ this.documentStore.getPerformedStoreOperations());
+ UptimeDocument document = this.documentStore.getDocument(
+ UptimeDocument.class, GABELMOO_FINGERPRINT);
+ assertEquals("Should not contain any graph.", 0,
+ document.getUptime().size());
+ }
+
+ @Test
+ public void testFiveYearsAtLeast20Percent() {
+ /* This relay was running for exactly 12 days over 2 years ago. This time
+ * period exactly matches 100% of a data point interval of 10 days plus 20%
+ * of the next data point interval. */
+ this.addStatusOneWeekSample("r 2012-03-05-00 288\n",
+ "r 2012-03-05-00 288\n");
+ UptimeDocumentWriter writer = new UptimeDocumentWriter();
+ DescriptorSourceFactory.getDescriptorSource().readDescriptors();
+ writer.writeDocuments();
+ assertEquals("Should write exactly one document.", 1,
+ this.documentStore.getPerformedStoreOperations());
+ UptimeDocument document = this.documentStore.getDocument(
+ UptimeDocument.class, GABELMOO_FINGERPRINT);
+ this.assertFiveYearGraph(document, 1, "2012-03-10 00:00:00",
+ "2012-03-20 00:00:00", 2, Arrays.asList(new Integer[] { 999, 999 }));
+ }
}
1
0
commit 372ddc7b7e74ccb3c3cf8f870419ec239585c9fd
Author: iwakeh <iwakeh(a)torproject.org>
Date: Wed Jan 17 19:29:12 2018 +0000
Use constants consistently.
Avoid multiple definitions of date-time related constants.
---
src/main/java/org/torproject/onionoo/server/NodeIndexer.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index 1a85a35..69baa53 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -3,6 +3,9 @@
package org.torproject.onionoo.server;
+import static org.torproject.onionoo.docs.DateTimeHelper.ONE_DAY;
+import static org.torproject.onionoo.docs.DateTimeHelper.ONE_MINUTE;
+
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.SummaryDocument;
@@ -97,10 +100,6 @@ public class NodeIndexer implements ServletContextListener, Runnable {
}
}
- private static final long ONE_MINUTE = 60L * 1000L;
-
- private static final long ONE_DAY = 24L * 60L * ONE_MINUTE;
-
@Override
public void run() {
try {
1
0

07 Feb '18
commit 3a50a4635b5799c5e8f6e56969976a35c56fce86
Author: iwakeh <iwakeh(a)torproject.org>
Date: Wed Jan 17 19:29:13 2018 +0000
Make old code comply to new standard.
---
src/main/java/org/torproject/onionoo/server/ResponseBuilder.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
index dbe911f..80ee223 100644
--- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
+++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
@@ -113,8 +113,8 @@ public class ResponseBuilder {
}
public void buildResponse(PrintWriter pw) {
- writeRelays(orderedRelays, pw);
- writeBridges(orderedBridges, pw);
+ writeRelays(this.orderedRelays, pw);
+ writeBridges(this.orderedBridges, pw);
}
private int charsWritten = 0;
1
0

[onionoo/master] Use the same last-seen time for writing documents.
by karsten@torproject.org 07 Feb '18
by karsten@torproject.org 07 Feb '18
07 Feb '18
commit 5fbc7fd5d6e6510e9e02f18bda8dec0e6bd2c8d1
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Fri Feb 2 15:27:27 2018 +0100
Use the same last-seen time for writing documents.
Previously, we'd have used the last-seen time of the relay/bridge that
we're writing a document for. But that doesn't make much sense,
because we don't have to provide, e.g., a 1 week graph for a relay
that has been offline for years. This change is going to shrink the
out/ directory to a similar size as it was when we used current system
time for writing documents.
---
.../onionoo/writer/BandwidthDocumentWriter.java | 20 ++++++----------
.../onionoo/writer/ClientsDocumentWriter.java | 16 ++++---------
.../onionoo/writer/DetailsDocumentWriter.java | 2 +-
.../torproject/onionoo/writer/DocumentWriter.java | 2 +-
.../onionoo/writer/DocumentWriterRunner.java | 18 +++++++++++++-
.../onionoo/writer/SummaryDocumentWriter.java | 2 +-
.../onionoo/writer/UptimeDocumentWriter.java | 12 ++++------
.../onionoo/writer/WeightsDocumentWriter.java | 9 +------
.../writer/BandwidthDocumentWriterTest.java | 9 ++-----
.../onionoo/writer/DetailsDocumentWriterTest.java | 8 +++----
.../onionoo/writer/UptimeDocumentWriterTest.java | 28 ++++++++++------------
11 files changed, 55 insertions(+), 71 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
index 71595e2..7089c06 100644
--- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
@@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.GraphHistory;
-import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UpdateStatus;
import org.slf4j.Logger;
@@ -32,7 +31,7 @@ public class BandwidthDocumentWriter implements DocumentWriter {
}
@Override
- public void writeDocuments() {
+ public void writeDocuments(long lastSeenMillis) {
UpdateStatus updateStatus = this.documentStore.retrieve(
UpdateStatus.class, true);
long updatedMillis = updateStatus != null
@@ -40,18 +39,13 @@ public class BandwidthDocumentWriter implements DocumentWriter {
SortedSet<String> updateBandwidthDocuments = this.documentStore.list(
BandwidthStatus.class, updatedMillis);
for (String fingerprint : updateBandwidthDocuments) {
- NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
- true, fingerprint);
- if (null == nodeStatus) {
- continue;
- }
BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
BandwidthStatus.class, true, fingerprint);
if (bandwidthStatus == null) {
continue;
}
BandwidthDocument bandwidthDocument = this.compileBandwidthDocument(
- fingerprint, nodeStatus, bandwidthStatus);
+ fingerprint, lastSeenMillis, bandwidthStatus);
this.documentStore.store(bandwidthDocument, fingerprint);
}
log.info("Wrote bandwidth document files");
@@ -59,13 +53,13 @@ public class BandwidthDocumentWriter implements DocumentWriter {
private BandwidthDocument compileBandwidthDocument(String fingerprint,
- NodeStatus nodeStatus, BandwidthStatus bandwidthStatus) {
+ long lastSeenMillis, BandwidthStatus bandwidthStatus) {
BandwidthDocument bandwidthDocument = new BandwidthDocument();
bandwidthDocument.setFingerprint(fingerprint);
- bandwidthDocument.setWriteHistory(this.compileGraphType(
- nodeStatus.getLastSeenMillis(), bandwidthStatus.getWriteHistory()));
- bandwidthDocument.setReadHistory(this.compileGraphType(
- nodeStatus.getLastSeenMillis(), bandwidthStatus.getReadHistory()));
+ bandwidthDocument.setWriteHistory(this.compileGraphType(lastSeenMillis,
+ bandwidthStatus.getWriteHistory()));
+ bandwidthDocument.setReadHistory(this.compileGraphType(lastSeenMillis,
+ bandwidthStatus.getReadHistory()));
return bandwidthDocument;
}
diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
index aba45cf..7ed2048 100644
--- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
@@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.ClientsStatus;
import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
-import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UpdateStatus;
import org.torproject.onionoo.util.FormattingUtils;
@@ -56,7 +55,7 @@ public class ClientsDocumentWriter implements DocumentWriter {
private int writtenDocuments = 0;
@Override
- public void writeDocuments() {
+ public void writeDocuments(long lastSeenMillis) {
UpdateStatus updateStatus = this.documentStore.retrieve(
UpdateStatus.class, true);
long updatedMillis = updateStatus != null
@@ -64,11 +63,6 @@ public class ClientsDocumentWriter implements DocumentWriter {
SortedSet<String> updateDocuments = this.documentStore.list(
ClientsStatus.class, updatedMillis);
for (String hashedFingerprint : updateDocuments) {
- NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
- true, hashedFingerprint);
- if (null == nodeStatus) {
- continue;
- }
ClientsStatus clientsStatus = this.documentStore.retrieve(
ClientsStatus.class, true, hashedFingerprint);
if (clientsStatus == null) {
@@ -76,7 +70,7 @@ public class ClientsDocumentWriter implements DocumentWriter {
}
SortedSet<ClientsHistory> history = clientsStatus.getHistory();
ClientsDocument clientsDocument = this.compileClientsDocument(
- hashedFingerprint, nodeStatus, history);
+ hashedFingerprint, lastSeenMillis, history);
this.documentStore.store(clientsDocument, hashedFingerprint);
this.writtenDocuments++;
}
@@ -105,11 +99,11 @@ public class ClientsDocumentWriter implements DocumentWriter {
DateTimeHelper.TEN_DAYS };
private ClientsDocument compileClientsDocument(String hashedFingerprint,
- NodeStatus nodeStatus, SortedSet<ClientsHistory> history) {
+ long lastSeenMillis, SortedSet<ClientsHistory> history) {
ClientsDocument clientsDocument = new ClientsDocument();
clientsDocument.setFingerprint(hashedFingerprint);
- GraphHistoryCompiler ghc = new GraphHistoryCompiler(
- nodeStatus.getLastSeenMillis() + DateTimeHelper.ONE_HOUR);
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(lastSeenMillis
+ + DateTimeHelper.ONE_HOUR);
ghc.setThreshold(2L);
for (int i = 0; i < this.graphIntervals.length; i++) {
ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java
index bb81ca9..7ffaa07 100644
--- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java
@@ -32,7 +32,7 @@ public class DetailsDocumentWriter implements DocumentWriter {
}
@Override
- public void writeDocuments() {
+ public void writeDocuments(long lastSeenMillis) {
UpdateStatus updateStatus = this.documentStore.retrieve(
UpdateStatus.class, true);
long updatedMillis = updateStatus != null
diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java
index 618a8e7..359de6f 100644
--- a/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java
@@ -5,7 +5,7 @@ package org.torproject.onionoo.writer;
public interface DocumentWriter {
- public abstract void writeDocuments();
+ public abstract void writeDocuments(long lastSeenMillis);
public abstract String getStatsString();
}
diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java
index dc3462c..968ab72 100644
--- a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java
+++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java
@@ -3,6 +3,10 @@
package org.torproject.onionoo.writer;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,10 +32,22 @@ public class DocumentWriterRunner {
/** Lets each configured document writer write its documents. */
public void writeDocuments() {
+ long lastSeenMillis = retrieveLastSeenMillis();
for (DocumentWriter dw : this.documentWriters) {
log.debug("Writing " + dw.getClass().getSimpleName());
- dw.writeDocuments();
+ dw.writeDocuments(lastSeenMillis);
+ }
+ }
+
+ private long retrieveLastSeenMillis() {
+ DocumentStore documentStore = DocumentStoreFactory.getDocumentStore();
+ long lastSeenMillis = -1L;
+ for (String fingerprint : documentStore.list(NodeStatus.class)) {
+ NodeStatus nodeStatus = documentStore.retrieve(
+ NodeStatus.class, true, fingerprint);
+ lastSeenMillis = Math.max(lastSeenMillis, nodeStatus.getLastSeenMillis());
}
+ return lastSeenMillis;
}
/** Logs statistics of all configured document writers. */
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
index 813be1b..74ef31c 100644
--- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -33,7 +33,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
private int deletedDocuments = 0;
@Override
- public void writeDocuments() {
+ public void writeDocuments(long lastSeenAllRelaysAndBridgesMillis) {
long relaysLastValidAfterMillis = -1L;
long bridgesLastPublishedMillis = -1L;
for (String fingerprint : this.documentStore.list(NodeStatus.class)) {
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index 12ba8fa..db09e3d 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -7,7 +7,6 @@ import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.GraphHistory;
-import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UpdateStatus;
import org.torproject.onionoo.docs.UptimeDocument;
import org.torproject.onionoo.docs.UptimeHistory;
@@ -37,7 +36,7 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
@Override
- public void writeDocuments() {
+ public void writeDocuments(long lastSeenMillis) {
UptimeStatus uptimeStatus = this.documentStore.retrieve(
UptimeStatus.class, true);
if (uptimeStatus == null) {
@@ -51,20 +50,18 @@ public class UptimeDocumentWriter implements DocumentWriter {
SortedSet<String> updatedUptimeStatuses = this.documentStore.list(
UptimeStatus.class, updatedMillis);
for (String fingerprint : updatedUptimeStatuses) {
- this.updateDocument(fingerprint, uptimeStatus);
+ this.updateDocument(fingerprint, lastSeenMillis, uptimeStatus);
}
log.info("Wrote uptime document files");
}
private int writtenDocuments = 0;
- private void updateDocument(String fingerprint,
+ private void updateDocument(String fingerprint, long lastSeenMillis,
UptimeStatus knownStatuses) {
- NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
- true, fingerprint);
UptimeStatus uptimeStatus = this.documentStore.retrieve(
UptimeStatus.class, true, fingerprint);
- if (null != nodeStatus && null != uptimeStatus) {
+ if (null != uptimeStatus) {
boolean relay = uptimeStatus.getBridgeHistory().isEmpty();
SortedSet<UptimeHistory> history = relay
? uptimeStatus.getRelayHistory()
@@ -72,7 +69,6 @@ public class UptimeDocumentWriter implements DocumentWriter {
SortedSet<UptimeHistory> knownStatusesHistory = relay
? knownStatuses.getRelayHistory()
: knownStatuses.getBridgeHistory();
- long lastSeenMillis = nodeStatus.getLastSeenMillis();
UptimeDocument uptimeDocument = this.compileUptimeDocument(relay,
fingerprint, history, knownStatusesHistory, lastSeenMillis);
this.documentStore.store(uptimeDocument, fingerprint);
diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
index b34a9e6..2d2239b 100644
--- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
@@ -7,7 +7,6 @@ import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.GraphHistory;
-import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UpdateStatus;
import org.torproject.onionoo.docs.WeightsDocument;
import org.torproject.onionoo.docs.WeightsStatus;
@@ -32,7 +31,7 @@ public class WeightsDocumentWriter implements DocumentWriter {
}
@Override
- public void writeDocuments() {
+ public void writeDocuments(long lastSeenMillis) {
UpdateStatus updateStatus = this.documentStore.retrieve(
UpdateStatus.class, true);
long updatedMillis = updateStatus != null
@@ -40,18 +39,12 @@ public class WeightsDocumentWriter implements DocumentWriter {
SortedSet<String> updateWeightsDocuments = this.documentStore.list(
WeightsStatus.class, updatedMillis);
for (String fingerprint : updateWeightsDocuments) {
- NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
- true, fingerprint);
- if (null == nodeStatus) {
- continue;
- }
WeightsStatus weightsStatus = this.documentStore.retrieve(
WeightsStatus.class, true, fingerprint);
if (weightsStatus == null) {
continue;
}
SortedMap<long[], double[]> history = weightsStatus.getHistory();
- long lastSeenMillis = nodeStatus.getLastSeenMillis();
WeightsDocument weightsDocument = this.compileWeightsDocument(
fingerprint, history, lastSeenMillis);
this.documentStore.store(weightsDocument, fingerprint);
diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
index fa76699..947eba9 100644
--- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
@@ -12,7 +12,6 @@ import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.DummyDocumentStore;
import org.torproject.onionoo.docs.GraphHistory;
-import org.torproject.onionoo.docs.NodeStatus;
import org.junit.Before;
import org.junit.Test;
@@ -57,17 +56,13 @@ public class BandwidthDocumentWriterTest {
+ "r " + future + "-08-06 17:31:45 " + future + "-08-06 21:31:45 0\n"
+ "r " + future + "-08-06 21:31:45 " + future + "-08-07 01:31:45 0\n"
+ "r " + future + "-08-07 01:31:45 " + future + "-08-07 05:31:45 0\n";
- NodeStatus nodeStatus = new NodeStatus(ibibUnc0Fingerprint);
- nodeStatus.setLastSeenMillis(DateTimeHelper.parse(
- yesterday + " 12:00:00"));
- this.documentStore.addDocument(nodeStatus, ibibUnc0Fingerprint);
BandwidthStatus status = new BandwidthStatus();
status.setFromDocumentString(documentString);
this.documentStore.addDocument(status, ibibUnc0Fingerprint);
BandwidthDocumentWriter writer = new BandwidthDocumentWriter();
- writer.writeDocuments();
+ writer.writeDocuments(DateTimeHelper.parse(yesterday + " 12:00:00"));
assertEquals(1, this.documentStore.getPerformedListOperations());
- assertEquals(3, this.documentStore.getPerformedRetrieveOperations());
+ assertEquals(2, this.documentStore.getPerformedRetrieveOperations());
assertEquals(1, this.documentStore.getPerformedStoreOperations());
BandwidthDocument document = this.documentStore.getDocument(
BandwidthDocument.class, ibibUnc0Fingerprint);
diff --git a/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java
index 3356a9c..d96af71 100644
--- a/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java
@@ -28,7 +28,7 @@ public class DetailsDocumentWriterTest {
@Test
public void testNoDetailsStatuses() {
DetailsDocumentWriter writer = new DetailsDocumentWriter();
- writer.writeDocuments();
+ writer.writeDocuments(-1L);
assertEquals("Without statuses, no documents should be written.", 0,
this.documentStore.getPerformedStoreOperations());
}
@@ -48,7 +48,7 @@ public class DetailsDocumentWriterTest {
GABELMOO_OR_ADDRESS)));
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
DetailsDocumentWriter writer = new DetailsDocumentWriter();
- writer.writeDocuments();
+ writer.writeDocuments(-1L);
assertEquals("One document should be written.", 1,
this.documentStore.getPerformedStoreOperations());
DetailsDocument document = this.documentStore.getDocument(
@@ -67,7 +67,7 @@ public class DetailsDocumentWriterTest {
GABELMOO_OR_ADDRESS)));
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
DetailsDocumentWriter writer = new DetailsDocumentWriter();
- writer.writeDocuments();
+ writer.writeDocuments(-1L);
assertEquals("One document should be written.", 1,
this.documentStore.getPerformedStoreOperations());
DetailsDocument document = this.documentStore.getDocument(
@@ -85,7 +85,7 @@ public class DetailsDocumentWriterTest {
status.setAdvertisedOrAddresses(Arrays.asList(GABELMOO_OR_ADDRESS));
this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
DetailsDocumentWriter writer = new DetailsDocumentWriter();
- writer.writeDocuments();
+ writer.writeDocuments(-1L);
assertEquals("One document should be written.", 1,
this.documentStore.getPerformedStoreOperations());
DetailsDocument document = this.documentStore.getDocument(
diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
index 030d100..2915a04 100644
--- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
@@ -10,7 +10,6 @@ import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.DummyDocumentStore;
import org.torproject.onionoo.docs.GraphHistory;
-import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UptimeDocument;
import org.torproject.onionoo.docs.UptimeStatus;
import org.torproject.onionoo.updater.DescriptorSourceFactory;
@@ -37,7 +36,7 @@ public class UptimeDocumentWriterTest {
@Test
public void testNoStatuses() {
UptimeDocumentWriter writer = new UptimeDocumentWriter();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Without providing any data, nothing should be written "
+ "to disk.", 0,
this.documentStore.getPerformedStoreOperations());
@@ -50,9 +49,6 @@ public class UptimeDocumentWriterTest {
private void addStatusOneWeekSample(String allRelaysUptime,
String gabelmooUptime) {
- NodeStatus nodeStatus = new NodeStatus(GABELMOO_FINGERPRINT);
- nodeStatus.setLastSeenMillis(TEST_TIME);
- this.documentStore.addDocument(nodeStatus, GABELMOO_FINGERPRINT);
UptimeStatus status = new UptimeStatus();
status.setFromDocumentString(allRelaysUptime);
this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT);
@@ -119,7 +115,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-23-11 1\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -134,7 +130,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-23-10 2\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -149,7 +145,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-23-09 1\nr 2014-03-23-11 1\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -164,7 +160,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-23-09 1\nr 2014-03-23-11 1\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -180,7 +176,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-23-09 2\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -196,7 +192,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-16-13 168\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -211,7 +207,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-16-12 169\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -226,7 +222,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-16-11 5\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -241,7 +237,7 @@ public class UptimeDocumentWriterTest {
"r 2014-03-16-10 1\nr 2014-03-16-12 1\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -260,7 +256,7 @@ public class UptimeDocumentWriterTest {
"r 2012-03-01-00 287\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
@@ -278,7 +274,7 @@ public class UptimeDocumentWriterTest {
"r 2012-03-01-00 288\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
- writer.writeDocuments();
+ writer.writeDocuments(TEST_TIME);
assertEquals("Should write exactly one document.", 1,
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
1
0

[onionoo/master] Retain more detailed histories for a longer time.
by karsten@torproject.org 07 Feb '18
by karsten@torproject.org 07 Feb '18
07 Feb '18
commit d153681176ba492af159e1126f9dde7d133d2720
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Fri Feb 2 15:38:03 2018 +0100
Retain more detailed histories for a longer time.
Previously, we'd have compressed histories to a resolution of 48 hours
after 3 months. But we're now planning to introduce a 6 month graph
with a resolution of 24 hours. With this change we're retaining
detailed histories for three months longer.
Related to #24729.
---
src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java | 4 ++--
src/main/java/org/torproject/onionoo/docs/ClientsStatus.java | 2 +-
src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java | 2 ++
src/main/java/org/torproject/onionoo/docs/WeightsStatus.java | 4 ++--
4 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
index ba22dd4..252a018 100644
--- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
@@ -134,8 +134,8 @@ public class BandwidthStatus extends Document {
<= DateTimeHelper.ROUGHLY_ONE_MONTH) {
intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
} else if (lastSeenMillis - endMillis
- <= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
- intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
+ <= DateTimeHelper.ROUGHLY_SIX_MONTHS) {
+ intervalLengthMillis = DateTimeHelper.ONE_DAY;
} else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_ONE_YEAR) {
intervalLengthMillis = DateTimeHelper.TWO_DAYS;
diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
index a5bc8e8..2e11b50 100644
--- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
@@ -77,7 +77,7 @@ public class ClientsStatus extends Document {
for (ClientsHistory responses : uncompressedHistory) {
long intervalLengthMillis;
if (lastSeenMillis - responses.getEndMillis()
- <= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
+ <= DateTimeHelper.ROUGHLY_SIX_MONTHS) {
intervalLengthMillis = DateTimeHelper.ONE_DAY;
} else if (lastSeenMillis - responses.getEndMillis()
<= DateTimeHelper.ROUGHLY_ONE_YEAR) {
diff --git a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java
index c80514e..7966af8 100644
--- a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java
+++ b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java
@@ -57,6 +57,8 @@ public class DateTimeHelper {
public static final long ROUGHLY_THREE_MONTHS = 92L * ONE_DAY;
+ public static final long ROUGHLY_SIX_MONTHS = 183L * ONE_DAY;
+
public static final long ROUGHLY_ONE_YEAR = 366L * ONE_DAY;
public static final long ROUGHLY_FIVE_YEARS = 5L * ROUGHLY_ONE_YEAR;
diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
index 94efda6..c221fdd 100644
--- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
@@ -146,8 +146,8 @@ public class WeightsStatus extends Document {
<= DateTimeHelper.ROUGHLY_ONE_MONTH) {
intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
} else if (lastSeenMillis - endMillis
- <= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
- intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
+ <= DateTimeHelper.ROUGHLY_SIX_MONTHS) {
+ intervalLengthMillis = DateTimeHelper.ONE_DAY;
} else if (lastSeenMillis - endMillis
<= DateTimeHelper.ROUGHLY_ONE_YEAR) {
intervalLengthMillis = DateTimeHelper.TWO_DAYS;
1
0

[onionoo/master] Add a comment to avoid future 'detective work'.
by karsten@torproject.org 07 Feb '18
by karsten@torproject.org 07 Feb '18
07 Feb '18
commit 70e5f3f741fa899531e0ad42e3a702db6bfcb122
Author: iwakeh <iwakeh(a)torproject.org>
Date: Wed Jan 17 19:29:14 2018 +0000
Add a comment to avoid future 'detective work'.
---
src/main/java/org/torproject/onionoo/server/NodeIndexer.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index 69baa53..e95eda9 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -125,6 +125,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
indexerThread.interrupt();
}
+ /* specialTime is only used for testing, see ResourceServletTest */
private long specialTime = -1L;
private void indexNodeStatuses() {
1
0