[tor-commits] [onionoo/master] Compile graph histories in a single place.
karsten at torproject.org
karsten at torproject.org
Wed Feb 7 10:19:32 UTC 2018
commit 314feacaf18dae7e1f29861b39a74fc3df2d1330
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed Jan 17 20:29:11 2018 +0100
Compile graph histories in a single place.
---
.../org/torproject/onionoo/docs/UptimeHistory.java | 4 +
.../onionoo/writer/BandwidthDocumentWriter.java | 120 +-------
.../onionoo/writer/ClientsDocumentWriter.java | 130 +--------
.../onionoo/writer/GraphHistoryCompiler.java | 254 +++++++++++++++++
.../onionoo/writer/UptimeDocumentWriter.java | 312 +++++++--------------
.../onionoo/writer/WeightsDocumentWriter.java | 124 +-------
.../writer/BandwidthDocumentWriterTest.java | 4 +-
.../onionoo/writer/GraphHistoryCompilerTest.java | 203 ++++++++++++++
.../onionoo/writer/UptimeDocumentWriterTest.java | 14 +-
9 files changed, 606 insertions(+), 559 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
index 60e283f..f8cc116 100644
--- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
+++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
@@ -32,6 +32,10 @@ public class UptimeHistory implements Comparable<UptimeHistory> {
return this.uptimeHours;
}
+ public long getEndMillis() {
+ return this.startMillis + DateTimeHelper.ONE_HOUR * this.uptimeHours;
+ }
+
private SortedSet<String> flags;
public SortedSet<String> getFlags() {
diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
index 2f27271..71595e2 100644
--- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
@@ -15,12 +15,7 @@ import org.torproject.onionoo.docs.UpdateStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -100,114 +95,17 @@ public class BandwidthDocumentWriter implements DocumentWriter {
private Map<String, GraphHistory> compileGraphType(long lastSeenMillis,
SortedMap<Long, long[]> history) {
- Map<String, GraphHistory> graphs = new LinkedHashMap<>();
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ lastSeenMillis + DateTimeHelper.ONE_HOUR);
for (int i = 0; i < this.graphIntervals.length; i++) {
- String graphName = this.graphNames[i];
- Period graphInterval = this.graphIntervals[i];
- long dataPointInterval = this.dataPointIntervals[i];
- List<Long> dataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- long totalMillis = 0L;
- long totalBandwidth = 0L;
- for (long[] v : history.values()) {
- long endMillis = v[1];
- if (endMillis <= intervalStartMillis) {
- continue;
- } else if (endMillis > graphEndMillis) {
- break;
- }
- long startMillis = v[0];
- if (endMillis - startMillis > dataPointInterval) {
- /* This history interval is too long for this graph's data point
- * interval. Maybe the next graph will contain it, but not this
- * one. */
- continue;
- }
- while ((intervalStartMillis / dataPointInterval)
- != ((endMillis - 1L) / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
- / totalMillis);
- totalBandwidth = 0L;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- long bandwidth = v[2];
- totalBandwidth += bandwidth;
- totalMillis += (endMillis - startMillis);
- }
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
- / totalMillis);
- long maxValue = 1L;
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int j = 0; j < dataPoints.size(); j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = j;
- }
- lastNonNullIndex = j;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- continue;
- }
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[i - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
-
- /* Skip bandwidth history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * bandwidth history object(s). Unless we did not include any of
- * the previous bandwidth history objects for other reasons, in
- * which case we should include this one. */
- continue;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (j - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = j;
- }
- values.add(dataPoint < 0L ? null
- : (int) ((dataPoint * 999L) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- graphs.put(graphName, graphHistory);
- }
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
+ }
+ for (long[] v : history.values()) {
+ ghc.addHistoryEntry(v[0], v[1],
+ (double) (v[2] * DateTimeHelper.ONE_SECOND));
}
- return graphs;
+ return ghc.compileGraphHistories();
}
@Override
diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
index 4eca33a..aba45cf 100644
--- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java
@@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.ClientsStatus;
import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
-import org.torproject.onionoo.docs.GraphHistory;
import org.torproject.onionoo.docs.NodeStatus;
import org.torproject.onionoo.docs.UpdateStatus;
import org.torproject.onionoo.util.FormattingUtils;
@@ -17,13 +16,7 @@ import org.torproject.onionoo.util.FormattingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
import java.util.SortedSet;
/*
@@ -115,122 +108,19 @@ public class ClientsDocumentWriter implements DocumentWriter {
NodeStatus nodeStatus, SortedSet<ClientsHistory> history) {
ClientsDocument clientsDocument = new ClientsDocument();
clientsDocument.setFingerprint(hashedFingerprint);
- Map<String, GraphHistory> averageClients = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileClientsHistory(
- graphIntervalIndex, history, nodeStatus.getLastSeenMillis());
- if (graphHistory != null) {
- averageClients.put(graphName, graphHistory);
- }
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ nodeStatus.getLastSeenMillis() + DateTimeHelper.ONE_HOUR);
+ ghc.setThreshold(2L);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
}
- clientsDocument.setAverageClients(averageClients);
- return clientsDocument;
- }
-
- private GraphHistory compileClientsHistory(
- int graphIntervalIndex, SortedSet<ClientsHistory> history,
- long lastSeenMillis) {
- Period graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- List<Double> dataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- long millis = 0L;
- double responses = 0.0;
for (ClientsHistory hist : history) {
- if (hist.getEndMillis() <= intervalStartMillis) {
- continue;
- } else if (hist.getEndMillis() > graphEndMillis) {
- break;
- }
- while ((intervalStartMillis / dataPointInterval)
- != ((hist.getEndMillis() - 1L) / dataPointInterval)) {
- dataPoints.add(millis * 2L < dataPointInterval
- ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
- / (((double) millis) * 10.0));
- responses = 0.0;
- millis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- responses += hist.getTotalResponses();
- millis += (hist.getEndMillis() - hist.getStartMillis());
- }
- dataPoints.add(millis * 2L < dataPointInterval
- ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
- / (((double) millis) * 10.0));
- double maxValue = 0.0;
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- /* Not a single non-negative value in the data points. */
- return null;
- }
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[graphIntervalIndex - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
- /* Skip clients history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * clients history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex
- <= lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < 0.0 ? null :
- (int) ((dataPoint * 999.0) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- /* There are no two adjacent values in the data points that are
- * required to draw a line graph. */
- return null;
+ ghc.addHistoryEntry(hist.getStartMillis(), hist.getEndMillis(),
+ hist.getTotalResponses() * ((double) DateTimeHelper.ONE_DAY) / 10.0);
}
+ clientsDocument.setAverageClients(ghc.compileGraphHistories());
+ return clientsDocument;
}
@Override
diff --git a/src/main/java/org/torproject/onionoo/writer/GraphHistoryCompiler.java b/src/main/java/org/torproject/onionoo/writer/GraphHistoryCompiler.java
new file mode 100644
index 0000000..853ab32
--- /dev/null
+++ b/src/main/java/org/torproject/onionoo/writer/GraphHistoryCompiler.java
@@ -0,0 +1,254 @@
+/* Copyright 2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.onionoo.writer;
+
+import org.torproject.onionoo.docs.DateTimeHelper;
+import org.torproject.onionoo.docs.GraphHistory;
+
+import java.time.LocalDateTime;
+import java.time.Period;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Helper class to compile graph histories. */
+public class GraphHistoryCompiler {
+
+ private long graphsEndMillis;
+
+ /**
+ * Instantiates a new graph history compiler with the provided end time for
+ * all compiled graphs.
+ *
+ * @param graphsEndMillis End time for all compiled graphs.
+ */
+ GraphHistoryCompiler(long graphsEndMillis) {
+ this.graphsEndMillis = graphsEndMillis;
+ }
+
+ private boolean divisible = false;
+
+ /**
+ * Set whether history elements are divisible in the sense that they may be
+ * longer than one data point; this is the case for uptime intervals where
+ * uptime is equally distributed over potentially many data point intervals,
+ * but it's not the case for bandwidth/weights/clients intervals where
+ * observations are given for fixed-size reporting intervals. */
+ void setDivisible(boolean divisible) {
+ this.divisible = divisible;
+ }
+
+ private long threshold = 5;
+
+ /**
+ * Set the threshold (given as reciprocal value) of available history entries
+ * for any given data points below which the data point will be counted as
+ * null (missing); default is 5 for 1/5 = 20%.
+ */
+ void setThreshold(long threshold) {
+ this.threshold = threshold;
+ }
+
+ private List<String> graphNames = new ArrayList<>();
+
+ private List<Period> graphIntervals = new ArrayList<>();
+
+ private List<Long> dataPointIntervals = new ArrayList<>();
+
+ /**
+ * Add a graph type with the given graph name, graph interval, and data point
+ * interval.
+ *
+ * @param graphName Graph name, like "1_week".
+ * @param graphInterval Graph interval, like Period.ofWeeks(1).
+ * @param dataPointInterval Data point interval, like 1 hour in milliseconds.
+ */
+ void addGraphType(String graphName, Period graphInterval,
+ Long dataPointInterval) {
+ this.graphNames.add(graphName);
+ this.graphIntervals.add(graphInterval);
+ this.dataPointIntervals.add(dataPointInterval);
+ }
+
+ private Map<long[], Double> history = new LinkedHashMap<>();
+
+ /**
+ * Add a history entry with given start and end time and value.
+ *
+ * @param startMillis Start time in milliseconds.
+ * @param endMillis End time in milliseconds.
+ * @param value History entry value.
+ */
+ void addHistoryEntry(long startMillis, long endMillis, double value) {
+ this.history.put(new long[] { startMillis, endMillis }, value);
+ }
+
+ /**
+ * Compile graph histories from the history entries provided earlier.
+ *
+ * @return Map with graph names as keys and GraphHistory instances as values.
+ */
+ Map<String, GraphHistory> compileGraphHistories() {
+ Map<String, GraphHistory> graphs = new LinkedHashMap<>();
+ for (int graphIntervalIndex = 0;
+ graphIntervalIndex < this.graphIntervals.size();
+ graphIntervalIndex++) {
+
+ /* Look up graph name, graph interval, and data point interval from the
+ * graph type details provided earlier. */
+ final String graphName = this.graphNames.get(graphIntervalIndex);
+ Period graphInterval = this.graphIntervals.get(graphIntervalIndex);
+ long dataPointInterval = this.dataPointIntervals.get(graphIntervalIndex);
+
+ /* Determine graph end time as the end time for all graphs, rounded down
+ * to the last full data point interval. */
+ long graphEndMillis = (this.graphsEndMillis / dataPointInterval)
+ * dataPointInterval;
+
+ /* Determine graph start time as graph end time minus graph interval,
+ * rounded down to the last full data point interval. */
+ long graphStartMillis = ((LocalDateTime
+ .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(graphInterval)
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) / dataPointInterval)
+ * dataPointInterval;
+
+ /* Keep input for graph values in two arrays, one for values * millis,
+ * another one for millis. */
+ int dataPoints = (int) ((graphEndMillis - graphStartMillis)
+ / dataPointInterval);
+ double[] totalValues = new double[dataPoints];
+ long[] totalMillis = new long[dataPoints];
+
+ /* Iterate over all history entries and see which ones we need for this
+ * graph. */
+ for (Map.Entry<long[], Double> h : this.history.entrySet()) {
+ long startMillis = h.getKey()[0];
+ long endMillis = h.getKey()[1];
+ double value = h.getValue();
+
+ /* If a history entry ends before this graph starts or starts before
+ * this graph ends, skip it. */
+ if (endMillis <= graphStartMillis || startMillis >= graphEndMillis) {
+ continue;
+ }
+
+ /* If history entries are not divisible and this entry is longer than
+ * the data point interval, skip it. Maybe the next graph will contain
+ * it, but not this one. */
+ if (!this.divisible && endMillis - startMillis > dataPointInterval) {
+ continue;
+ }
+
+ /* Iterate over all data points that this history element falls into.
+ * Even if history entries are not divisible, we may have to split it
+ * over two data points, because reported statistics rarely align with
+ * our data point intervals. And if history entries are divisible, we
+ * may have to split them over many data points. */
+ for (long intervalStartMillis = startMillis;
+ intervalStartMillis < endMillis;
+ intervalStartMillis = ((intervalStartMillis + dataPointInterval)
+ / dataPointInterval) * dataPointInterval) {
+
+ /* Determine the data point that this (partial) history entry falls
+ * into. And if it's out of bounds, skip it. */
+ int dataPointIndex = (int) ((intervalStartMillis - graphStartMillis)
+ / dataPointInterval);
+ if (dataPointIndex < 0 || dataPointIndex >= dataPoints) {
+ continue;
+ }
+
+ /* Determine the interval end, which may be the end of the data point
+ * or the end of the history entry, whichever comes first. Then add
+ * values and millis to the data point. */
+ long intervalEndMillis = Math.min(endMillis, ((intervalStartMillis
+ + dataPointInterval) / dataPointInterval) * dataPointInterval);
+ long millis = intervalEndMillis - intervalStartMillis;
+ totalValues[dataPointIndex] += (value * (double) millis)
+ / (double) (endMillis - startMillis);
+ totalMillis[dataPointIndex] += millis;
+ }
+ }
+
+ /* Go through the previously compiled data points and extract some pieces
+ * that will be relevant for deciding whether to include this graph and
+ * for adding meta data to the GraphHistory object. */
+ double maxValue = 0.0;
+ int firstNonNullIndex = -1;
+ int lastNonNullIndex = -1;
+ boolean foundTwoAdjacentDataPoints = false;
+ for (int dataPointIndex = 0, previousNonNullIndex = -2;
+ dataPointIndex < dataPoints; dataPointIndex++) {
+
+ /* Only consider data points containing values for at least the given
+ * threshold of time (20% by default). If so, record first and last
+ * data point containing data, whether there exist two adjacent data
+ * points containing data, and determine the maximum value. */
+ if (totalMillis[dataPointIndex] * this.threshold >= dataPointInterval) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ maxValue = Math.max(maxValue, totalValues[dataPointIndex]
+ / totalMillis[dataPointIndex]);
+ }
+ }
+
+ /* If there are not at least two adjacent data points containing data,
+ * skip the graph. */
+ if (!foundTwoAdjacentDataPoints) {
+ continue;
+ }
+
+ /* Calculate the timestamp of the first data point containing data. */
+ long firstDataPointMillis = graphStartMillis + firstNonNullIndex
+ * dataPointInterval + dataPointInterval / 2L;
+
+ /* If the graph doesn't contain anything new that wasn't already contained
+ * in previously compiled graphs, skip this graph. */
+ if (graphIntervalIndex > 0 && !graphs.isEmpty()
+ && firstDataPointMillis >= LocalDateTime.ofEpochSecond(
+ graphEndMillis / 1000L, 0, ZoneOffset.UTC)
+ .minus(this.graphIntervals.get(graphIntervalIndex - 1))
+ .toEpochSecond(ZoneOffset.UTC) * 1000L) {
+ continue;
+ }
+
+ /* Put together the list of values that will go into the graph. */
+ List<Integer> values = new ArrayList<>();
+ for (int dataPointIndex = firstNonNullIndex;
+ dataPointIndex <= lastNonNullIndex; dataPointIndex++) {
+ if (totalMillis[dataPointIndex] * this.threshold >= dataPointInterval) {
+ values.add((int) ((totalValues[dataPointIndex] * 999.0)
+ / (maxValue * totalMillis[dataPointIndex])));
+ } else {
+ values.add(null);
+ }
+ }
+
+ /* Put together a GraphHistory object and add it to the map under the
+ * given graph name. */
+ GraphHistory graphHistory = new GraphHistory();
+ graphHistory.setFirst(firstDataPointMillis);
+ graphHistory.setLast(firstDataPointMillis + (lastNonNullIndex
+ - firstNonNullIndex) * dataPointInterval);
+ graphHistory.setInterval((int) (dataPointInterval
+ / DateTimeHelper.ONE_SECOND));
+ graphHistory.setFactor(maxValue / 999.0);
+ graphHistory.setCount(lastNonNullIndex - firstNonNullIndex + 1);
+ graphHistory.setValues(values);
+ graphs.put(graphName, graphHistory);
+ }
+
+ /* We're done. Return the map of compiled graphs. */
+ return graphs;
+ }
+}
+
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index d1e2003..12ba8fa 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -17,12 +17,8 @@ import org.torproject.onionoo.util.FormattingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
+import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -110,18 +106,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis) {
UptimeDocument uptimeDocument = new UptimeDocument();
uptimeDocument.setFingerprint(fingerprint);
- Map<String, GraphHistory> uptime = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileUptimeHistory(
- graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis,
- null);
- if (graphHistory != null) {
- uptime.put(graphName, graphHistory);
- }
- }
- uptimeDocument.setUptime(uptime);
+ uptimeDocument.setUptime(this.compileUptimeHistory(relay, history,
+ knownStatuses, lastSeenMillis, null));
SortedMap<String, Map<String, GraphHistory>> flags = new TreeMap<>();
SortedSet<String> allFlags = new TreeSet<>();
for (UptimeHistory hist : history) {
@@ -130,17 +116,8 @@ public class UptimeDocumentWriter implements DocumentWriter {
}
}
for (String flag : allFlags) {
- Map<String, GraphHistory> graphsForFlags = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileUptimeHistory(
- graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis,
- flag);
- if (graphHistory != null) {
- graphsForFlags.put(graphName, graphHistory);
- }
- }
+ Map<String, GraphHistory> graphsForFlags = this.compileUptimeHistory(
+ relay, history, knownStatuses, lastSeenMillis, flag);
if (!graphsForFlags.isEmpty()) {
flags.put(flag, graphsForFlags);
}
@@ -151,187 +128,116 @@ public class UptimeDocumentWriter implements DocumentWriter {
return uptimeDocument;
}
- private GraphHistory compileUptimeHistory(int graphIntervalIndex,
- boolean relay, SortedSet<UptimeHistory> history,
- SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis,
- String flag) {
- Period graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- int dataPointIntervalHours = (int) (dataPointInterval
- / DateTimeHelper.ONE_HOUR);
- List<Integer> uptimeDataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- int uptimeHours = 0;
- long firstStatusStartMillis = -1L;
- for (UptimeHistory hist : history) {
- if (hist.isRelay() != relay
- || (flag != null && (hist.getFlags() == null
- || !hist.getFlags().contains(flag)))) {
- continue;
- }
- if (firstStatusStartMillis < 0L) {
- firstStatusStartMillis = hist.getStartMillis();
- }
- long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
- * hist.getUptimeHours();
- if (histEndMillis <= intervalStartMillis) {
- continue;
- } else if (histEndMillis > graphEndMillis) {
- histEndMillis = graphEndMillis;
- }
- while (hist.getStartMillis() >= intervalStartMillis
- + dataPointInterval) {
- if (firstStatusStartMillis < intervalStartMillis
- + dataPointInterval) {
- uptimeDataPoints.add(uptimeHours);
- } else {
- uptimeDataPoints.add(-1);
- }
- uptimeHours = 0;
- intervalStartMillis += dataPointInterval;
- }
- while (histEndMillis >= intervalStartMillis + dataPointInterval) {
- uptimeHours += (int) ((intervalStartMillis + dataPointInterval
- - Math.max(hist.getStartMillis(), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- uptimeDataPoints.add(uptimeHours);
- uptimeHours = 0;
- intervalStartMillis += dataPointInterval;
- }
- uptimeHours += (int) ((histEndMillis - Math.max(
- hist.getStartMillis(), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- }
- uptimeDataPoints.add(uptimeHours);
- List<Integer> statusDataPoints = new ArrayList<>();
- intervalStartMillis = graphStartMillis;
- int statusHours = -1;
- for (UptimeHistory hist : knownStatuses) {
- if (hist.getStartMillis() >= graphEndMillis) {
- break;
- }
- if (hist.isRelay() != relay
- || (flag != null && (hist.getFlags() == null
- || !hist.getFlags().contains(flag)))) {
- continue;
- }
- long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
- * hist.getUptimeHours();
- if (histEndMillis <= intervalStartMillis) {
- continue;
- } else if (histEndMillis > graphEndMillis) {
- histEndMillis = graphEndMillis;
- }
- while (hist.getStartMillis() >= intervalStartMillis
- + dataPointInterval) {
- statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
- ? -1 : statusHours);
- statusHours = -1;
- intervalStartMillis += dataPointInterval;
- }
- while (histEndMillis >= intervalStartMillis + dataPointInterval) {
- if (statusHours < 0) {
- statusHours = 0;
- }
- statusHours += (int) ((intervalStartMillis + dataPointInterval
- - Math.max(Math.max(hist.getStartMillis(),
- firstStatusStartMillis), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
- ? -1 : statusHours);
- statusHours = -1;
- intervalStartMillis += dataPointInterval;
- }
- if (statusHours < 0) {
- statusHours = 0;
- }
- statusHours += (int) ((histEndMillis - Math.max(Math.max(
- hist.getStartMillis(), firstStatusStartMillis),
- intervalStartMillis)) / DateTimeHelper.ONE_HOUR);
- }
- if (statusHours > 0) {
- statusDataPoints.add(statusHours * 5 < dataPointIntervalHours
- ? -1 : statusHours);
- }
- List<Double> dataPoints = new ArrayList<>();
- for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size();
- dataPointIndex++) {
- if (dataPointIndex >= uptimeDataPoints.size()) {
- dataPoints.add(0.0);
- } else if (uptimeDataPoints.get(dataPointIndex) >= 0
- && statusDataPoints.get(dataPointIndex) > 0) {
- dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex))
- / ((double) statusDataPoints.get(dataPointIndex)));
- } else {
- dataPoints.add(-1.0);
- }
- }
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- }
- }
- if (firstNonNullIndex < 0) {
- /* Not a single non-negative value in the data points. */
+ private Map<String, GraphHistory> compileUptimeHistory(boolean relay,
+ SortedSet<UptimeHistory> history, SortedSet<UptimeHistory> knownStatuses,
+ long lastSeenMillis, String flag) {
+
+ /* Extracting history entries for compiling GraphHistory objects is a bit
+ * harder than for the other document types. The reason is that we have to
+ * combine (A) uptime history of all relays/bridges and (B) uptime history
+ * of the relay/bridge that we're writing the document for. We're going to
+ * refer to A and B below, to simplify descriptions a bit. */
+
+ /* If there are either no A entries or no B entries, we can't compile
+ * graphs. */
+ if (history.isEmpty() || knownStatuses.isEmpty()) {
return null;
}
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[graphIntervalIndex - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
- /* Skip uptime history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * uptime history object(s). */
+
+ /* Initialize the graph history compiler, and tell it that history entries
+ * are divisible. This is different from the other history writers. */
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ lastSeenMillis + DateTimeHelper.ONE_HOUR);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
+ }
+ ghc.setDivisible(true);
+
+ /* The general idea for extracting history entries and passing them to the
+ * graph history compiler is to iterate over A entries one by one and keep
+ * an Iterator for B entries to move forward as "time" proceeds. */
+ Iterator<UptimeHistory> historyIterator = history.iterator();
+ UptimeHistory hist;
+ do {
+ hist = historyIterator.hasNext() ? historyIterator.next() : null;
+ } while (null != hist && (hist.isRelay() != relay
+ || (null != flag && (null == hist.getFlags()
+ || !hist.getFlags().contains(flag)))));
+
+ /* If there is not at least one B entry, we can't compile graphs. */
+ if (null == hist) {
return null;
}
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(1.0 / 999.0);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex
- <= lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
+
+ for (UptimeHistory statuses : knownStatuses) {
+
+ /* If this A entry contains uptime information that we're not interested
+ * in, skip it. */
+ if (statuses.isRelay() != relay
+ || (null != flag && (null == statuses.getFlags()
+ || !statuses.getFlags().contains(flag)))) {
+ continue;
}
- values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0)));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- /* There are no two adjacent values in the data points that are
- * required to draw a line graph. */
- return null;
+
+ /* The "current" time is the time that we're currently considering as part
+ * of the A entry. It starts out as the interval start, but as we may
+ * consider multiple B entries, it may proceed. The loop ends when
+ * "current" time has reached the end of the considered A entry. */
+ long currentTimeMillis = statuses.getStartMillis();
+ do {
+ if (null == hist) {
+
+ /* There is no B entry left, which means that the relay/bridge was
+ * offline from "current" time to the end of the A entry. */
+ ghc.addHistoryEntry(currentTimeMillis, statuses.getEndMillis(),0.0);
+ currentTimeMillis = statuses.getEndMillis();
+ } else if (statuses.getEndMillis() <= hist.getStartMillis()) {
+
+ /* This A entry ends before the B entry starts. If there was an
+ * earlier B entry, count this time as offline time. */
+ if (history.first().getStartMillis() <= currentTimeMillis) {
+ ghc.addHistoryEntry(currentTimeMillis, statuses.getEndMillis(),
+ 0.0);
+ }
+ currentTimeMillis = statuses.getEndMillis();
+ } else {
+
+ /* A and B entries overlap. First, if there's time between "current"
+ * time and the time when B starts, possibly count that as offline
+ * time, but only if the relay was around earlier. */
+ if (currentTimeMillis < hist.getStartMillis()) {
+ if (history.first().getStartMillis() <= currentTimeMillis) {
+ ghc.addHistoryEntry(currentTimeMillis, hist.getStartMillis(),
+ 0.0);
+ }
+ currentTimeMillis = hist.getStartMillis();
+ }
+
+ /* Now handle the actually overlapping part. First determine when the
+ * overlap ends, then add a history entry with the number of uptime
+ * milliseconds as value. */
+ long overlapEndMillis = Math.min(statuses.getEndMillis(),
+ hist.getEndMillis());
+ ghc.addHistoryEntry(currentTimeMillis, overlapEndMillis,
+ overlapEndMillis - currentTimeMillis);
+ currentTimeMillis = overlapEndMillis;
+
+ /* If A ends after B, move on to the next B entry. */
+ if (statuses.getEndMillis() >= hist.getEndMillis()) {
+ do {
+ hist = historyIterator.hasNext() ? historyIterator.next() : null;
+ } while (null != hist && (hist.isRelay() != relay
+ || (null != flag && (null == hist.getFlags()
+ || !hist.getFlags().contains(flag)))));
+ }
+ }
+ } while (currentTimeMillis < statuses.getEndMillis());
}
+
+ /* Now that the graph history compiler knows all relevant history, ask it to
+ * compile graphs for us, and return them. */
+ return ghc.compileGraphHistories();
}
@Override
diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
index f4e2c3a..b34a9e6 100644
--- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java
@@ -15,12 +15,7 @@ import org.torproject.onionoo.docs.WeightsStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.LocalDateTime;
import java.time.Period;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -105,125 +100,22 @@ public class WeightsDocumentWriter implements DocumentWriter {
private Map<String, GraphHistory> compileGraphType(
SortedMap<long[], double[]> history, long lastSeenMillis,
int graphTypeIndex) {
- Map<String, GraphHistory> graphs = new LinkedHashMap<>();
- for (int graphIntervalIndex = 0; graphIntervalIndex
- < this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileWeightsHistory(
- graphTypeIndex, graphIntervalIndex, history, lastSeenMillis);
- if (graphHistory != null) {
- graphs.put(graphName, graphHistory);
- }
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(
+ lastSeenMillis + DateTimeHelper.ONE_HOUR);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
}
- return graphs;
- }
-
- private GraphHistory compileWeightsHistory(int graphTypeIndex,
- int graphIntervalIndex, SortedMap<long[], double[]> history,
- long lastSeenMillis) {
- Period graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- List<Double> dataPoints = new ArrayList<>();
- long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR)
- / dataPointInterval) * dataPointInterval;
- long graphStartMillis = LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphInterval)
- .toEpochSecond(ZoneOffset.UTC) * 1000L;
- long intervalStartMillis = graphStartMillis;
- long totalMillis = 0L;
- double totalWeightTimesMillis = 0.0;
for (Map.Entry<long[], double[]> e : history.entrySet()) {
long startMillis = e.getKey()[0];
long endMillis = e.getKey()[1];
double weight = e.getValue()[graphTypeIndex];
- if (endMillis <= intervalStartMillis) {
- continue;
- } else if (endMillis > graphEndMillis) {
- break;
- }
- while ((intervalStartMillis / dataPointInterval)
- != ((endMillis - 1L) / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
- totalWeightTimesMillis = 0.0;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
if (weight >= 0.0) {
- totalWeightTimesMillis += weight
- * ((double) (endMillis - startMillis));
- totalMillis += (endMillis - startMillis);
+ ghc.addHistoryEntry(startMillis, endMillis,
+ weight * ((double) (endMillis - startMillis)));
}
}
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
- double maxValue = 0.0;
- int firstNonNullIndex = -1;
- int lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- /* Not a single non-negative value in the data points. */
- return null;
- }
- long firstDataPointMillis = graphStartMillis + firstNonNullIndex
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime
- .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC)
- .minus(graphIntervals[graphIntervalIndex - 1])
- .toEpochSecond(ZoneOffset.UTC) * 1000L) {
- /* Skip weights history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * weights history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(firstDataPointMillis);
- graphHistory.setLast(lastDataPointMillis);
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex
- <= lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < 0.0 ? null :
- (int) ((dataPoint * 999.0) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- /* There are no two adjacent values in the data points that are
- * required to draw a line graph. */
- return null;
- }
+ return ghc.compileGraphHistories();
}
@Override
diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
index 324122c..fa76699 100644
--- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java
@@ -74,9 +74,9 @@ public class BandwidthDocumentWriterTest {
assertEquals(1, document.getReadHistory().size());
assertTrue(document.getReadHistory().containsKey("1_month"));
GraphHistory history = document.getReadHistory().get("1_month");
- assertEquals(DateTimeHelper.parse(dayBeforeYesterday + " 14:00:00"),
+ assertEquals(DateTimeHelper.parse(dayBeforeYesterday + " 10:00:00"),
history.getFirst());
- assertEquals(DateTimeHelper.parse(yesterday + " 02:00:00"),
+ assertEquals(DateTimeHelper.parse(dayBeforeYesterday + " 22:00:00"),
history.getLast());
assertEquals(DateTimeHelper.FOUR_HOURS / DateTimeHelper.ONE_SECOND,
(int) history.getInterval());
diff --git a/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java b/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
new file mode 100644
index 0000000..6d9c461
--- /dev/null
+++ b/src/test/java/org/torproject/onionoo/writer/GraphHistoryCompilerTest.java
@@ -0,0 +1,203 @@
+/* Copyright 2018 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.onionoo.writer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.torproject.onionoo.docs.DateTimeHelper;
+import org.torproject.onionoo.docs.GraphHistory;
+
+import com.google.gson.Gson;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.time.Period;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+ at RunWith(Parameterized.class)
+public class GraphHistoryCompilerTest {
+
+ /** Provide test data. */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { "Empty history",
+ false, new String[0][], 0, null, null, null, null, null, null,
+ null },
+ { "Single entry right before graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 23:00", "2018-01-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive entries right before graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 22:00", "2017-12-31 23:00", "1" },
+ new String[] { "2017-12-31 23:00", "2018-01-01 00:00", "1" }},
+ 1, "1_week", "2017-12-31 22:30", "2017-12-31 23:30", 3600, 0.001, 2,
+ new Integer[] { 999, 999 } },
+ { "Two non-consecutive entries towards graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 21:00", "2017-12-31 22:00", "1" },
+ new String[] { "2017-12-31 23:00", "2018-01-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive entries passing 1 week threshold",
+ false, new String[][] {
+ new String[] { "2017-12-24 23:00", "2017-12-25 00:00", "1" },
+ new String[] { "2017-12-25 00:00", "2017-12-25 01:00", "1" }},
+ 1, "1_month", "2017-12-24 22:00", "2017-12-25 02:00", 14400,
+ 2.7805583361138913E-10, 2, new Integer[] { 999, 999 } },
+ { "Two consecutive 1-hour entries over 1 week from graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-21 22:00", "2017-12-21 23:00", "1" },
+ new String[] { "2017-12-21 23:00", "2017-12-22 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive 4-hour entries over 1 week from graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-21 16:00", "2017-12-21 20:00", "1" },
+ new String[] { "2017-12-21 20:00", "2017-12-22 00:00", "1" }},
+ 1, "1_month", "2017-12-21 18:00", "2017-12-21 22:00", 14400, 0.001,
+ 2, new Integer[] { 999, 999 } },
+ { "Two consecutive 4-hour entries right before graphs end",
+ false, new String[][] {
+ new String[] { "2017-12-31 16:00", "2017-12-31 20:00", "1" },
+ new String[] { "2017-12-31 20:00", "2018-01-01 00:00", "1" }},
+ 1, "1_month", "2017-12-31 18:00", "2017-12-31 22:00", 14400, 0.001,
+ 2, new Integer[] { 999, 999 } },
+ { "Single 1-week divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-25 00:00", "2018-01-01 00:00", "1" }},
+ 1, "1_week", "2017-12-25 00:30", "2017-12-31 23:30", 3600, 0.001,
+ 168, null },
+ { "Single 1-week-and-1-hour divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-24 23:00", "2018-01-01 00:00", "1" }},
+ 2, "1_month", "2017-12-24 22:00", "2017-12-31 22:00", 14400, 0.001,
+ 43, null },
+ { "Single 66-minute divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-31 22:54", "2018-01-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Single 72-minute divisible entry right before graphs end",
+ true, new String[][] {
+ new String[] { "2017-12-31 22:48", "2018-01-01 00:00", "1" }},
+ 1, "1_week", "2017-12-31 22:30", "2017-12-31 23:30", 3600, 0.001,
+ 2, null },
+ { "Single 6-month divisible entry 6 years before graphs end",
+ true, new String[][] {
+ new String[] { "2012-01-01 00:00", "2012-07-01 00:00", "1" }},
+ 0, null, null, null, null, null, null, null },
+ { "Two consecutive 1-hour entries right after graphs end",
+ false, new String[][] {
+ new String[] { "2018-01-01 00:00", "2018-01-01 01:00", "1" },
+ new String[] { "2018-01-01 01:00", "2018-01-01 02:00", "1" }},
+ 0, null, null, null, null, null, null, null }
+ });
+ }
+
+ @Parameter
+ public String testDescription;
+
+ @Parameter(1)
+ public boolean divisible;
+
+ @Parameter(2)
+ public String[][] historyEntries;
+
+ @Parameter(3)
+ public int expectedGraphs;
+
+ @Parameter(4)
+ public String expectedGraphName;
+
+ @Parameter(5)
+ public String expectedFirst;
+
+ @Parameter(6)
+ public String expectedLast;
+
+ @Parameter(7)
+ public Integer expectedInterval;
+
+ @Parameter(8)
+ public Double expectedFactor;
+
+ @Parameter(9)
+ public Integer expectedCount;
+
+ @Parameter(10)
+ public Integer[] expectedValues;
+
+ private final String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private final Period[] graphIntervals = new Period[] {
+ Period.ofWeeks(1),
+ Period.ofMonths(1),
+ Period.ofMonths(3),
+ Period.ofYears(1),
+ Period.ofYears(5) };
+
+ private final long[] dataPointIntervals = new long[] {
+ DateTimeHelper.ONE_HOUR,
+ DateTimeHelper.FOUR_HOURS,
+ DateTimeHelper.TWELVE_HOURS,
+ DateTimeHelper.TWO_DAYS,
+ DateTimeHelper.TEN_DAYS };
+
+ @Test
+ public void test() {
+ GraphHistoryCompiler ghc = new GraphHistoryCompiler(DateTimeHelper.parse(
+ "2018-01-01 00:00:00"));
+ ghc.setDivisible(this.divisible);
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ ghc.addGraphType(this.graphNames[i], this.graphIntervals[i],
+ this.dataPointIntervals[i]);
+ }
+ for (String[] historyEntry : this.historyEntries) {
+ ghc.addHistoryEntry(DateTimeHelper.parse(historyEntry[0] + ":00"),
+ DateTimeHelper.parse(historyEntry[1] + ":00"),
+ Double.parseDouble(historyEntry[2]));
+ }
+ Map<String, GraphHistory> compiledGraphHistories =
+ ghc.compileGraphHistories();
+ String message = this.testDescription + "; "
+ + new Gson().toJson(compiledGraphHistories);
+ assertEquals(message, this.expectedGraphs, compiledGraphHistories.size());
+ if (null != this.expectedGraphName) {
+ GraphHistory gh = compiledGraphHistories.get(this.expectedGraphName);
+ assertNotNull(message, gh);
+ if (null != this.expectedFirst) {
+ assertEquals(message, DateTimeHelper.parse(this.expectedFirst + ":00"),
+ gh.getFirst());
+ }
+ if (null != this.expectedLast) {
+ assertEquals(message, DateTimeHelper.parse(this.expectedLast + ":00"),
+ gh.getLast());
+ }
+ if (null != this.expectedInterval) {
+ assertEquals(message, this.expectedInterval, gh.getInterval());
+ }
+ if (null != this.expectedFactor) {
+ assertEquals(message, this.expectedFactor, gh.getFactor(), 0.01);
+ }
+ if (null != this.expectedCount) {
+ assertEquals(message, this.expectedCount, gh.getCount());
+ }
+ if (null != this.expectedValues) {
+ assertEquals(message, Arrays.asList(this.expectedValues),
+ gh.getValues());
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
index b1ba2ed..030d100 100644
--- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
+++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java
@@ -248,7 +248,7 @@ public class UptimeDocumentWriterTest {
UptimeDocument.class, GABELMOO_FINGERPRINT);
this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00",
"2014-03-16 14:00:00", 2,
- Arrays.asList(new Integer[] { 499, 249 }));
+ Arrays.asList(new Integer[] { 999, 499 }));
}
@Test
@@ -256,8 +256,8 @@ public class UptimeDocumentWriterTest {
/* This relay was running for exactly 11 days and 23 hours over 2 years ago.
* This time period exactly matches 100% of a data point interval of 10 days
* plus a tiny bit less than 20% of the next data point interval. */
- this.addStatusOneWeekSample("r 2012-03-05-00 287\n",
- "r 2012-03-05-00 287\n");
+ this.addStatusOneWeekSample("r 2012-03-01-00 287\n",
+ "r 2012-03-01-00 287\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
writer.writeDocuments();
@@ -274,8 +274,8 @@ public class UptimeDocumentWriterTest {
/* This relay was running for exactly 12 days over 2 years ago. This time
* period exactly matches 100% of a data point interval of 10 days plus 20%
* of the next data point interval. */
- this.addStatusOneWeekSample("r 2012-03-05-00 288\n",
- "r 2012-03-05-00 288\n");
+ this.addStatusOneWeekSample("r 2012-03-01-00 288\n",
+ "r 2012-03-01-00 288\n");
UptimeDocumentWriter writer = new UptimeDocumentWriter();
DescriptorSourceFactory.getDescriptorSource().readDescriptors();
writer.writeDocuments();
@@ -283,8 +283,8 @@ public class UptimeDocumentWriterTest {
this.documentStore.getPerformedStoreOperations());
UptimeDocument document = this.documentStore.getDocument(
UptimeDocument.class, GABELMOO_FINGERPRINT);
- this.assertFiveYearGraph(document, 1, "2012-03-10 00:00:00",
- "2012-03-20 00:00:00", 2, Arrays.asList(new Integer[] { 999, 999 }));
+ this.assertFiveYearGraph(document, 1, "2012-03-06 00:00:00",
+ "2012-03-16 00:00:00", 2, Arrays.asList(new Integer[] { 999, 999 }));
}
}
More information about the tor-commits
mailing list