commit a7418b7c4d1b84ae1c1c062df847b013cb002e82 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Jan 9 11:48:25 2018 +0100 Don't rely on system time for writing histories. Rather than on system time we're now depending on the last time a relay or bridge was seen in a consensus or status to determine when history ends for this relay. This has the advantage of making the write step deterministic, and it produces the exact same graph intervals for the different documents of a given relay or bridge. A minor downside is that we're now depending on node statuses _and_ another status file in order to produce a history document. Should be okay. Define graph end as last full data point before the relay/bridge was last seen. Also make sure that graphs end at that defined graph end and do not continue just because there's more history available. This is mostly to exclude falsely-reported statistics. Implements #16513. --- CHANGELOG.md | 7 +++ .../onionoo/writer/BandwidthDocumentWriter.java | 38 ++++++++------- .../onionoo/writer/ClientsDocumentWriter.java | 34 ++++++++------ .../onionoo/writer/UptimeDocumentWriter.java | 44 +++++++++++------- .../onionoo/writer/WeightsDocumentWriter.java | 48 +++++++++++-------- .../writer/BandwidthDocumentWriterTest.java | 23 ++++----- .../onionoo/writer/UptimeDocumentWriterTest.java | 54 +++++----------------- 7 files changed, 127 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dad79f..1873443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version ?? + + * Medium changes + - Make writing of bandwidth, clients, uptime, and weights documents + independent of system time. + + # Changes in version 5.0-1.9.0 - 2017-12-20 * Medium changes diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index f0ab771..99a5a00 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -9,6 +9,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.slf4j.Logger; @@ -28,11 +29,8 @@ public class BandwidthDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public BandwidthDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } @Override @@ -44,13 +42,18 @@ public class BandwidthDocumentWriter implements DocumentWriter { SortedSet<String> updateBandwidthDocuments = this.documentStore.list( BandwidthStatus.class, updatedMillis); for (String fingerprint : updateBandwidthDocuments) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null == nodeStatus) { + continue; + } BandwidthStatus bandwidthStatus = this.documentStore.retrieve( BandwidthStatus.class, true, fingerprint); if (bandwidthStatus == null) { continue; } BandwidthDocument bandwidthDocument = this.compileBandwidthDocument( - fingerprint, bandwidthStatus); + fingerprint, nodeStatus, bandwidthStatus); this.documentStore.store(bandwidthDocument, fingerprint); } log.info("Wrote bandwidth document files"); @@ -58,13 +61,13 @@ public class BandwidthDocumentWriter implements DocumentWriter { private BandwidthDocument compileBandwidthDocument(String fingerprint, - BandwidthStatus bandwidthStatus) { + NodeStatus nodeStatus, BandwidthStatus bandwidthStatus) { BandwidthDocument bandwidthDocument = new BandwidthDocument(); bandwidthDocument.setFingerprint(fingerprint); bandwidthDocument.setWriteHistory(this.compileGraphType( - bandwidthStatus.getWriteHistory())); + nodeStatus.getLastSeenMillis(), bandwidthStatus.getWriteHistory())); bandwidthDocument.setReadHistory(this.compileGraphType( - bandwidthStatus.getReadHistory())); + nodeStatus.getLastSeenMillis(), bandwidthStatus.getReadHistory())); return bandwidthDocument; } @@ -92,7 +95,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { DateTimeHelper.TWO_DAYS, DateTimeHelper.TEN_DAYS }; - private Map<String, GraphHistory> compileGraphType( + private Map<String, GraphHistory> compileGraphType(long lastSeenMillis, SortedMap<Long, long[]> history) { Map<String, GraphHistory> graphs = new LinkedHashMap<>(); for (int i = 0; i < this.graphIntervals.length; i++) { @@ -100,19 +103,20 @@ public class BandwidthDocumentWriter implements DocumentWriter { long graphInterval = this.graphIntervals[i]; long dataPointInterval = this.dataPointIntervals[i]; List<Long> dataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; long totalMillis = 0L; long totalBandwidth = 0L; for (long[] v : history.values()) { long endMillis = v[1]; if (endMillis < intervalStartMillis) { continue; - } - long startMillis = v[0]; - if (startMillis > this.now) { + } else if (endMillis > graphEndMillis) { break; } + long startMillis = v[0]; if (endMillis - startMillis > dataPointInterval) { /* This history interval is too long for this graph's data point * interval. Maybe the next graph will contain it, but not this @@ -153,11 +157,11 @@ public class BandwidthDocumentWriter implements DocumentWriter { if (firstNonNullIndex < 0) { continue; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (i > 0 && !graphs.isEmpty() - && firstDataPointMillis >= this.now - graphIntervals[i - 1]) { + long firstDataPointMillis = graphStartMillis + firstNonNullIndex + * dataPointInterval + dataPointInterval / 2L; + if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[i - 1]) { /* Skip bandwidth history object, because it doesn't contain * anything new that wasn't already contained in the last * bandwidth history object(s). Unless we did not include any of diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index e74f078..834df9c 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.FormattingUtils; @@ -52,11 +53,8 @@ public class ClientsDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public ClientsDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } private int writtenDocuments = 0; @@ -70,6 +68,11 @@ public class ClientsDocumentWriter implements DocumentWriter { SortedSet<String> updateDocuments = this.documentStore.list( ClientsStatus.class, updatedMillis); for (String hashedFingerprint : updateDocuments) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, hashedFingerprint); + if (null == nodeStatus) { + continue; + } ClientsStatus clientsStatus = this.documentStore.retrieve( ClientsStatus.class, true, hashedFingerprint); if (clientsStatus == null) { @@ -77,7 +80,7 @@ public class ClientsDocumentWriter implements DocumentWriter { } SortedSet<ClientsHistory> history = clientsStatus.getHistory(); ClientsDocument clientsDocument = this.compileClientsDocument( - hashedFingerprint, history); + hashedFingerprint, nodeStatus, history); this.documentStore.store(clientsDocument, hashedFingerprint); this.writtenDocuments++; } @@ -106,7 +109,7 @@ public class ClientsDocumentWriter implements DocumentWriter { DateTimeHelper.TEN_DAYS }; private ClientsDocument compileClientsDocument(String hashedFingerprint, - SortedSet<ClientsHistory> history) { + NodeStatus nodeStatus, SortedSet<ClientsHistory> history) { ClientsDocument clientsDocument = new ClientsDocument(); clientsDocument.setFingerprint(hashedFingerprint); Map<String, GraphHistory> averageClients = new LinkedHashMap<>(); @@ -114,7 +117,7 @@ public class ClientsDocumentWriter implements DocumentWriter { < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileClientsHistory( - graphIntervalIndex, history); + graphIntervalIndex, history, nodeStatus.getLastSeenMillis()); if (graphHistory != null) { averageClients.put(graphName, graphHistory); } @@ -124,18 +127,23 @@ public class ClientsDocumentWriter implements DocumentWriter { } private GraphHistory compileClientsHistory( - int graphIntervalIndex, SortedSet<ClientsHistory> history) { + int graphIntervalIndex, SortedSet<ClientsHistory> history, + long lastSeenMillis) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; List<Double> dataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; long millis = 0L; double responses = 0.0; for (ClientsHistory hist : history) { if (hist.getEndMillis() < intervalStartMillis) { continue; + } else if (hist.getEndMillis() > graphEndMillis) { + break; } while ((intervalStartMillis / dataPointInterval) != (hist.getEndMillis() / dataPointInterval)) { @@ -172,11 +180,11 @@ public class ClientsDocumentWriter implements DocumentWriter { /* Not a single non-negative value in the data points. */ return null; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis - >= this.now - graphIntervals[graphIntervalIndex - 1]) { + long firstDataPointMillis = graphStartMillis + firstNonNullIndex + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { /* Skip clients history object, because it doesn't contain * anything new that wasn't already contained in the last * clients history object(s). */ diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index e28359b..96537ef 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -7,6 +7,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeHistory; @@ -32,11 +33,8 @@ public class UptimeDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public UptimeDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } @Override @@ -63,9 +61,11 @@ public class UptimeDocumentWriter implements DocumentWriter { private void updateDocument(String fingerprint, UptimeStatus knownStatuses) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true, fingerprint); - if (uptimeStatus != null) { + if (null != nodeStatus && null != uptimeStatus) { boolean relay = uptimeStatus.getBridgeHistory().isEmpty(); SortedSet<UptimeHistory> history = relay ? uptimeStatus.getRelayHistory() @@ -73,8 +73,9 @@ public class UptimeDocumentWriter implements DocumentWriter { SortedSet<UptimeHistory> knownStatusesHistory = relay ? knownStatuses.getRelayHistory() : knownStatuses.getBridgeHistory(); + long lastSeenMillis = nodeStatus.getLastSeenMillis(); UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, - fingerprint, history, knownStatusesHistory); + fingerprint, history, knownStatusesHistory, lastSeenMillis); this.documentStore.store(uptimeDocument, fingerprint); this.writtenDocuments++; } @@ -103,7 +104,7 @@ public class UptimeDocumentWriter implements DocumentWriter { private UptimeDocument compileUptimeDocument(boolean relay, String fingerprint, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { + SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis) { UptimeDocument uptimeDocument = new UptimeDocument(); uptimeDocument.setFingerprint(fingerprint); Map<String, GraphHistory> uptime = new LinkedHashMap<>(); @@ -111,7 +112,8 @@ public class UptimeDocumentWriter implements DocumentWriter { < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses, null); + graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis, + null); if (graphHistory != null) { uptime.put(graphName, graphHistory); } @@ -130,7 +132,8 @@ public class UptimeDocumentWriter implements DocumentWriter { < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses, flag); + graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis, + flag); if (graphHistory != null) { graphsForFlags.put(graphName, graphHistory); } @@ -147,15 +150,18 @@ public class UptimeDocumentWriter implements DocumentWriter { private GraphHistory compileUptimeHistory(int graphIntervalIndex, boolean relay, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses, String flag) { + SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis, + String flag) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; int dataPointIntervalHours = (int) (dataPointInterval / DateTimeHelper.ONE_HOUR); List<Integer> uptimeDataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; int uptimeHours = 0; long firstStatusStartMillis = -1L; for (UptimeHistory hist : history) { @@ -171,6 +177,8 @@ public class UptimeDocumentWriter implements DocumentWriter { * hist.getUptimeHours(); if (histEndMillis < intervalStartMillis) { continue; + } else if (histEndMillis > graphEndMillis) { + histEndMillis = graphEndMillis; } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { @@ -197,10 +205,12 @@ public class UptimeDocumentWriter implements DocumentWriter { } uptimeDataPoints.add(uptimeHours); List<Integer> statusDataPoints = new ArrayList<>(); - intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; + intervalStartMillis = graphStartMillis; int statusHours = -1; for (UptimeHistory hist : knownStatuses) { + if (hist.getStartMillis() >= graphEndMillis) { + break; + } if (hist.isRelay() != relay || (flag != null && (hist.getFlags() == null || !hist.getFlags().contains(flag)))) { @@ -210,6 +220,8 @@ public class UptimeDocumentWriter implements DocumentWriter { * hist.getUptimeHours(); if (histEndMillis < intervalStartMillis) { continue; + } else if (histEndMillis > graphEndMillis) { + histEndMillis = graphEndMillis; } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { @@ -271,11 +283,11 @@ public class UptimeDocumentWriter implements DocumentWriter { /* Not a single non-negative value in the data points. */ return null; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) + long firstDataPointMillis = graphStartMillis + firstNonNullIndex * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis - >= this.now - graphIntervals[graphIntervalIndex - 1]) { + if (graphIntervalIndex > 0 && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { /* Skip uptime history object, because it doesn't contain * anything new that wasn't already contained in the last * uptime history object(s). */ diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index 75626a3..ad54fea 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -7,6 +7,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.WeightsDocument; import org.torproject.onionoo.docs.WeightsStatus; @@ -28,11 +29,8 @@ public class WeightsDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public WeightsDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } @Override @@ -44,14 +42,20 @@ public class WeightsDocumentWriter implements DocumentWriter { SortedSet<String> updateWeightsDocuments = this.documentStore.list( WeightsStatus.class, updatedMillis); for (String fingerprint : updateWeightsDocuments) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null == nodeStatus) { + continue; + } WeightsStatus weightsStatus = this.documentStore.retrieve( WeightsStatus.class, true, fingerprint); if (weightsStatus == null) { continue; } SortedMap<long[], double[]> history = weightsStatus.getHistory(); + long lastSeenMillis = nodeStatus.getLastSeenMillis(); WeightsDocument weightsDocument = this.compileWeightsDocument( - fingerprint, history); + fingerprint, history, lastSeenMillis); this.documentStore.store(weightsDocument, fingerprint); } log.info("Wrote weights document files"); @@ -79,30 +83,31 @@ public class WeightsDocumentWriter implements DocumentWriter { DateTimeHelper.TEN_DAYS }; private WeightsDocument compileWeightsDocument(String fingerprint, - SortedMap<long[], double[]> history) { + SortedMap<long[], double[]> history, long lastSeenMillis) { WeightsDocument weightsDocument = new WeightsDocument(); weightsDocument.setFingerprint(fingerprint); weightsDocument.setConsensusWeightFraction( - this.compileGraphType(history, 1)); + this.compileGraphType(history, lastSeenMillis, 1)); weightsDocument.setGuardProbability( - this.compileGraphType(history, 2)); + this.compileGraphType(history, lastSeenMillis, 2)); weightsDocument.setMiddleProbability( - this.compileGraphType(history, 3)); + this.compileGraphType(history, lastSeenMillis, 3)); weightsDocument.setExitProbability( - this.compileGraphType(history, 4)); + this.compileGraphType(history, lastSeenMillis, 4)); weightsDocument.setConsensusWeight( - this.compileGraphType(history, 6)); + this.compileGraphType(history, lastSeenMillis, 6)); return weightsDocument; } private Map<String, GraphHistory> compileGraphType( - SortedMap<long[], double[]> history, int graphTypeIndex) { + SortedMap<long[], double[]> history, long lastSeenMillis, + int graphTypeIndex) { Map<String, GraphHistory> graphs = new LinkedHashMap<>(); for (int graphIntervalIndex = 0; graphIntervalIndex < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileWeightsHistory( - graphTypeIndex, graphIntervalIndex, history); + graphTypeIndex, graphIntervalIndex, history, lastSeenMillis); if (graphHistory != null) { graphs.put(graphName, graphHistory); } @@ -111,13 +116,16 @@ public class WeightsDocumentWriter implements DocumentWriter { } private GraphHistory compileWeightsHistory(int graphTypeIndex, - int graphIntervalIndex, SortedMap<long[], double[]> history) { + int graphIntervalIndex, SortedMap<long[], double[]> history, + long lastSeenMillis) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; List<Double> dataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; long totalMillis = 0L; double totalWeightTimesMillis = 0.0; for (Map.Entry<long[], double[]> e : history.entrySet()) { @@ -126,6 +134,8 @@ public class WeightsDocumentWriter implements DocumentWriter { double weight = e.getValue()[graphTypeIndex]; if (endMillis < intervalStartMillis) { continue; + } else if (endMillis > graphEndMillis) { + break; } while ((intervalStartMillis / dataPointInterval) != (endMillis / dataPointInterval)) { @@ -163,11 +173,11 @@ public class WeightsDocumentWriter implements DocumentWriter { /* Not a single non-negative value in the data points. */ return null; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis - >= this.now - graphIntervals[graphIntervalIndex - 1]) { + long firstDataPointMillis = graphStartMillis + firstNonNullIndex + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { /* Skip weights history object, because it doesn't contain * anything new that wasn't already contained in the last * weights history object(s). */ diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java index 3c74ae5..324122c 100644 --- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java @@ -12,8 +12,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.DummyDocumentStore; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DummyDescriptorSource; +import org.torproject.onionoo.docs.NodeStatus; import org.junit.Before; import org.junit.Test; @@ -23,14 +22,6 @@ import java.util.Date; public class BandwidthDocumentWriterTest { - private DummyDescriptorSource descriptorSource; - - @Before - public void createDummyDescriptorSource() { - this.descriptorSource = new DummyDescriptorSource(); - DescriptorSourceFactory.setDescriptorSource(this.descriptorSource); - } - private DummyDocumentStore documentStore; @Before @@ -41,7 +32,7 @@ public class BandwidthDocumentWriterTest { @Test public void testIgnoreFuture() { - BandwidthStatus status = new BandwidthStatus(); + String ibibUnc0Fingerprint = "7C0AA4E3B73E407E9F5FEB1912F8BE26D8AA124D"; String future = new SimpleDateFormat("yyyy") .format(new Date(System.currentTimeMillis() + DateTimeHelper.ROUGHLY_ONE_YEAR)); @@ -66,13 +57,17 @@ public class BandwidthDocumentWriterTest { + "r " + future + "-08-06 17:31:45 " + future + "-08-06 21:31:45 0\n" + "r " + future + "-08-06 21:31:45 " + future + "-08-07 01:31:45 0\n" + "r " + future + "-08-07 01:31:45 " + future + "-08-07 05:31:45 0\n"; - + NodeStatus nodeStatus = new NodeStatus(ibibUnc0Fingerprint); + nodeStatus.setLastSeenMillis(DateTimeHelper.parse( + yesterday + " 12:00:00")); + this.documentStore.addDocument(nodeStatus, ibibUnc0Fingerprint); + BandwidthStatus status = new BandwidthStatus(); status.setFromDocumentString(documentString); - String ibibUnc0Fingerprint = "7C0AA4E3B73E407E9F5FEB1912F8BE26D8AA124D"; this.documentStore.addDocument(status, ibibUnc0Fingerprint); BandwidthDocumentWriter writer = new BandwidthDocumentWriter(); - DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); + assertEquals(1, this.documentStore.getPerformedListOperations()); + assertEquals(3, this.documentStore.getPerformedRetrieveOperations()); assertEquals(1, this.documentStore.getPerformedStoreOperations()); BandwidthDocument document = this.documentStore.getDocument( BandwidthDocument.class, ibibUnc0Fingerprint); diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java index 5554bf0..3504509 100644 --- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java @@ -5,21 +5,19 @@ package org.torproject.onionoo.writer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.DummyDocumentStore; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeStatus; import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DummyDescriptorSource; import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.util.Arrays; import java.util.List; @@ -28,14 +26,6 @@ public class UptimeDocumentWriterTest { private static final long TEST_TIME = DateTimeHelper.parse( "2014-03-23 12:00:00"); - private DummyDescriptorSource descriptorSource; - - @Before - public void createDummyDescriptorSource() { - this.descriptorSource = new DummyDescriptorSource(); - DescriptorSourceFactory.setDescriptorSource(this.descriptorSource); - } - private DummyDocumentStore documentStore; @Before @@ -44,21 +34,9 @@ public class UptimeDocumentWriterTest { DocumentStoreFactory.setDocumentStore(this.documentStore); } - private void setTime(UptimeDocumentWriter udw) { - try { - Field nowField = udw.getClass() - .getDeclaredField("now"); - nowField.setAccessible(true); - nowField.set(udw, TEST_TIME); - } catch (Exception ex) { - fail("Cannot manipulate test-time. Failing all."); - } - } - @Test public void testNoStatuses() { UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); writer.writeDocuments(); assertEquals("Without providing any data, nothing should be written " + "to disk.", 0, @@ -72,6 +50,9 @@ public class UptimeDocumentWriterTest { private void addStatusOneWeekSample(String allRelaysUptime, String gabelmooUptime) { + NodeStatus nodeStatus = new NodeStatus(GABELMOO_FINGERPRINT); + nodeStatus.setLastSeenMillis(TEST_TIME); + this.documentStore.addDocument(nodeStatus, GABELMOO_FINGERPRINT); UptimeStatus status = new UptimeStatus(); status.setFromDocumentString(allRelaysUptime); this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT); @@ -131,7 +112,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-11 1\n", "r 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -147,7 +127,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-10 2\n", "r 2014-03-23-10 2\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -163,7 +142,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-09 1\nr 2014-03-23-11 1\n", "r 2014-03-23-09 1\nr 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -179,7 +157,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-09 3\n", "r 2014-03-23-09 1\nr 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -196,7 +173,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-09 3\n", "r 2014-03-23-09 2\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -210,36 +186,32 @@ public class UptimeDocumentWriterTest { @Test public void testOneWeekUptime() { - this.addStatusOneWeekSample("r 2014-03-16-12 168\n", - "r 2014-03-16-12 168\n"); + this.addStatusOneWeekSample("r 2014-03-16-13 168\n", + "r 2014-03-16-13 168\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( UptimeDocument.class, GABELMOO_FINGERPRINT); - this.assertOneWeekGraph(document, 1, "2014-03-16 12:30:00", - "2014-03-23 11:30:00", 168, null); + this.assertOneWeekGraph(document, 1, "2014-03-16 13:30:00", + "2014-03-23 12:30:00", 168, null); } @Test public void testOneWeekOneHourUptime() { - this.addStatusOneWeekSample("r 2014-03-16-11 169\n", - "r 2014-03-16-11 169\n"); + this.addStatusOneWeekSample("r 2014-03-16-12 169\n", + "r 2014-03-16-12 169\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( UptimeDocument.class, GABELMOO_FINGERPRINT); - this.assertOneWeekGraph(document, 2, "2014-03-16 12:30:00", - "2014-03-23 11:30:00", 168, null); - this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00", - "2014-03-23 10:00:00", 43, null); + this.assertOneWeekGraph(document, 1, "2014-03-16 13:30:00", + "2014-03-23 12:30:00", 168, null); } @Test @@ -247,7 +219,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-16-08 8\n", "r 2014-03-16-11 5\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -263,7 +234,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-16-08 8\n", "r 2014-03-16-10 1\nr 2014-03-16-12 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1,