February 2018 - tor-commits - lists.torproject.org

[onionoo/release] Bump version to 5.0-1.9.0-dev.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 7fc763699142a93fd75389da8bb3f4c7e9b667fd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 20 17:16:02 2017 +0100 Bump version to 5.0-1.9.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index d2db15d..7833807 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ <property name="implementation-title" value="Onionoo" /> <property name="onionoo.protocol.version" value="5.0"/> <property name="release.version" - value="${onionoo.protocol.version}-1.9.0"/> + value="${onionoo.protocol.version}-1.9.0-dev"/> <property name="metricslibversion" value="2.1.1"/> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile"

1 0

[onionoo/release] Don't rely on system time for writing histories.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit a7418b7c4d1b84ae1c1c062df847b013cb002e82 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 9 11:48:25 2018 +0100 Don't rely on system time for writing histories. Rather than on system time we're now depending on the last time a relay or bridge was seen in a consensus or status to determine when history ends for this relay. This has the advantage of making the write step deterministic, and it produces the exact same graph intervals for the different documents of a given relay or bridge. A minor downside is that we're now depending on node statuses _and_ another status file in order to produce a history document. Should be okay. Define graph end as last full data point before the relay/bridge was last seen. Also make sure that graphs end at that defined graph end and do not continue just because there's more history available. This is mostly to exclude falsely-reported statistics. Implements #16513. --- CHANGELOG.md | 7 +++ .../onionoo/writer/BandwidthDocumentWriter.java | 38 ++++++++------- .../onionoo/writer/ClientsDocumentWriter.java | 34 ++++++++------ .../onionoo/writer/UptimeDocumentWriter.java | 44 +++++++++++------- .../onionoo/writer/WeightsDocumentWriter.java | 48 +++++++++++-------- .../writer/BandwidthDocumentWriterTest.java | 23 ++++----- .../onionoo/writer/UptimeDocumentWriterTest.java | 54 +++++----------------- 7 files changed, 127 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dad79f..1873443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version ?? + + * Medium changes + - Make writing of bandwidth, clients, uptime, and weights documents + independent of system time. + + # Changes in version 5.0-1.9.0 - 2017-12-20 * Medium changes diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index f0ab771..99a5a00 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -9,6 +9,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.slf4j.Logger; @@ -28,11 +29,8 @@ public class BandwidthDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public BandwidthDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } @Override @@ -44,13 +42,18 @@ public class BandwidthDocumentWriter implements DocumentWriter { SortedSet<String> updateBandwidthDocuments = this.documentStore.list( BandwidthStatus.class, updatedMillis); for (String fingerprint : updateBandwidthDocuments) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null == nodeStatus) { + continue; + } BandwidthStatus bandwidthStatus = this.documentStore.retrieve( BandwidthStatus.class, true, fingerprint); if (bandwidthStatus == null) { continue; } BandwidthDocument bandwidthDocument = this.compileBandwidthDocument( - fingerprint, bandwidthStatus); + fingerprint, nodeStatus, bandwidthStatus); this.documentStore.store(bandwidthDocument, fingerprint); } log.info("Wrote bandwidth document files"); @@ -58,13 +61,13 @@ public class BandwidthDocumentWriter implements DocumentWriter { private BandwidthDocument compileBandwidthDocument(String fingerprint, - BandwidthStatus bandwidthStatus) { + NodeStatus nodeStatus, BandwidthStatus bandwidthStatus) { BandwidthDocument bandwidthDocument = new BandwidthDocument(); bandwidthDocument.setFingerprint(fingerprint); bandwidthDocument.setWriteHistory(this.compileGraphType( - bandwidthStatus.getWriteHistory())); + nodeStatus.getLastSeenMillis(), bandwidthStatus.getWriteHistory())); bandwidthDocument.setReadHistory(this.compileGraphType( - bandwidthStatus.getReadHistory())); + nodeStatus.getLastSeenMillis(), bandwidthStatus.getReadHistory())); return bandwidthDocument; } @@ -92,7 +95,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { DateTimeHelper.TWO_DAYS, DateTimeHelper.TEN_DAYS }; - private Map<String, GraphHistory> compileGraphType( + private Map<String, GraphHistory> compileGraphType(long lastSeenMillis, SortedMap<Long, long[]> history) { Map<String, GraphHistory> graphs = new LinkedHashMap<>(); for (int i = 0; i < this.graphIntervals.length; i++) { @@ -100,19 +103,20 @@ public class BandwidthDocumentWriter implements DocumentWriter { long graphInterval = this.graphIntervals[i]; long dataPointInterval = this.dataPointIntervals[i]; List<Long> dataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; long totalMillis = 0L; long totalBandwidth = 0L; for (long[] v : history.values()) { long endMillis = v[1]; if (endMillis < intervalStartMillis) { continue; - } - long startMillis = v[0]; - if (startMillis > this.now) { + } else if (endMillis > graphEndMillis) { break; } + long startMillis = v[0]; if (endMillis - startMillis > dataPointInterval) { /* This history interval is too long for this graph's data point * interval. Maybe the next graph will contain it, but not this @@ -153,11 +157,11 @@ public class BandwidthDocumentWriter implements DocumentWriter { if (firstNonNullIndex < 0) { continue; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (i > 0 && !graphs.isEmpty() - && firstDataPointMillis >= this.now - graphIntervals[i - 1]) { + long firstDataPointMillis = graphStartMillis + firstNonNullIndex + * dataPointInterval + dataPointInterval / 2L; + if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[i - 1]) { /* Skip bandwidth history object, because it doesn't contain * anything new that wasn't already contained in the last * bandwidth history object(s). Unless we did not include any of diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index e74f078..834df9c 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.FormattingUtils; @@ -52,11 +53,8 @@ public class ClientsDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public ClientsDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } private int writtenDocuments = 0; @@ -70,6 +68,11 @@ public class ClientsDocumentWriter implements DocumentWriter { SortedSet<String> updateDocuments = this.documentStore.list( ClientsStatus.class, updatedMillis); for (String hashedFingerprint : updateDocuments) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, hashedFingerprint); + if (null == nodeStatus) { + continue; + } ClientsStatus clientsStatus = this.documentStore.retrieve( ClientsStatus.class, true, hashedFingerprint); if (clientsStatus == null) { @@ -77,7 +80,7 @@ public class ClientsDocumentWriter implements DocumentWriter { } SortedSet<ClientsHistory> history = clientsStatus.getHistory(); ClientsDocument clientsDocument = this.compileClientsDocument( - hashedFingerprint, history); + hashedFingerprint, nodeStatus, history); this.documentStore.store(clientsDocument, hashedFingerprint); this.writtenDocuments++; } @@ -106,7 +109,7 @@ public class ClientsDocumentWriter implements DocumentWriter { DateTimeHelper.TEN_DAYS }; private ClientsDocument compileClientsDocument(String hashedFingerprint, - SortedSet<ClientsHistory> history) { + NodeStatus nodeStatus, SortedSet<ClientsHistory> history) { ClientsDocument clientsDocument = new ClientsDocument(); clientsDocument.setFingerprint(hashedFingerprint); Map<String, GraphHistory> averageClients = new LinkedHashMap<>(); @@ -114,7 +117,7 @@ public class ClientsDocumentWriter implements DocumentWriter { < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileClientsHistory( - graphIntervalIndex, history); + graphIntervalIndex, history, nodeStatus.getLastSeenMillis()); if (graphHistory != null) { averageClients.put(graphName, graphHistory); } @@ -124,18 +127,23 @@ public class ClientsDocumentWriter implements DocumentWriter { } private GraphHistory compileClientsHistory( - int graphIntervalIndex, SortedSet<ClientsHistory> history) { + int graphIntervalIndex, SortedSet<ClientsHistory> history, + long lastSeenMillis) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; List<Double> dataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; long millis = 0L; double responses = 0.0; for (ClientsHistory hist : history) { if (hist.getEndMillis() < intervalStartMillis) { continue; + } else if (hist.getEndMillis() > graphEndMillis) { + break; } while ((intervalStartMillis / dataPointInterval) != (hist.getEndMillis() / dataPointInterval)) { @@ -172,11 +180,11 @@ public class ClientsDocumentWriter implements DocumentWriter { /* Not a single non-negative value in the data points. */ return null; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis - >= this.now - graphIntervals[graphIntervalIndex - 1]) { + long firstDataPointMillis = graphStartMillis + firstNonNullIndex + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { /* Skip clients history object, because it doesn't contain * anything new that wasn't already contained in the last * clients history object(s). */ diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index e28359b..96537ef 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -7,6 +7,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeHistory; @@ -32,11 +33,8 @@ public class UptimeDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public UptimeDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } @Override @@ -63,9 +61,11 @@ public class UptimeDocumentWriter implements DocumentWriter { private void updateDocument(String fingerprint, UptimeStatus knownStatuses) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true, fingerprint); - if (uptimeStatus != null) { + if (null != nodeStatus && null != uptimeStatus) { boolean relay = uptimeStatus.getBridgeHistory().isEmpty(); SortedSet<UptimeHistory> history = relay ? uptimeStatus.getRelayHistory() @@ -73,8 +73,9 @@ public class UptimeDocumentWriter implements DocumentWriter { SortedSet<UptimeHistory> knownStatusesHistory = relay ? knownStatuses.getRelayHistory() : knownStatuses.getBridgeHistory(); + long lastSeenMillis = nodeStatus.getLastSeenMillis(); UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, - fingerprint, history, knownStatusesHistory); + fingerprint, history, knownStatusesHistory, lastSeenMillis); this.documentStore.store(uptimeDocument, fingerprint); this.writtenDocuments++; } @@ -103,7 +104,7 @@ public class UptimeDocumentWriter implements DocumentWriter { private UptimeDocument compileUptimeDocument(boolean relay, String fingerprint, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { + SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis) { UptimeDocument uptimeDocument = new UptimeDocument(); uptimeDocument.setFingerprint(fingerprint); Map<String, GraphHistory> uptime = new LinkedHashMap<>(); @@ -111,7 +112,8 @@ public class UptimeDocumentWriter implements DocumentWriter { < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses, null); + graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis, + null); if (graphHistory != null) { uptime.put(graphName, graphHistory); } @@ -130,7 +132,8 @@ public class UptimeDocumentWriter implements DocumentWriter { < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses, flag); + graphIntervalIndex, relay, history, knownStatuses, lastSeenMillis, + flag); if (graphHistory != null) { graphsForFlags.put(graphName, graphHistory); } @@ -147,15 +150,18 @@ public class UptimeDocumentWriter implements DocumentWriter { private GraphHistory compileUptimeHistory(int graphIntervalIndex, boolean relay, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses, String flag) { + SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis, + String flag) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; int dataPointIntervalHours = (int) (dataPointInterval / DateTimeHelper.ONE_HOUR); List<Integer> uptimeDataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; int uptimeHours = 0; long firstStatusStartMillis = -1L; for (UptimeHistory hist : history) { @@ -171,6 +177,8 @@ public class UptimeDocumentWriter implements DocumentWriter { * hist.getUptimeHours(); if (histEndMillis < intervalStartMillis) { continue; + } else if (histEndMillis > graphEndMillis) { + histEndMillis = graphEndMillis; } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { @@ -197,10 +205,12 @@ public class UptimeDocumentWriter implements DocumentWriter { } uptimeDataPoints.add(uptimeHours); List<Integer> statusDataPoints = new ArrayList<>(); - intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; + intervalStartMillis = graphStartMillis; int statusHours = -1; for (UptimeHistory hist : knownStatuses) { + if (hist.getStartMillis() >= graphEndMillis) { + break; + } if (hist.isRelay() != relay || (flag != null && (hist.getFlags() == null || !hist.getFlags().contains(flag)))) { @@ -210,6 +220,8 @@ public class UptimeDocumentWriter implements DocumentWriter { * hist.getUptimeHours(); if (histEndMillis < intervalStartMillis) { continue; + } else if (histEndMillis > graphEndMillis) { + histEndMillis = graphEndMillis; } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { @@ -271,11 +283,11 @@ public class UptimeDocumentWriter implements DocumentWriter { /* Not a single non-negative value in the data points. */ return null; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) + long firstDataPointMillis = graphStartMillis + firstNonNullIndex * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis - >= this.now - graphIntervals[graphIntervalIndex - 1]) { + if (graphIntervalIndex > 0 && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { /* Skip uptime history object, because it doesn't contain * anything new that wasn't already contained in the last * uptime history object(s). */ diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index 75626a3..ad54fea 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -7,6 +7,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.WeightsDocument; import org.torproject.onionoo.docs.WeightsStatus; @@ -28,11 +29,8 @@ public class WeightsDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public WeightsDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = System.currentTimeMillis(); } @Override @@ -44,14 +42,20 @@ public class WeightsDocumentWriter implements DocumentWriter { SortedSet<String> updateWeightsDocuments = this.documentStore.list( WeightsStatus.class, updatedMillis); for (String fingerprint : updateWeightsDocuments) { + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null == nodeStatus) { + continue; + } WeightsStatus weightsStatus = this.documentStore.retrieve( WeightsStatus.class, true, fingerprint); if (weightsStatus == null) { continue; } SortedMap<long[], double[]> history = weightsStatus.getHistory(); + long lastSeenMillis = nodeStatus.getLastSeenMillis(); WeightsDocument weightsDocument = this.compileWeightsDocument( - fingerprint, history); + fingerprint, history, lastSeenMillis); this.documentStore.store(weightsDocument, fingerprint); } log.info("Wrote weights document files"); @@ -79,30 +83,31 @@ public class WeightsDocumentWriter implements DocumentWriter { DateTimeHelper.TEN_DAYS }; private WeightsDocument compileWeightsDocument(String fingerprint, - SortedMap<long[], double[]> history) { + SortedMap<long[], double[]> history, long lastSeenMillis) { WeightsDocument weightsDocument = new WeightsDocument(); weightsDocument.setFingerprint(fingerprint); weightsDocument.setConsensusWeightFraction( - this.compileGraphType(history, 1)); + this.compileGraphType(history, lastSeenMillis, 1)); weightsDocument.setGuardProbability( - this.compileGraphType(history, 2)); + this.compileGraphType(history, lastSeenMillis, 2)); weightsDocument.setMiddleProbability( - this.compileGraphType(history, 3)); + this.compileGraphType(history, lastSeenMillis, 3)); weightsDocument.setExitProbability( - this.compileGraphType(history, 4)); + this.compileGraphType(history, lastSeenMillis, 4)); weightsDocument.setConsensusWeight( - this.compileGraphType(history, 6)); + this.compileGraphType(history, lastSeenMillis, 6)); return weightsDocument; } private Map<String, GraphHistory> compileGraphType( - SortedMap<long[], double[]> history, int graphTypeIndex) { + SortedMap<long[], double[]> history, long lastSeenMillis, + int graphTypeIndex) { Map<String, GraphHistory> graphs = new LinkedHashMap<>(); for (int graphIntervalIndex = 0; graphIntervalIndex < this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileWeightsHistory( - graphTypeIndex, graphIntervalIndex, history); + graphTypeIndex, graphIntervalIndex, history, lastSeenMillis); if (graphHistory != null) { graphs.put(graphName, graphHistory); } @@ -111,13 +116,16 @@ public class WeightsDocumentWriter implements DocumentWriter { } private GraphHistory compileWeightsHistory(int graphTypeIndex, - int graphIntervalIndex, SortedMap<long[], double[]> history) { + int graphIntervalIndex, SortedMap<long[], double[]> history, + long lastSeenMillis) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; List<Double> dataPoints = new ArrayList<>(); - long intervalStartMillis = ((this.now - graphInterval) + long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; + long graphStartMillis = graphEndMillis - graphInterval; + long intervalStartMillis = graphStartMillis; long totalMillis = 0L; double totalWeightTimesMillis = 0.0; for (Map.Entry<long[], double[]> e : history.entrySet()) { @@ -126,6 +134,8 @@ public class WeightsDocumentWriter implements DocumentWriter { double weight = e.getValue()[graphTypeIndex]; if (endMillis < intervalStartMillis) { continue; + } else if (endMillis > graphEndMillis) { + break; } while ((intervalStartMillis / dataPointInterval) != (endMillis / dataPointInterval)) { @@ -163,11 +173,11 @@ public class WeightsDocumentWriter implements DocumentWriter { /* Not a single non-negative value in the data points. */ return null; } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis - >= this.now - graphIntervals[graphIntervalIndex - 1]) { + long firstDataPointMillis = graphStartMillis + firstNonNullIndex + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) + * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { /* Skip weights history object, because it doesn't contain * anything new that wasn't already contained in the last * weights history object(s). */ diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java index 3c74ae5..324122c 100644 --- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java @@ -12,8 +12,7 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.DummyDocumentStore; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DummyDescriptorSource; +import org.torproject.onionoo.docs.NodeStatus; import org.junit.Before; import org.junit.Test; @@ -23,14 +22,6 @@ import java.util.Date; public class BandwidthDocumentWriterTest { - private DummyDescriptorSource descriptorSource; - - @Before - public void createDummyDescriptorSource() { - this.descriptorSource = new DummyDescriptorSource(); - DescriptorSourceFactory.setDescriptorSource(this.descriptorSource); - } - private DummyDocumentStore documentStore; @Before @@ -41,7 +32,7 @@ public class BandwidthDocumentWriterTest { @Test public void testIgnoreFuture() { - BandwidthStatus status = new BandwidthStatus(); + String ibibUnc0Fingerprint = "7C0AA4E3B73E407E9F5FEB1912F8BE26D8AA124D"; String future = new SimpleDateFormat("yyyy") .format(new Date(System.currentTimeMillis() + DateTimeHelper.ROUGHLY_ONE_YEAR)); @@ -66,13 +57,17 @@ public class BandwidthDocumentWriterTest { + "r " + future + "-08-06 17:31:45 " + future + "-08-06 21:31:45 0\n" + "r " + future + "-08-06 21:31:45 " + future + "-08-07 01:31:45 0\n" + "r " + future + "-08-07 01:31:45 " + future + "-08-07 05:31:45 0\n"; - + NodeStatus nodeStatus = new NodeStatus(ibibUnc0Fingerprint); + nodeStatus.setLastSeenMillis(DateTimeHelper.parse( + yesterday + " 12:00:00")); + this.documentStore.addDocument(nodeStatus, ibibUnc0Fingerprint); + BandwidthStatus status = new BandwidthStatus(); status.setFromDocumentString(documentString); - String ibibUnc0Fingerprint = "7C0AA4E3B73E407E9F5FEB1912F8BE26D8AA124D"; this.documentStore.addDocument(status, ibibUnc0Fingerprint); BandwidthDocumentWriter writer = new BandwidthDocumentWriter(); - DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); + assertEquals(1, this.documentStore.getPerformedListOperations()); + assertEquals(3, this.documentStore.getPerformedRetrieveOperations()); assertEquals(1, this.documentStore.getPerformedStoreOperations()); BandwidthDocument document = this.documentStore.getDocument( BandwidthDocument.class, ibibUnc0Fingerprint); diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java index 5554bf0..3504509 100644 --- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java @@ -5,21 +5,19 @@ package org.torproject.onionoo.writer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.DummyDocumentStore; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeStatus; import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DummyDescriptorSource; import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.util.Arrays; import java.util.List; @@ -28,14 +26,6 @@ public class UptimeDocumentWriterTest { private static final long TEST_TIME = DateTimeHelper.parse( "2014-03-23 12:00:00"); - private DummyDescriptorSource descriptorSource; - - @Before - public void createDummyDescriptorSource() { - this.descriptorSource = new DummyDescriptorSource(); - DescriptorSourceFactory.setDescriptorSource(this.descriptorSource); - } - private DummyDocumentStore documentStore; @Before @@ -44,21 +34,9 @@ public class UptimeDocumentWriterTest { DocumentStoreFactory.setDocumentStore(this.documentStore); } - private void setTime(UptimeDocumentWriter udw) { - try { - Field nowField = udw.getClass() - .getDeclaredField("now"); - nowField.setAccessible(true); - nowField.set(udw, TEST_TIME); - } catch (Exception ex) { - fail("Cannot manipulate test-time. Failing all."); - } - } - @Test public void testNoStatuses() { UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); writer.writeDocuments(); assertEquals("Without providing any data, nothing should be written " + "to disk.", 0, @@ -72,6 +50,9 @@ public class UptimeDocumentWriterTest { private void addStatusOneWeekSample(String allRelaysUptime, String gabelmooUptime) { + NodeStatus nodeStatus = new NodeStatus(GABELMOO_FINGERPRINT); + nodeStatus.setLastSeenMillis(TEST_TIME); + this.documentStore.addDocument(nodeStatus, GABELMOO_FINGERPRINT); UptimeStatus status = new UptimeStatus(); status.setFromDocumentString(allRelaysUptime); this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT); @@ -131,7 +112,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-11 1\n", "r 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -147,7 +127,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-10 2\n", "r 2014-03-23-10 2\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -163,7 +142,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-09 1\nr 2014-03-23-11 1\n", "r 2014-03-23-09 1\nr 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -179,7 +157,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-09 3\n", "r 2014-03-23-09 1\nr 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -196,7 +173,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-23-09 3\n", "r 2014-03-23-09 2\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -210,36 +186,32 @@ public class UptimeDocumentWriterTest { @Test public void testOneWeekUptime() { - this.addStatusOneWeekSample("r 2014-03-16-12 168\n", - "r 2014-03-16-12 168\n"); + this.addStatusOneWeekSample("r 2014-03-16-13 168\n", + "r 2014-03-16-13 168\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( UptimeDocument.class, GABELMOO_FINGERPRINT); - this.assertOneWeekGraph(document, 1, "2014-03-16 12:30:00", - "2014-03-23 11:30:00", 168, null); + this.assertOneWeekGraph(document, 1, "2014-03-16 13:30:00", + "2014-03-23 12:30:00", 168, null); } @Test public void testOneWeekOneHourUptime() { - this.addStatusOneWeekSample("r 2014-03-16-11 169\n", - "r 2014-03-16-11 169\n"); + this.addStatusOneWeekSample("r 2014-03-16-12 169\n", + "r 2014-03-16-12 169\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( UptimeDocument.class, GABELMOO_FINGERPRINT); - this.assertOneWeekGraph(document, 2, "2014-03-16 12:30:00", - "2014-03-23 11:30:00", 168, null); - this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00", - "2014-03-23 10:00:00", 43, null); + this.assertOneWeekGraph(document, 1, "2014-03-16 13:30:00", + "2014-03-23 12:30:00", 168, null); } @Test @@ -247,7 +219,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-16-08 8\n", "r 2014-03-16-11 5\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1, @@ -263,7 +234,6 @@ public class UptimeDocumentWriterTest { this.addStatusOneWeekSample("r 2014-03-16-08 8\n", "r 2014-03-16-10 1\nr 2014-03-16-12 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); - setTime(writer); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); writer.writeDocuments(); assertEquals("Should write exactly one document.", 1,

1 0

[onionoo/release] Don't rely on system time for compressing histories.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 8b14cb159b7ee5b4f4e2c11a9a4b03269bda6974 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jan 11 12:45:09 2018 +0100 Don't rely on system time for compressing histories. Related to not relying on system time for writing histories, we also shouldn't rely on it for compressing histories anymore. Otherwise we might be compressing the history of a relay or bridge that recently went offline while expecting an uncompressed history of that relay for writing history documents. In short, let's do the same when updating *Status objects as we do when writing *Document objects. Related to #16513, but not strictly part of it. --- .../torproject/onionoo/docs/BandwidthStatus.java | 23 ++++++++++------------ .../org/torproject/onionoo/docs/ClientsStatus.java | 7 +++---- .../org/torproject/onionoo/docs/WeightsStatus.java | 11 +++++------ .../onionoo/updater/BandwidthStatusUpdater.java | 7 ++++++- .../onionoo/updater/ClientsStatusUpdater.java | 7 ++++++- .../onionoo/updater/WeightsStatusUpdater.java | 7 ++++++- .../torproject/onionoo/docs/WeightsStatusTest.java | 14 ++++++------- 7 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java index 1a81d20..ba22dd4 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java @@ -108,16 +108,13 @@ public class BandwidthStatus extends Document { } } - public void compressHistory() { - this.compressHistory(System.currentTimeMillis()); + public void compressHistory(long lastSeenMillis) { + this.compressHistory(this.writeHistory, lastSeenMillis); + this.compressHistory(this.readHistory, lastSeenMillis); } - public void compressHistory(long now) { - this.compressHistory(this.writeHistory, now); - this.compressHistory(this.readHistory, now); - } - - private void compressHistory(SortedMap<Long, long[]> history, long now) { + private void compressHistory(SortedMap<Long, long[]> history, + long lastSeenMillis) { SortedMap<Long, long[]> uncompressedHistory = new TreeMap<>(history); history.clear(); long lastStartMillis = 0L; @@ -129,17 +126,17 @@ public class BandwidthStatus extends Document { long endMillis = v[1]; long bandwidth = v[2]; long intervalLengthMillis; - if (now - endMillis <= DateTimeHelper.THREE_DAYS) { + if (lastSeenMillis - endMillis <= DateTimeHelper.THREE_DAYS) { intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES; - } else if (now - endMillis <= DateTimeHelper.ONE_WEEK) { + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) { intervalLengthMillis = DateTimeHelper.ONE_HOUR; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_MONTH) { intervalLengthMillis = DateTimeHelper.FOUR_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_THREE_MONTHS) { intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_YEAR) { intervalLengthMillis = DateTimeHelper.TWO_DAYS; } else { diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java index 5310abf..a5bc8e8 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java @@ -69,18 +69,17 @@ public class ClientsStatus extends Document { /** Compresses the history of clients objects by merging adjacent * intervals, depending on how far back in the past they lie. */ - public void compressHistory() { + public void compressHistory(long lastSeenMillis) { SortedSet<ClientsHistory> uncompressedHistory = new TreeSet<>(this.history); history.clear(); ClientsHistory lastResponses = null; String lastMonthString = "1970-01"; - long now = System.currentTimeMillis(); for (ClientsHistory responses : uncompressedHistory) { long intervalLengthMillis; - if (now - responses.getEndMillis() + if (lastSeenMillis - responses.getEndMillis() <= DateTimeHelper.ROUGHLY_THREE_MONTHS) { intervalLengthMillis = DateTimeHelper.ONE_DAY; - } else if (now - responses.getEndMillis() + } else if (lastSeenMillis - responses.getEndMillis() <= DateTimeHelper.ROUGHLY_ONE_YEAR) { intervalLengthMillis = DateTimeHelper.TWO_DAYS; } else { diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index 0dfd4f1..94efda6 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -125,7 +125,7 @@ public class WeightsStatus extends Document { /** Compresses the history of weights objects by merging adjacent * intervals, depending on how far back in the past they lie. */ - public void compressHistory() { + public void compressHistory(long lastSeenMillis) { SortedMap<long[], double[]> uncompressedHistory = new TreeMap<>(histComparator); uncompressedHistory.putAll(this.history); @@ -135,21 +135,20 @@ public class WeightsStatus extends Document { double[] lastWeights = null; String lastMonthString = "1970-01"; int lastMissingValues = -1; - long now = System.currentTimeMillis(); for (Map.Entry<long[], double[]> e : uncompressedHistory.entrySet()) { long startMillis = e.getKey()[0]; long endMillis = e.getKey()[1]; double[] weights = e.getValue(); long intervalLengthMillis; - if (now - endMillis <= DateTimeHelper.ONE_WEEK) { + if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) { intervalLengthMillis = DateTimeHelper.ONE_HOUR; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_MONTH) { intervalLengthMillis = DateTimeHelper.FOUR_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_THREE_MONTHS) { intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; - } else if (now - endMillis + } else if (lastSeenMillis - endMillis <= DateTimeHelper.ROUGHLY_ONE_YEAR) { intervalLengthMillis = DateTimeHelper.TWO_DAYS; } else { diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java index d8288f8..b5e5ddb 100644 --- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.ExtraInfoDescriptor; import org.torproject.onionoo.docs.BandwidthStatus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus; public class BandwidthStatusUpdater implements DescriptorListener, StatusUpdater { @@ -58,7 +59,11 @@ public class BandwidthStatusUpdater implements DescriptorListener, bandwidthStatus.addToReadHistory(descriptor.getReadHistory()); } if (bandwidthStatus.isDirty()) { - bandwidthStatus.compressHistory(); + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null != nodeStatus) { + bandwidthStatus.compressHistory(nodeStatus.getLastSeenMillis()); + } this.documentStore.store(bandwidthStatus, fingerprint); bandwidthStatus.clearDirty(); } diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java index 0021068..0529dcf 100644 --- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java @@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.ClientsStatus; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.util.FormattingUtils; import java.util.Map; @@ -158,7 +159,11 @@ public class ClientsStatusUpdater implements DescriptorListener, } clientsStatus.addToHistory(e.getValue()); if (clientsStatus.isDirty()) { - clientsStatus.compressHistory(); + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, hashedFingerprint); + if (null != nodeStatus) { + clientsStatus.compressHistory(nodeStatus.getLastSeenMillis()); + } this.documentStore.store(clientsStatus, hashedFingerprint); clientsStatus.clearDirty(); } diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java index 6c5ec0d..886f2f8 100644 --- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.WeightsStatus; import java.util.Arrays; @@ -76,7 +77,11 @@ public class WeightsStatusUpdater implements DescriptorListener, weightsStatus.addToHistory(validAfterMillis, freshUntilMillis, weights); if (weightsStatus.isDirty()) { - weightsStatus.compressHistory(); + NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (null != nodeStatus) { + weightsStatus.compressHistory(nodeStatus.getLastSeenMillis()); + } this.documentStore.store(weightsStatus, fingerprint); weightsStatus.clearDirty(); } diff --git a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java index c0457e7..e28f989 100644 --- a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java @@ -38,16 +38,16 @@ public class WeightsStatusTest { WeightsStatus ws = new WeightsStatus(); SortedMap<long[], double[]> history = ws.getHistory(); assertTrue("actually: " + mapToString(history), history.isEmpty()); - long now = System.currentTimeMillis(); + long lastSeenMillis = 1515670935476L; long hourMillis = 60L * 60L * 1000L; for (long j = 1L; j < 10L; j++) { // add arbitrary data - ws.addToHistory(now - j * hourMillis - hourMillis, - now - j * hourMillis, + ws.addToHistory(lastSeenMillis - j * hourMillis - hourMillis, + lastSeenMillis - j * hourMillis, new double[]{ 1.0 * j, 2.0 * j, 1.0 * j, 2.0 * j, 1.0 * j, 2.0 * j, Double.NaN}); assertEquals("have: " + mapToString(history), (int) j, history.size()); } - ws.compressHistory(); + ws.compressHistory(lastSeenMillis); assertEquals("history map: " + mapToString(history), 9, history.size()); assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(), ws.toDocumentString().contains("NaN")); @@ -67,7 +67,7 @@ public class WeightsStatusTest { 2.0 * j, j * 0.5}); assertEquals("have: " + mapToString(history), (int) j, history.size()); } - ws.compressHistory(); + ws.compressHistory(1515670935476L); assertEquals("history map: " + mapToString(history), 1, history.size()); assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(), ws.toDocumentString().contains("NaN")); @@ -101,7 +101,7 @@ public class WeightsStatusTest { } assertEquals("history: ", correctHistory, mapToString(ws.getHistory())); assertEquals(correctLines.length, ws.getHistory().size()); - ws.compressHistory(); + ws.compressHistory(1515670935476L); assertEquals("found: " + mapToString(ws.getHistory()), 1, ws.getHistory().size()); assertEquals("[1431032400000, 1431043200000] : [-1.0, 1.78279826E-4, " @@ -128,7 +128,7 @@ public class WeightsStatusTest { assertEquals("document string: ", exp, ws.toDocumentString()); } assertEquals(correctLines.length, ws.getHistory().size()); - ws.compressHistory(); + ws.compressHistory(1515670935476L); assertEquals("found: " + mapToString(ws.getHistory()), 3, ws.getHistory().size()); }

1 0

[onionoo/release] Update copyright to 2018.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 0049663b19caad900342dc2d76909d7642deadd5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 9 10:18:08 2018 +0100 Update copyright to 2018. --- src/main/java/org/torproject/onionoo/cron/Main.java | 2 +- src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java | 2 +- src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java | 2 +- src/main/java/org/torproject/onionoo/docs/ClientsDocument.java | 2 +- src/main/java/org/torproject/onionoo/docs/ClientsHistory.java | 2 +- src/main/java/org/torproject/onionoo/docs/ClientsStatus.java | 2 +- src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java | 2 +- src/main/java/org/torproject/onionoo/docs/DetailsDocument.java | 2 +- src/main/java/org/torproject/onionoo/docs/DetailsDocumentFields.java | 2 +- src/main/java/org/torproject/onionoo/docs/DetailsStatus.java | 2 +- src/main/java/org/torproject/onionoo/docs/Document.java | 2 +- src/main/java/org/torproject/onionoo/docs/DocumentStore.java | 2 +- src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java | 2 +- src/main/java/org/torproject/onionoo/docs/GraphHistory.java | 2 +- src/main/java/org/torproject/onionoo/docs/NodeStatus.java | 2 +- src/main/java/org/torproject/onionoo/docs/SummaryDocument.java | 2 +- src/main/java/org/torproject/onionoo/docs/UpdateStatus.java | 2 +- src/main/java/org/torproject/onionoo/docs/UptimeDocument.java | 2 +- src/main/java/org/torproject/onionoo/docs/UptimeHistory.java | 2 +- src/main/java/org/torproject/onionoo/docs/UptimeStatus.java | 2 +- src/main/java/org/torproject/onionoo/docs/WeightsDocument.java | 2 +- src/main/java/org/torproject/onionoo/docs/WeightsStatus.java | 2 +- src/main/java/org/torproject/onionoo/server/Counter.java | 2 +- .../java/org/torproject/onionoo/server/HttpServletRequestWrapper.java | 2 +- .../java/org/torproject/onionoo/server/HttpServletResponseWrapper.java | 2 +- src/main/java/org/torproject/onionoo/server/IntegerDistribution.java | 2 +- src/main/java/org/torproject/onionoo/server/MostFrequentString.java | 2 +- src/main/java/org/torproject/onionoo/server/NodeIndex.java | 2 +- src/main/java/org/torproject/onionoo/server/NodeIndexer.java | 2 +- src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java | 2 +- src/main/java/org/torproject/onionoo/server/OrderParameterValues.java | 2 +- src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java | 2 +- src/main/java/org/torproject/onionoo/server/RequestHandler.java | 2 +- src/main/java/org/torproject/onionoo/server/ResourceServlet.java | 2 +- src/main/java/org/torproject/onionoo/server/ResponseBuilder.java | 2 +- src/main/java/org/torproject/onionoo/server/ServerMain.java | 2 +- .../java/org/torproject/onionoo/server/SummaryDocumentComparator.java | 2 +- .../java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java | 2 +- src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java | 2 +- src/main/java/org/torproject/onionoo/updater/DescriptorHistory.java | 2 +- src/main/java/org/torproject/onionoo/updater/DescriptorListener.java | 2 +- src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java | 2 +- src/main/java/org/torproject/onionoo/updater/DescriptorSource.java | 2 +- .../java/org/torproject/onionoo/updater/DescriptorSourceFactory.java | 2 +- src/main/java/org/torproject/onionoo/updater/DescriptorType.java | 2 +- src/main/java/org/torproject/onionoo/updater/LookupResult.java | 2 +- src/main/java/org/torproject/onionoo/updater/LookupService.java | 2 +- .../java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java | 2 +- src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java | 2 +- src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java | 2 +- .../java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java | 2 +- src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java | 2 +- src/main/java/org/torproject/onionoo/updater/StatusUpdater.java | 2 +- src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java | 2 +- src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java | 2 +- src/main/java/org/torproject/onionoo/util/FormattingUtils.java | 2 +- .../java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java | 2 +- src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java | 2 +- src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java | 2 +- src/main/java/org/torproject/onionoo/writer/DocumentWriter.java | 2 +- src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java | 2 +- src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java | 2 +- src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java | 2 +- src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java | 2 +- src/main/resources/bootstrap-development.sh | 2 +- src/test/java/org/torproject/onionoo/docs/BandwidthStatusTest.java | 2 +- src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java | 2 +- src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java | 2 +- src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java | 2 +- src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java | 2 +- src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java | 2 +- src/test/java/org/torproject/onionoo/server/PerformanceMetricsTest.java | 2 +- src/test/java/org/torproject/onionoo/server/ResourceServletTest.java | 2 +- .../org/torproject/onionoo/server/SummaryDocumentComparatorTest.java | 2 +- src/test/java/org/torproject/onionoo/updater/DummyBridgeStatus.java | 2 +- src/test/java/org/torproject/onionoo/updater/DummyConsensus.java | 2 +- src/test/java/org/torproject/onionoo/updater/DummyDescriptorSource.java | 2 +- src/test/java/org/torproject/onionoo/updater/DummyStatusEntry.java | 2 +- src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java | 2 +- .../org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java | 2 +- .../java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java | 2 +- .../java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java | 2 +- .../java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java | 2 +- 83 files changed, 83 insertions(+), 83 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java index f6f7099..9b34356 100644 --- a/src/main/java/org/torproject/onionoo/cron/Main.java +++ b/src/main/java/org/torproject/onionoo/cron/Main.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.cron; diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java b/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java index 38f5f62..bccbb87 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java index b58e57c..1a81d20 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java b/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java index 0b625b0..16e7bc5 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java index c1c93fd..ba21458 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java index 45a4ddf..5310abf 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java index 7a63f53..c80514e 100644 --- a/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java +++ b/src/main/java/org/torproject/onionoo/docs/DateTimeHelper.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 911a978..c749fba 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocumentFields.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocumentFields.java index df46149..2abcd53 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocumentFields.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocumentFields.java @@ -1,4 +1,4 @@ -/* Copyright 2017 The Tor Project +/* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 9ed8658..c3cdc28 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/Document.java b/src/main/java/org/torproject/onionoo/docs/Document.java index 355a543..54bb0d5 100644 --- a/src/main/java/org/torproject/onionoo/docs/Document.java +++ b/src/main/java/org/torproject/onionoo/docs/Document.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 23f11c7..4622a34 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java b/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java index 8764d69..fd7330a 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStoreFactory.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/GraphHistory.java b/src/main/java/org/torproject/onionoo/docs/GraphHistory.java index 7d41780..495f01a 100644 --- a/src/main/java/org/torproject/onionoo/docs/GraphHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/GraphHistory.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 7a8510e..cfd13aa 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 11f6070..42a5a64 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java index 32461fc..889a52d 100644 --- a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java index c88b9ff..813d597 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java index 7c78f54..60e283f 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java index 18cd259..87ee962 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java index 6d5c6a6..f075f0b 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index 4c9c899..0dfd4f1 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/main/java/org/torproject/onionoo/server/Counter.java b/src/main/java/org/torproject/onionoo/server/Counter.java index 6c53587..5847bb4 100644 --- a/src/main/java/org/torproject/onionoo/server/Counter.java +++ b/src/main/java/org/torproject/onionoo/server/Counter.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java b/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java index 8da8821..904e55c 100644 --- a/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java +++ b/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/HttpServletResponseWrapper.java b/src/main/java/org/torproject/onionoo/server/HttpServletResponseWrapper.java index c591106..f6888a4 100644 --- a/src/main/java/org/torproject/onionoo/server/HttpServletResponseWrapper.java +++ b/src/main/java/org/torproject/onionoo/server/HttpServletResponseWrapper.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java b/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java index adc3db2..f462765 100644 --- a/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java +++ b/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/MostFrequentString.java b/src/main/java/org/torproject/onionoo/server/MostFrequentString.java index e9c8667..9fa51c7 100644 --- a/src/main/java/org/torproject/onionoo/server/MostFrequentString.java +++ b/src/main/java/org/torproject/onionoo/server/MostFrequentString.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index 8ab22a4..77993a3 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 815a02e..1a85a35 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java b/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java index 5d93bf4..cef287a 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexerFactory.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/OrderParameterValues.java b/src/main/java/org/torproject/onionoo/server/OrderParameterValues.java index eec47ef..6f76a5f 100644 --- a/src/main/java/org/torproject/onionoo/server/OrderParameterValues.java +++ b/src/main/java/org/torproject/onionoo/server/OrderParameterValues.java @@ -1,4 +1,4 @@ -/* Copyright 2017 The Tor Project +/* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java index 935915c..98401ae 100644 --- a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java +++ b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index c81051e..55938c5 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 3449407..a64b89c 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index 9b2b442..dbe911f 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/ServerMain.java b/src/main/java/org/torproject/onionoo/server/ServerMain.java index d712ba8..a60974b 100644 --- a/src/main/java/org/torproject/onionoo/server/ServerMain.java +++ b/src/main/java/org/torproject/onionoo/server/ServerMain.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/server/SummaryDocumentComparator.java b/src/main/java/org/torproject/onionoo/server/SummaryDocumentComparator.java index 64f61cc..0e6729b 100644 --- a/src/main/java/org/torproject/onionoo/server/SummaryDocumentComparator.java +++ b/src/main/java/org/torproject/onionoo/server/SummaryDocumentComparator.java @@ -1,4 +1,4 @@ -/* Copyright 2017 The Tor Project +/* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java index d750358..d8288f8 100644 --- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java index 44c4e1b..0021068 100644 --- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorHistory.java b/src/main/java/org/torproject/onionoo/updater/DescriptorHistory.java index 238546a..8fd92ca 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorHistory.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorHistory.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorListener.java b/src/main/java/org/torproject/onionoo/updater/DescriptorListener.java index f655d19..5362ce2 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorListener.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorListener.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java index 48fb4bf..c19fabd 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 07ae4f8..6bddd02 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java index c1d093b..cf85a15 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSourceFactory.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorType.java b/src/main/java/org/torproject/onionoo/updater/DescriptorType.java index 6647a51..e039254 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorType.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorType.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/LookupResult.java b/src/main/java/org/torproject/onionoo/updater/LookupResult.java index 94297c0..9447c07 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupResult.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupResult.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index fb9bf4a..4bcc754 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 0a5a93d..4fdf98b 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java index 959468a..cb9169c 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java index 72494ed..4c188ed 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 366d5ce..3c82d14 100644 --- a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java b/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java index 6c55c96..19d5c4b 100644 --- a/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java +++ b/src/main/java/org/torproject/onionoo/updater/StatusUpdateRunner.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/StatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/StatusUpdater.java index bc5b610..f17d797 100644 --- a/src/main/java/org/torproject/onionoo/updater/StatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/StatusUpdater.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java index 2b5f5fc..8405bf8 100644 --- a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java index 9ae60a9..6c5ec0d 100644 --- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -1,4 +1,4 @@ -/* Copyright 2012--2017 The Tor Project +/* Copyright 2012--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/main/java/org/torproject/onionoo/util/FormattingUtils.java b/src/main/java/org/torproject/onionoo/util/FormattingUtils.java index 1c3aa55..7ed1377 100644 --- a/src/main/java/org/torproject/onionoo/util/FormattingUtils.java +++ b/src/main/java/org/torproject/onionoo/util/FormattingUtils.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.util; diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index d4b20d7..f0ab771 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2017 The Tor Project +/* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index f65a53a..e74f078 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 7de83f4..bb81ca9 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java index f68e962..618a8e7 100644 --- a/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java index 9abc669..dc3462c 100644 --- a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java +++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index a412eec..813be1b 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index 8910e3a..e28359b 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index a4568f5..75626a3 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2012--2017 The Tor Project +/* Copyright 2012--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/main/resources/bootstrap-development.sh b/src/main/resources/bootstrap-development.sh index 28fb975..d301e25 100755 --- a/src/main/resources/bootstrap-development.sh +++ b/src/main/resources/bootstrap-development.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2016--2017 The Tor Project +# Copyright 2016--2018 The Tor Project # See LICENSE for licensing information # # Use for submodule initialization and checkout. diff --git a/src/test/java/org/torproject/onionoo/docs/BandwidthStatusTest.java b/src/test/java/org/torproject/onionoo/docs/BandwidthStatusTest.java index c74b1ec..34e8654 100644 --- a/src/test/java/org/torproject/onionoo/docs/BandwidthStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/BandwidthStatusTest.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java b/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java index 2f8063e..8827725 100644 --- a/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java +++ b/src/test/java/org/torproject/onionoo/docs/DummyDocumentStore.java @@ -1,4 +1,4 @@ -/* Copyright 2015--2017 The Tor Project +/* Copyright 2015--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java index 0b4344d..e2c5dac 100644 --- a/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java @@ -1,4 +1,4 @@ -/* Copyright 2015--2017 The Tor Project +/* Copyright 2015--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index ce13ccc..a832ca0 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -1,4 +1,4 @@ -/* Copyright 2015--2017 The Tor Project +/* Copyright 2015--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java index 0d95925..aff5ce5 100644 --- a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java index b4a06d7..c0457e7 100644 --- a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2017 The Tor Project +/* Copyright 2016--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; diff --git a/src/test/java/org/torproject/onionoo/server/PerformanceMetricsTest.java b/src/test/java/org/torproject/onionoo/server/PerformanceMetricsTest.java index f804409..80328e5 100644 --- a/src/test/java/org/torproject/onionoo/server/PerformanceMetricsTest.java +++ b/src/test/java/org/torproject/onionoo/server/PerformanceMetricsTest.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index dee5139..5f07140 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index 6820372..e24e26f 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -1,4 +1,4 @@ -/* Copyright 2017 The Tor Project +/* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.server; diff --git a/src/test/java/org/torproject/onionoo/updater/DummyBridgeStatus.java b/src/test/java/org/torproject/onionoo/updater/DummyBridgeStatus.java index 4b06c99..f16a713 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyBridgeStatus.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyBridgeStatus.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java index 5faf709..245226b 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/updater/DummyDescriptorSource.java b/src/test/java/org/torproject/onionoo/updater/DummyDescriptorSource.java index 2312ef9..c311b2b 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyDescriptorSource.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyDescriptorSource.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/updater/DummyStatusEntry.java b/src/test/java/org/torproject/onionoo/updater/DummyStatusEntry.java index 75359bb..38b8a3e 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyStatusEntry.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyStatusEntry.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java index 6ffeaf6..6311316 100644 --- a/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java +++ b/src/test/java/org/torproject/onionoo/updater/LookupServiceTest.java @@ -1,4 +1,4 @@ -/* Copyright 2013--2017 The Tor Project +/* Copyright 2013--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java b/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java index 84c1fa6..1bcc7eb 100644 --- a/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java +++ b/src/test/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdaterTest.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java index d890e2c..4797356 100644 --- a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java +++ b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.updater; diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java index 436ee43..3c74ae5 100644 --- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java @@ -1,4 +1,4 @@ -/* Copyright 2017 The Tor Project +/* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer; diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java index b49853d..5554bf0 100644 --- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java @@ -1,4 +1,4 @@ -/* Copyright 2014--2017 The Tor Project +/* Copyright 2014--2018 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.writer;

1 0

[onionoo/release] Fix off-by-one bug in writing history documents.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit bd2c7af771a64dd393970e2e483c59778b32dbfc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jan 15 11:12:51 2018 +0100 Fix off-by-one bug in writing history documents. When we consider a history interval from start to end, we really mean the interval [start, end[. Fixing a few off-by-one bugs in the code. --- .../java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java | 4 ++-- .../java/org/torproject/onionoo/writer/ClientsDocumentWriter.java | 4 ++-- src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java | 4 ++-- .../java/org/torproject/onionoo/writer/WeightsDocumentWriter.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index 99a5a00..f1c5041 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -111,7 +111,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { long totalBandwidth = 0L; for (long[] v : history.values()) { long endMillis = v[1]; - if (endMillis < intervalStartMillis) { + if (endMillis <= intervalStartMillis) { continue; } else if (endMillis > graphEndMillis) { break; @@ -124,7 +124,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { continue; } while ((intervalStartMillis / dataPointInterval) - != (endMillis / dataPointInterval)) { + != ((endMillis - 1L) / dataPointInterval)) { dataPoints.add(totalMillis * 5L < dataPointInterval ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND) / totalMillis); diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index 834df9c..81168f5 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -140,13 +140,13 @@ public class ClientsDocumentWriter implements DocumentWriter { long millis = 0L; double responses = 0.0; for (ClientsHistory hist : history) { - if (hist.getEndMillis() < intervalStartMillis) { + if (hist.getEndMillis() <= intervalStartMillis) { continue; } else if (hist.getEndMillis() > graphEndMillis) { break; } while ((intervalStartMillis / dataPointInterval) - != (hist.getEndMillis() / dataPointInterval)) { + != ((hist.getEndMillis() - 1L) / dataPointInterval)) { dataPoints.add(millis * 2L < dataPointInterval ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY) / (((double) millis) * 10.0)); diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index f739b9e..2aee7a1 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -175,7 +175,7 @@ public class UptimeDocumentWriter implements DocumentWriter { } long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR * hist.getUptimeHours(); - if (histEndMillis < intervalStartMillis) { + if (histEndMillis <= intervalStartMillis) { continue; } else if (histEndMillis > graphEndMillis) { histEndMillis = graphEndMillis; @@ -218,7 +218,7 @@ public class UptimeDocumentWriter implements DocumentWriter { } long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR * hist.getUptimeHours(); - if (histEndMillis < intervalStartMillis) { + if (histEndMillis <= intervalStartMillis) { continue; } else if (histEndMillis > graphEndMillis) { histEndMillis = graphEndMillis; diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index ad54fea..10d8a94 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -132,13 +132,13 @@ public class WeightsDocumentWriter implements DocumentWriter { long startMillis = e.getKey()[0]; long endMillis = e.getKey()[1]; double weight = e.getValue()[graphTypeIndex]; - if (endMillis < intervalStartMillis) { + if (endMillis <= intervalStartMillis) { continue; } else if (endMillis > graphEndMillis) { break; } while ((intervalStartMillis / dataPointInterval) - != (endMillis / dataPointInterval)) { + != ((endMillis - 1L) / dataPointInterval)) { dataPoints.add(totalMillis * 5L < dataPointInterval ? -1.0 : totalWeightTimesMillis / (double) totalMillis); totalWeightTimesMillis = 0.0;

1 0

[onionoo/release] Fix comparison in writing uptime documents.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 203d5d885d4155ed719c5168430ba237bad734aa Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jan 15 11:11:20 2018 +0100 Fix comparison in writing uptime documents. In the spec we write: "Contained graph history objects may contain null values if less than 20% of network statuses have been processed for a given time period." However, in the code we checked less than or equal. We should fix that. --- .../onionoo/writer/UptimeDocumentWriter.java | 12 +++--- .../onionoo/writer/UptimeDocumentWriterTest.java | 44 +++++++++++++++++++++- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index 96537ef..f739b9e 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -225,8 +225,8 @@ public class UptimeDocumentWriter implements DocumentWriter { } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); + statusDataPoints.add(statusHours * 5 < dataPointIntervalHours + ? -1 : statusHours); statusHours = -1; intervalStartMillis += dataPointInterval; } @@ -238,8 +238,8 @@ public class UptimeDocumentWriter implements DocumentWriter { - Math.max(Math.max(hist.getStartMillis(), firstStatusStartMillis), intervalStartMillis)) / DateTimeHelper.ONE_HOUR); - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); + statusDataPoints.add(statusHours * 5 < dataPointIntervalHours + ? -1 : statusHours); statusHours = -1; intervalStartMillis += dataPointInterval; } @@ -251,8 +251,8 @@ public class UptimeDocumentWriter implements DocumentWriter { intervalStartMillis)) / DateTimeHelper.ONE_HOUR); } if (statusHours > 0) { - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); + statusDataPoints.add(statusHours * 5 < dataPointIntervalHours + ? -1 : statusHours); } List<Double> dataPoints = new ArrayList<>(); for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java index 3504509..b1ba2ed 100644 --- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java @@ -77,6 +77,12 @@ public class UptimeDocumentWriterTest { (int) (FOUR_HOURS / ONE_SECOND), count, values); } + private void assertFiveYearGraph(UptimeDocument document, int graphs, + String first, String last, int count, List<Integer> values) { + this.assertGraph(document, graphs, "5_years", first, last, + (int) (DateTimeHelper.TEN_DAYS / ONE_SECOND), count, values); + } + private void assertGraph(UptimeDocument document, int graphs, String graphName, String first, String last, int interval, int count, List<Integer> values) { @@ -93,7 +99,7 @@ public class UptimeDocumentWriterTest { (int) history.getInterval()); assertEquals("Factor should be 1.0 / 999.0.", 1.0 / 999.0, (double) history.getFactor(), 0.01); - assertEquals("There should be one data point per hour.", count, + assertEquals("There should be " + count + " data points.", count, (int) history.getCount()); assertEquals("Count should be the same as the number of values.", count, history.getValues().size()); @@ -244,5 +250,41 @@ public class UptimeDocumentWriterTest { "2014-03-16 14:00:00", 2, Arrays.asList(new Integer[] { 499, 249 })); } + + @Test + public void testFiveYearsLessThan20Percent() { + /* This relay was running for exactly 11 days and 23 hours over 2 years ago. + * This time period exactly matches 100% of a data point interval of 10 days + * plus a tiny bit less than 20% of the next data point interval. */ + this.addStatusOneWeekSample("r 2012-03-05-00 287\n", + "r 2012-03-05-00 287\n"); + UptimeDocumentWriter writer = new UptimeDocumentWriter(); + DescriptorSourceFactory.getDescriptorSource().readDescriptors(); + writer.writeDocuments(); + assertEquals("Should write exactly one document.", 1, + this.documentStore.getPerformedStoreOperations()); + UptimeDocument document = this.documentStore.getDocument( + UptimeDocument.class, GABELMOO_FINGERPRINT); + assertEquals("Should not contain any graph.", 0, + document.getUptime().size()); + } + + @Test + public void testFiveYearsAtLeast20Percent() { + /* This relay was running for exactly 12 days over 2 years ago. This time + * period exactly matches 100% of a data point interval of 10 days plus 20% + * of the next data point interval. */ + this.addStatusOneWeekSample("r 2012-03-05-00 288\n", + "r 2012-03-05-00 288\n"); + UptimeDocumentWriter writer = new UptimeDocumentWriter(); + DescriptorSourceFactory.getDescriptorSource().readDescriptors(); + writer.writeDocuments(); + assertEquals("Should write exactly one document.", 1, + this.documentStore.getPerformedStoreOperations()); + UptimeDocument document = this.documentStore.getDocument( + UptimeDocument.class, GABELMOO_FINGERPRINT); + this.assertFiveYearGraph(document, 1, "2012-03-10 00:00:00", + "2012-03-20 00:00:00", 2, Arrays.asList(new Integer[] { 999, 999 })); + } }

1 0

[onionoo/release] Use exact periods for graphs covering months/years.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 5cefee7b9de2a746ccaff42c6a0adb27cac8578a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jan 15 15:20:02 2018 +0100 Use exact periods for graphs covering months/years. --- .../onionoo/writer/BandwidthDocumentWriter.java | 32 ++++++++++++++-------- .../onionoo/writer/ClientsDocumentWriter.java | 29 ++++++++++++-------- .../onionoo/writer/UptimeDocumentWriter.java | 29 ++++++++++++-------- .../onionoo/writer/WeightsDocumentWriter.java | 29 ++++++++++++-------- 4 files changed, 74 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index f1c5041..2f27271 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -15,6 +15,9 @@ import org.torproject.onionoo.docs.UpdateStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDateTime; +import java.time.Period; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -79,13 +82,13 @@ public class BandwidthDocumentWriter implements DocumentWriter { "1_year", "5_years" }; - private long[] graphIntervals = new long[] { - DateTimeHelper.THREE_DAYS, - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; + private Period[] graphIntervals = new Period[] { + Period.ofDays(3), + Period.ofWeeks(1), + Period.ofMonths(1), + Period.ofMonths(3), + Period.ofYears(1), + Period.ofYears(5) }; private long[] dataPointIntervals = new long[] { DateTimeHelper.FIFTEEN_MINUTES, @@ -100,12 +103,15 @@ public class BandwidthDocumentWriter implements DocumentWriter { Map<String, GraphHistory> graphs = new LinkedHashMap<>(); for (int i = 0; i < this.graphIntervals.length; i++) { String graphName = this.graphNames[i]; - long graphInterval = this.graphIntervals[i]; + Period graphInterval = this.graphIntervals[i]; long dataPointInterval = this.dataPointIntervals[i]; List<Long> dataPoints = new ArrayList<>(); long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; - long graphStartMillis = graphEndMillis - graphInterval; + long graphStartMillis = LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphInterval) + .toEpochSecond(ZoneOffset.UTC) * 1000L; long intervalStartMillis = graphStartMillis; long totalMillis = 0L; long totalBandwidth = 0L; @@ -159,9 +165,11 @@ public class BandwidthDocumentWriter implements DocumentWriter { } long firstDataPointMillis = graphStartMillis + firstNonNullIndex * dataPointInterval + dataPointInterval / 2L; - if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= - ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) - * dataPointInterval - graphIntervals[i - 1]) { + if (i > 0 && !graphs.isEmpty() && firstDataPointMillis >= LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphIntervals[i - 1]) + .toEpochSecond(ZoneOffset.UTC) * 1000L) { + /* Skip bandwidth history object, because it doesn't contain * anything new that wasn't already contained in the last * bandwidth history object(s). Unless we did not include any of diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index 81168f5..4eca33a 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -17,6 +17,9 @@ import org.torproject.onionoo.util.FormattingUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDateTime; +import java.time.Period; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -94,12 +97,12 @@ public class ClientsDocumentWriter implements DocumentWriter { "1_year", "5_years" }; - private long[] graphIntervals = new long[] { - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; + private Period[] graphIntervals = new Period[] { + Period.ofWeeks(1), + Period.ofMonths(1), + Period.ofMonths(3), + Period.ofYears(1), + Period.ofYears(5) }; private long[] dataPointIntervals = new long[] { DateTimeHelper.ONE_DAY, @@ -129,13 +132,16 @@ public class ClientsDocumentWriter implements DocumentWriter { private GraphHistory compileClientsHistory( int graphIntervalIndex, SortedSet<ClientsHistory> history, long lastSeenMillis) { - long graphInterval = this.graphIntervals[graphIntervalIndex]; + Period graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; List<Double> dataPoints = new ArrayList<>(); long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; - long graphStartMillis = graphEndMillis - graphInterval; + long graphStartMillis = LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphInterval) + .toEpochSecond(ZoneOffset.UTC) * 1000L; long intervalStartMillis = graphStartMillis; long millis = 0L; double responses = 0.0; @@ -182,9 +188,10 @@ public class ClientsDocumentWriter implements DocumentWriter { } long firstDataPointMillis = graphStartMillis + firstNonNullIndex * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) - * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { + if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphIntervals[graphIntervalIndex - 1]) + .toEpochSecond(ZoneOffset.UTC) * 1000L) { /* Skip clients history object, because it doesn't contain * anything new that wasn't already contained in the last * clients history object(s). */ diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index 2aee7a1..d1e2003 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -17,6 +17,9 @@ import org.torproject.onionoo.util.FormattingUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDateTime; +import java.time.Period; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -88,12 +91,12 @@ public class UptimeDocumentWriter implements DocumentWriter { "1_year", "5_years" }; - private long[] graphIntervals = new long[] { - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; + private Period[] graphIntervals = new Period[] { + Period.ofWeeks(1), + Period.ofMonths(1), + Period.ofMonths(3), + Period.ofYears(1), + Period.ofYears(5) }; private long[] dataPointIntervals = new long[] { DateTimeHelper.ONE_HOUR, @@ -152,7 +155,7 @@ public class UptimeDocumentWriter implements DocumentWriter { boolean relay, SortedSet<UptimeHistory> history, SortedSet<UptimeHistory> knownStatuses, long lastSeenMillis, String flag) { - long graphInterval = this.graphIntervals[graphIntervalIndex]; + Period graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; int dataPointIntervalHours = (int) (dataPointInterval @@ -160,7 +163,10 @@ public class UptimeDocumentWriter implements DocumentWriter { List<Integer> uptimeDataPoints = new ArrayList<>(); long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; - long graphStartMillis = graphEndMillis - graphInterval; + long graphStartMillis = LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphInterval) + .toEpochSecond(ZoneOffset.UTC) * 1000L; long intervalStartMillis = graphStartMillis; int uptimeHours = 0; long firstStatusStartMillis = -1L; @@ -285,9 +291,10 @@ public class UptimeDocumentWriter implements DocumentWriter { } long firstDataPointMillis = graphStartMillis + firstNonNullIndex * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) - * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { + if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphIntervals[graphIntervalIndex - 1]) + .toEpochSecond(ZoneOffset.UTC) * 1000L) { /* Skip uptime history object, because it doesn't contain * anything new that wasn't already contained in the last * uptime history object(s). */ diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index 10d8a94..f4e2c3a 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -15,6 +15,9 @@ import org.torproject.onionoo.docs.WeightsStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDateTime; +import java.time.Period; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -68,12 +71,12 @@ public class WeightsDocumentWriter implements DocumentWriter { "1_year", "5_years" }; - private long[] graphIntervals = new long[] { - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; + private Period[] graphIntervals = new Period[] { + Period.ofWeeks(1), + Period.ofMonths(1), + Period.ofMonths(3), + Period.ofYears(1), + Period.ofYears(5) }; private long[] dataPointIntervals = new long[] { DateTimeHelper.ONE_HOUR, @@ -118,13 +121,16 @@ public class WeightsDocumentWriter implements DocumentWriter { private GraphHistory compileWeightsHistory(int graphTypeIndex, int graphIntervalIndex, SortedMap<long[], double[]> history, long lastSeenMillis) { - long graphInterval = this.graphIntervals[graphIntervalIndex]; + Period graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; List<Double> dataPoints = new ArrayList<>(); long graphEndMillis = ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) * dataPointInterval; - long graphStartMillis = graphEndMillis - graphInterval; + long graphStartMillis = LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphInterval) + .toEpochSecond(ZoneOffset.UTC) * 1000L; long intervalStartMillis = graphStartMillis; long totalMillis = 0L; double totalWeightTimesMillis = 0.0; @@ -175,9 +181,10 @@ public class WeightsDocumentWriter implements DocumentWriter { } long firstDataPointMillis = graphStartMillis + firstNonNullIndex * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - ((lastSeenMillis + DateTimeHelper.ONE_HOUR) / dataPointInterval) - * dataPointInterval - graphIntervals[graphIntervalIndex - 1]) { + if (graphIntervalIndex > 0 && firstDataPointMillis >= LocalDateTime + .ofEpochSecond(graphEndMillis / 1000L, 0, ZoneOffset.UTC) + .minus(graphIntervals[graphIntervalIndex - 1]) + .toEpochSecond(ZoneOffset.UTC) * 1000L) { /* Skip weights history object, because it doesn't contain * anything new that wasn't already contained in the last * weights history object(s). */

1 0

[onionoo/release] Use the same last-seen time for writing documents.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 5fbc7fd5d6e6510e9e02f18bda8dec0e6bd2c8d1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Feb 2 15:27:27 2018 +0100 Use the same last-seen time for writing documents. Previously, we'd have used the last-seen time of the relay/bridge that we're writing a document for. But that doesn't make much sense, because we don't have to provide, e.g., a 1 week graph for a relay that has been offline for years. This change is going to shrink the out/ directory to a similar size as it was when we used current system time for writing documents. --- .../onionoo/writer/BandwidthDocumentWriter.java | 20 ++++++---------- .../onionoo/writer/ClientsDocumentWriter.java | 16 ++++--------- .../onionoo/writer/DetailsDocumentWriter.java | 2 +- .../torproject/onionoo/writer/DocumentWriter.java | 2 +- .../onionoo/writer/DocumentWriterRunner.java | 18 +++++++++++++- .../onionoo/writer/SummaryDocumentWriter.java | 2 +- .../onionoo/writer/UptimeDocumentWriter.java | 12 ++++------ .../onionoo/writer/WeightsDocumentWriter.java | 9 +------ .../writer/BandwidthDocumentWriterTest.java | 9 ++----- .../onionoo/writer/DetailsDocumentWriterTest.java | 8 +++---- .../onionoo/writer/UptimeDocumentWriterTest.java | 28 ++++++++++------------ 11 files changed, 55 insertions(+), 71 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index 71595e2..7089c06 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.slf4j.Logger; @@ -32,7 +31,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { } @Override - public void writeDocuments() { + public void writeDocuments(long lastSeenMillis) { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); long updatedMillis = updateStatus != null @@ -40,18 +39,13 @@ public class BandwidthDocumentWriter implements DocumentWriter { SortedSet<String> updateBandwidthDocuments = this.documentStore.list( BandwidthStatus.class, updatedMillis); for (String fingerprint : updateBandwidthDocuments) { - NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (null == nodeStatus) { - continue; - } BandwidthStatus bandwidthStatus = this.documentStore.retrieve( BandwidthStatus.class, true, fingerprint); if (bandwidthStatus == null) { continue; } BandwidthDocument bandwidthDocument = this.compileBandwidthDocument( - fingerprint, nodeStatus, bandwidthStatus); + fingerprint, lastSeenMillis, bandwidthStatus); this.documentStore.store(bandwidthDocument, fingerprint); } log.info("Wrote bandwidth document files"); @@ -59,13 +53,13 @@ public class BandwidthDocumentWriter implements DocumentWriter { private BandwidthDocument compileBandwidthDocument(String fingerprint, - NodeStatus nodeStatus, BandwidthStatus bandwidthStatus) { + long lastSeenMillis, BandwidthStatus bandwidthStatus) { BandwidthDocument bandwidthDocument = new BandwidthDocument(); bandwidthDocument.setFingerprint(fingerprint); - bandwidthDocument.setWriteHistory(this.compileGraphType( - nodeStatus.getLastSeenMillis(), bandwidthStatus.getWriteHistory())); - bandwidthDocument.setReadHistory(this.compileGraphType( - nodeStatus.getLastSeenMillis(), bandwidthStatus.getReadHistory())); + bandwidthDocument.setWriteHistory(this.compileGraphType(lastSeenMillis, + bandwidthStatus.getWriteHistory())); + bandwidthDocument.setReadHistory(this.compileGraphType(lastSeenMillis, + bandwidthStatus.getReadHistory())); return bandwidthDocument; } diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index aba45cf..7ed2048 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.ClientsStatus; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.FormattingUtils; @@ -56,7 +55,7 @@ public class ClientsDocumentWriter implements DocumentWriter { private int writtenDocuments = 0; @Override - public void writeDocuments() { + public void writeDocuments(long lastSeenMillis) { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); long updatedMillis = updateStatus != null @@ -64,11 +63,6 @@ public class ClientsDocumentWriter implements DocumentWriter { SortedSet<String> updateDocuments = this.documentStore.list( ClientsStatus.class, updatedMillis); for (String hashedFingerprint : updateDocuments) { - NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, - true, hashedFingerprint); - if (null == nodeStatus) { - continue; - } ClientsStatus clientsStatus = this.documentStore.retrieve( ClientsStatus.class, true, hashedFingerprint); if (clientsStatus == null) { @@ -76,7 +70,7 @@ public class ClientsDocumentWriter implements DocumentWriter { } SortedSet<ClientsHistory> history = clientsStatus.getHistory(); ClientsDocument clientsDocument = this.compileClientsDocument( - hashedFingerprint, nodeStatus, history); + hashedFingerprint, lastSeenMillis, history); this.documentStore.store(clientsDocument, hashedFingerprint); this.writtenDocuments++; } @@ -105,11 +99,11 @@ public class ClientsDocumentWriter implements DocumentWriter { DateTimeHelper.TEN_DAYS }; private ClientsDocument compileClientsDocument(String hashedFingerprint, - NodeStatus nodeStatus, SortedSet<ClientsHistory> history) { + long lastSeenMillis, SortedSet<ClientsHistory> history) { ClientsDocument clientsDocument = new ClientsDocument(); clientsDocument.setFingerprint(hashedFingerprint); - GraphHistoryCompiler ghc = new GraphHistoryCompiler( - nodeStatus.getLastSeenMillis() + DateTimeHelper.ONE_HOUR); + GraphHistoryCompiler ghc = new GraphHistoryCompiler(lastSeenMillis + + DateTimeHelper.ONE_HOUR); ghc.setThreshold(2L); for (int i = 0; i < this.graphIntervals.length; i++) { ghc.addGraphType(this.graphNames[i], this.graphIntervals[i], diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index bb81ca9..7ffaa07 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -32,7 +32,7 @@ public class DetailsDocumentWriter implements DocumentWriter { } @Override - public void writeDocuments() { + public void writeDocuments(long lastSeenMillis) { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); long updatedMillis = updateStatus != null diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java index 618a8e7..359de6f 100644 --- a/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriter.java @@ -5,7 +5,7 @@ package org.torproject.onionoo.writer; public interface DocumentWriter { - public abstract void writeDocuments(); + public abstract void writeDocuments(long lastSeenMillis); public abstract String getStatsString(); } diff --git a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java index dc3462c..968ab72 100644 --- a/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java +++ b/src/main/java/org/torproject/onionoo/writer/DocumentWriterRunner.java @@ -3,6 +3,10 @@ package org.torproject.onionoo.writer; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.NodeStatus; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +32,22 @@ public class DocumentWriterRunner { /** Lets each configured document writer write its documents. */ public void writeDocuments() { + long lastSeenMillis = retrieveLastSeenMillis(); for (DocumentWriter dw : this.documentWriters) { log.debug("Writing " + dw.getClass().getSimpleName()); - dw.writeDocuments(); + dw.writeDocuments(lastSeenMillis); + } + } + + private long retrieveLastSeenMillis() { + DocumentStore documentStore = DocumentStoreFactory.getDocumentStore(); + long lastSeenMillis = -1L; + for (String fingerprint : documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = documentStore.retrieve( + NodeStatus.class, true, fingerprint); + lastSeenMillis = Math.max(lastSeenMillis, nodeStatus.getLastSeenMillis()); } + return lastSeenMillis; } /** Logs statistics of all configured document writers. */ diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 813be1b..74ef31c 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -33,7 +33,7 @@ public class SummaryDocumentWriter implements DocumentWriter { private int deletedDocuments = 0; @Override - public void writeDocuments() { + public void writeDocuments(long lastSeenAllRelaysAndBridgesMillis) { long relaysLastValidAfterMillis = -1L; long bridgesLastPublishedMillis = -1L; for (String fingerprint : this.documentStore.list(NodeStatus.class)) { diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index 12ba8fa..db09e3d 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -7,7 +7,6 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeHistory; @@ -37,7 +36,7 @@ public class UptimeDocumentWriter implements DocumentWriter { } @Override - public void writeDocuments() { + public void writeDocuments(long lastSeenMillis) { UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true); if (uptimeStatus == null) { @@ -51,20 +50,18 @@ public class UptimeDocumentWriter implements DocumentWriter { SortedSet<String> updatedUptimeStatuses = this.documentStore.list( UptimeStatus.class, updatedMillis); for (String fingerprint : updatedUptimeStatuses) { - this.updateDocument(fingerprint, uptimeStatus); + this.updateDocument(fingerprint, lastSeenMillis, uptimeStatus); } log.info("Wrote uptime document files"); } private int writtenDocuments = 0; - private void updateDocument(String fingerprint, + private void updateDocument(String fingerprint, long lastSeenMillis, UptimeStatus knownStatuses) { - NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true, fingerprint); - if (null != nodeStatus && null != uptimeStatus) { + if (null != uptimeStatus) { boolean relay = uptimeStatus.getBridgeHistory().isEmpty(); SortedSet<UptimeHistory> history = relay ? uptimeStatus.getRelayHistory() @@ -72,7 +69,6 @@ public class UptimeDocumentWriter implements DocumentWriter { SortedSet<UptimeHistory> knownStatusesHistory = relay ? knownStatuses.getRelayHistory() : knownStatuses.getBridgeHistory(); - long lastSeenMillis = nodeStatus.getLastSeenMillis(); UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, fingerprint, history, knownStatusesHistory, lastSeenMillis); this.documentStore.store(uptimeDocument, fingerprint); diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index b34a9e6..2d2239b 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -7,7 +7,6 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.WeightsDocument; import org.torproject.onionoo.docs.WeightsStatus; @@ -32,7 +31,7 @@ public class WeightsDocumentWriter implements DocumentWriter { } @Override - public void writeDocuments() { + public void writeDocuments(long lastSeenMillis) { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); long updatedMillis = updateStatus != null @@ -40,18 +39,12 @@ public class WeightsDocumentWriter implements DocumentWriter { SortedSet<String> updateWeightsDocuments = this.documentStore.list( WeightsStatus.class, updatedMillis); for (String fingerprint : updateWeightsDocuments) { - NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (null == nodeStatus) { - continue; - } WeightsStatus weightsStatus = this.documentStore.retrieve( WeightsStatus.class, true, fingerprint); if (weightsStatus == null) { continue; } SortedMap<long[], double[]> history = weightsStatus.getHistory(); - long lastSeenMillis = nodeStatus.getLastSeenMillis(); WeightsDocument weightsDocument = this.compileWeightsDocument( fingerprint, history, lastSeenMillis); this.documentStore.store(weightsDocument, fingerprint); diff --git a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java index fa76699..947eba9 100644 --- a/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/BandwidthDocumentWriterTest.java @@ -12,7 +12,6 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.DummyDocumentStore; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.docs.NodeStatus; import org.junit.Before; import org.junit.Test; @@ -57,17 +56,13 @@ public class BandwidthDocumentWriterTest { + "r " + future + "-08-06 17:31:45 " + future + "-08-06 21:31:45 0\n" + "r " + future + "-08-06 21:31:45 " + future + "-08-07 01:31:45 0\n" + "r " + future + "-08-07 01:31:45 " + future + "-08-07 05:31:45 0\n"; - NodeStatus nodeStatus = new NodeStatus(ibibUnc0Fingerprint); - nodeStatus.setLastSeenMillis(DateTimeHelper.parse( - yesterday + " 12:00:00")); - this.documentStore.addDocument(nodeStatus, ibibUnc0Fingerprint); BandwidthStatus status = new BandwidthStatus(); status.setFromDocumentString(documentString); this.documentStore.addDocument(status, ibibUnc0Fingerprint); BandwidthDocumentWriter writer = new BandwidthDocumentWriter(); - writer.writeDocuments(); + writer.writeDocuments(DateTimeHelper.parse(yesterday + " 12:00:00")); assertEquals(1, this.documentStore.getPerformedListOperations()); - assertEquals(3, this.documentStore.getPerformedRetrieveOperations()); + assertEquals(2, this.documentStore.getPerformedRetrieveOperations()); assertEquals(1, this.documentStore.getPerformedStoreOperations()); BandwidthDocument document = this.documentStore.getDocument( BandwidthDocument.class, ibibUnc0Fingerprint); diff --git a/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java index 3356a9c..d96af71 100644 --- a/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java @@ -28,7 +28,7 @@ public class DetailsDocumentWriterTest { @Test public void testNoDetailsStatuses() { DetailsDocumentWriter writer = new DetailsDocumentWriter(); - writer.writeDocuments(); + writer.writeDocuments(-1L); assertEquals("Without statuses, no documents should be written.", 0, this.documentStore.getPerformedStoreOperations()); } @@ -48,7 +48,7 @@ public class DetailsDocumentWriterTest { GABELMOO_OR_ADDRESS))); this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); DetailsDocumentWriter writer = new DetailsDocumentWriter(); - writer.writeDocuments(); + writer.writeDocuments(-1L); assertEquals("One document should be written.", 1, this.documentStore.getPerformedStoreOperations()); DetailsDocument document = this.documentStore.getDocument( @@ -67,7 +67,7 @@ public class DetailsDocumentWriterTest { GABELMOO_OR_ADDRESS))); this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); DetailsDocumentWriter writer = new DetailsDocumentWriter(); - writer.writeDocuments(); + writer.writeDocuments(-1L); assertEquals("One document should be written.", 1, this.documentStore.getPerformedStoreOperations()); DetailsDocument document = this.documentStore.getDocument( @@ -85,7 +85,7 @@ public class DetailsDocumentWriterTest { status.setAdvertisedOrAddresses(Arrays.asList(GABELMOO_OR_ADDRESS)); this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); DetailsDocumentWriter writer = new DetailsDocumentWriter(); - writer.writeDocuments(); + writer.writeDocuments(-1L); assertEquals("One document should be written.", 1, this.documentStore.getPerformedStoreOperations()); DetailsDocument document = this.documentStore.getDocument( diff --git a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java index 030d100..2915a04 100644 --- a/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/writer/UptimeDocumentWriterTest.java @@ -10,7 +10,6 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.DummyDocumentStore; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeStatus; import org.torproject.onionoo.updater.DescriptorSourceFactory; @@ -37,7 +36,7 @@ public class UptimeDocumentWriterTest { @Test public void testNoStatuses() { UptimeDocumentWriter writer = new UptimeDocumentWriter(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Without providing any data, nothing should be written " + "to disk.", 0, this.documentStore.getPerformedStoreOperations()); @@ -50,9 +49,6 @@ public class UptimeDocumentWriterTest { private void addStatusOneWeekSample(String allRelaysUptime, String gabelmooUptime) { - NodeStatus nodeStatus = new NodeStatus(GABELMOO_FINGERPRINT); - nodeStatus.setLastSeenMillis(TEST_TIME); - this.documentStore.addDocument(nodeStatus, GABELMOO_FINGERPRINT); UptimeStatus status = new UptimeStatus(); status.setFromDocumentString(allRelaysUptime); this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT); @@ -119,7 +115,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -134,7 +130,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-23-10 2\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -149,7 +145,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-23-09 1\nr 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -164,7 +160,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-23-09 1\nr 2014-03-23-11 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -180,7 +176,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-23-09 2\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -196,7 +192,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-16-13 168\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -211,7 +207,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-16-12 169\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -226,7 +222,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-16-11 5\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -241,7 +237,7 @@ public class UptimeDocumentWriterTest { "r 2014-03-16-10 1\nr 2014-03-16-12 1\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -260,7 +256,7 @@ public class UptimeDocumentWriterTest { "r 2012-03-01-00 287\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument( @@ -278,7 +274,7 @@ public class UptimeDocumentWriterTest { "r 2012-03-01-00 288\n"); UptimeDocumentWriter writer = new UptimeDocumentWriter(); DescriptorSourceFactory.getDescriptorSource().readDescriptors(); - writer.writeDocuments(); + writer.writeDocuments(TEST_TIME); assertEquals("Should write exactly one document.", 1, this.documentStore.getPerformedStoreOperations()); UptimeDocument document = this.documentStore.getDocument(

1 0

[onionoo/release] Use constants consistently.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit 372ddc7b7e74ccb3c3cf8f870419ec239585c9fd Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jan 17 19:29:12 2018 +0000 Use constants consistently. Avoid multiple definitions of date-time related constants. --- src/main/java/org/torproject/onionoo/server/NodeIndexer.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 1a85a35..69baa53 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -3,6 +3,9 @@ package org.torproject.onionoo.server; +import static org.torproject.onionoo.docs.DateTimeHelper.ONE_DAY; +import static org.torproject.onionoo.docs.DateTimeHelper.ONE_MINUTE; + import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.SummaryDocument; @@ -97,10 +100,6 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } - private static final long ONE_MINUTE = 60L * 1000L; - - private static final long ONE_DAY = 24L * 60L * ONE_MINUTE; - @Override public void run() { try {

1 0

[onionoo/release] Change 3 month bandwidth graph to 24 hours detail.
by karsten＠torproject.org 07 Feb '18

07 Feb '18

commit bc90de89712bff8a62290adfa61667c013380504 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Feb 2 15:33:21 2018 +0100 Change 3 month bandwidth graph to 24 hours detail. Relays and bridges have recently changed their bandwidth statistics reporting interval from 4 hours to 24 hours. As a result, our 3 month bandwidth graph cannot accommodate new bandwidth statistics anymore. Increasing the bandwidth graph interval to 24 hours changes this, by reducing data resolution from 184 to 92 data points. Fixes part of #24729. --- .../java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index 7089c06..ad66bde 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -83,7 +83,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { DateTimeHelper.FIFTEEN_MINUTES, DateTimeHelper.ONE_HOUR, DateTimeHelper.FOUR_HOURS, - DateTimeHelper.TWELVE_HOURS, + DateTimeHelper.ONE_DAY, DateTimeHelper.TWO_DAYS, DateTimeHelper.TEN_DAYS };

1 0