commit 43434dc5f4b8f9c588accc6c4e8e0b8981becc98 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Mar 19 22:44:20 2015 +0100
Add fractional time of having a flag assigned.
Implements #15177. --- build.xml | 4 +- .../torproject/onionoo/docs/UptimeDocument.java | 10 ++ .../org/torproject/onionoo/docs/UptimeHistory.java | 37 ++++-- .../org/torproject/onionoo/docs/UptimeStatus.java | 74 +++++++----- .../torproject/onionoo/server/ResponseBuilder.java | 2 +- .../onionoo/updater/UptimeStatusUpdater.java | 94 +++++++++++---- .../onionoo/writer/UptimeDocumentWriter.java | 42 ++++++- .../torproject/onionoo/docs/UptimeStatusTest.java | 123 +++++++++++++++----- .../torproject/onionoo/updater/DummyConsensus.java | 7 +- .../onionoo/updater/UptimeStatusUpdaterTest.java | 5 +- web/protocol.html | 19 +++ 11 files changed, 319 insertions(+), 98 deletions(-)
diff --git a/build.xml b/build.xml index a3f105e..5986416 100644 --- a/build.xml +++ b/build.xml @@ -1,8 +1,8 @@ <project default="dist" name="onionoo" basedir=".">
- <property name="onionoo.protocol.version" value="2.2"/> + <property name="onionoo.protocol.version" value="2.3"/> <property name="release.version" - value="${onionoo.protocol.version}.1"/> + value="${onionoo.protocol.version}.0"/> <property name="javasources" value="src/main/java"/> <property name="tests" value="src/test/java"/> <property name="classes" value="classes"/> diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java index 7f0bacc..9b84a31 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java @@ -3,6 +3,7 @@ package org.torproject.onionoo.docs;
import java.util.Map; +import java.util.SortedMap;
public class UptimeDocument extends Document {
@@ -19,5 +20,14 @@ public class UptimeDocument extends Document { public Map<String, GraphHistory> getUptime() { return this.uptime; } + + private SortedMap<String, Map<String, GraphHistory>> flags; + public void setFlags( + SortedMap<String, Map<String, GraphHistory>> flags) { + this.flags = flags; + } + public SortedMap<String, Map<String, GraphHistory>> getFlags() { + return this.flags; + } }
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java index b686c7e..e98f72a 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java @@ -1,5 +1,8 @@ package org.torproject.onionoo.docs;
+import java.util.SortedSet; +import java.util.TreeSet; + import org.slf4j.Logger; import org.slf4j.LoggerFactory;
@@ -24,27 +27,33 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return this.uptimeHours; }
+ private SortedSet<String> flags; + public SortedSet<String> getFlags() { + return this.flags; + } + UptimeHistory(boolean relay, long startMillis, - int uptimeHours) { + int uptimeHours, SortedSet<String> flags) { this.relay = relay; this.startMillis = startMillis; this.uptimeHours = uptimeHours; + this.flags = flags; }
public static UptimeHistory fromString(String uptimeHistoryString) { - String[] parts = uptimeHistoryString.split(" ", 3); - if (parts.length != 3) { + String[] parts = uptimeHistoryString.split(" ", -1); + if (parts.length < 3) { log.warn("Invalid number of space-separated strings in uptime " + "history: '" + uptimeHistoryString + "'. Skipping"); return null; } boolean relay = false; - if (parts[0].equals("r")) { + if (parts[0].equalsIgnoreCase("r")) { relay = true; } else if (!parts[0].equals("b")) { log.warn("Invalid node type in uptime history: '" - + uptimeHistoryString + "'. Supported types are 'r' and 'b'. " - + "Skipping."); + + uptimeHistoryString + "'. Supported types are 'r', 'R', and " + + "'b'. Skipping."); return null; } long startMillis = DateTimeHelper.parse(parts[1], @@ -62,15 +71,27 @@ public class UptimeHistory implements Comparable<UptimeHistory> { + uptimeHistoryString + "'. Skipping."); return null; } - return new UptimeHistory(relay, startMillis, uptimeHours); + SortedSet<String> flags = null; + if (parts[0].equals("R")) { + flags = new TreeSet<String>(); + for (int i = 3; i < parts.length; i++) { + flags.add(parts[i]); + } + } + return new UptimeHistory(relay, startMillis, uptimeHours, flags); }
public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(this.relay ? "r" : "b"); + sb.append(this.relay ? (this.flags == null ? "r" : "R") : "b"); sb.append(" " + DateTimeHelper.format(this.startMillis, DateTimeHelper.DATEHOUR_NOSPACE_FORMAT)); sb.append(" " + String.format("%d", this.uptimeHours)); + if (this.flags != null) { + for (String flag : this.flags) { + sb.append(" " + flag); + } + } return sb.toString(); }
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java index e48fd09..266a245 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java @@ -2,6 +2,7 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.docs;
+import java.util.NavigableSet; import java.util.Scanner; import java.util.SortedSet; import java.util.TreeSet; @@ -24,18 +25,12 @@ public class UptimeStatus extends Document {
private SortedSet<UptimeHistory> relayHistory = new TreeSet<UptimeHistory>(); - public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) { - this.relayHistory = relayHistory; - } public SortedSet<UptimeHistory> getRelayHistory() { return this.relayHistory; }
private SortedSet<UptimeHistory> bridgeHistory = new TreeSet<UptimeHistory>(); - public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) { - this.bridgeHistory = bridgeHistory; - } public SortedSet<UptimeHistory> getBridgeHistory() { return this.bridgeHistory; } @@ -59,30 +54,50 @@ public class UptimeStatus extends Document { s.close(); }
- public void addToHistory(boolean relay, SortedSet<Long> newIntervals) { - for (long startMillis : newIntervals) { - SortedSet<UptimeHistory> history = relay ? this.relayHistory - : this.bridgeHistory; - UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); - if (!history.headSet(interval).isEmpty()) { - UptimeHistory prev = history.headSet(interval).last(); - if (prev.isRelay() == interval.isRelay() && - prev.getStartMillis() + DateTimeHelper.ONE_HOUR - * prev.getUptimeHours() > interval.getStartMillis()) { - continue; - } + public void addToHistory(boolean relay, long startMillis, + SortedSet<String> flags) { + SortedSet<UptimeHistory> history = relay ? this.relayHistory + : this.bridgeHistory; + UptimeHistory interval = new UptimeHistory(relay, startMillis, 1, + flags); + NavigableSet<UptimeHistory> existingIntervals = + new TreeSet<UptimeHistory>(history.headSet(new UptimeHistory( + relay, startMillis + DateTimeHelper.ONE_HOUR, 0, flags))); + for (UptimeHistory prev : existingIntervals.descendingSet()) { + if (prev.isRelay() != interval.isRelay() || + prev.getStartMillis() + DateTimeHelper.ONE_HOUR + * prev.getUptimeHours() <= interval.getStartMillis()) { + break; } - if (!history.tailSet(interval).isEmpty()) { - UptimeHistory next = history.tailSet(interval).first(); - if (next.isRelay() == interval.isRelay() && - next.getStartMillis() < interval.getStartMillis() - + DateTimeHelper.ONE_HOUR) { - continue; - } + if (prev.getFlags() == interval.getFlags() || + (prev.getFlags() != null && interval.getFlags() != null && + prev.getFlags().equals(interval.getFlags()))) { + /* The exact same interval is already contained in history. */ + return; + } + /* There is an interval that includes the new interval, but it + * contains different flags. Remove the old interval, put in any + * parts before or after the new interval, and add the new interval + * further down below. */ + history.remove(prev); + int hoursBefore = (int) ((interval.getStartMillis() + - prev.getStartMillis()) / DateTimeHelper.ONE_HOUR); + if (hoursBefore > 0) { + history.add(new UptimeHistory(relay, + prev.getStartMillis(), hoursBefore, prev.getFlags())); + } + int hoursAfter = (int) (prev.getStartMillis() + / DateTimeHelper.ONE_HOUR + prev.getUptimeHours() - + interval.getStartMillis() / DateTimeHelper.ONE_HOUR - 1); + if (hoursAfter > 0) { + history.add(new UptimeHistory(relay, + interval.getStartMillis() + DateTimeHelper.ONE_HOUR, + hoursAfter, prev.getFlags())); } - history.add(interval); - this.isDirty = true; + break; } + history.add(interval); + this.isDirty = true; }
public void compressHistory() { @@ -99,7 +114,10 @@ public class UptimeStatus extends Document { if (lastInterval != null && lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR * lastInterval.getUptimeHours() == interval.getStartMillis() && - lastInterval.isRelay() == interval.isRelay()) { + lastInterval.isRelay() == interval.isRelay() && + (lastInterval.getFlags() == interval.getFlags() || + (lastInterval.getFlags() != null && interval.getFlags() != null + && lastInterval.getFlags().equals(interval.getFlags())))) { lastInterval.addUptime(interval); } else { if (lastInterval != null) { diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index d1e7f23..af0f67e 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -70,7 +70,7 @@ public class ResponseBuilder { return this.charsWritten; }
- private static final String PROTOCOL_VERSION = "2.2"; + private static final String PROTOCOL_VERSION = "2.3";
private static final String NEXT_MAJOR_VERSION_SCHEDULED = null;
diff --git a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java index c3c98ed..e2cee78 100644 --- a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java @@ -2,6 +2,8 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.updater;
+import java.util.BitSet; +import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -48,41 +50,70 @@ public class UptimeStatusUpdater implements DescriptorListener, } }
- private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), - newBridgeStatuses = new TreeSet<Long>(); + private static class Flags { + + private static Map<String, Integer> flagIndexes = + new HashMap<String, Integer>(); + + private static Map<Integer, String> flagStrings = + new HashMap<Integer, String>(); + + private BitSet flags; + + private Flags(SortedSet<String> flags) { + this.flags = new BitSet(flagIndexes.size()); + for (String flag : flags) { + if (!flagIndexes.containsKey(flag)) { + flagStrings.put(flagIndexes.size(), flag); + flagIndexes.put(flag, flagIndexes.size()); + } + this.flags.set(flagIndexes.get(flag)); + } + } + + public SortedSet<String> getFlags() { + SortedSet<String> result = new TreeSet<String>(); + if (this.flags != null) { + for (int i = this.flags.nextSetBit(0); i >= 0; + i = this.flags.nextSetBit(i + 1)) { + result.add(flagStrings.get(i)); + } + } + return result; + } + } + + private SortedMap<Long, Flags> + newRelayStatuses = new TreeMap<Long, Flags>(); + private SortedMap<String, SortedMap<Long, Flags>> + newRunningRelays = new TreeMap<String, SortedMap<Long, Flags>>(); + private SortedSet<Long> newBridgeStatuses = new TreeSet<Long>(); private SortedMap<String, SortedSet<Long>> - newRunningRelays = new TreeMap<String, SortedSet<Long>>(), newRunningBridges = new TreeMap<String, SortedSet<Long>>();
private void processRelayNetworkStatusConsensus( RelayNetworkStatusConsensus consensus) { - SortedSet<String> fingerprints = new TreeSet<String>(); + long dateHourMillis = (consensus.getValidAfterMillis() + / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } - } - if (!fingerprints.isEmpty()) { - long dateHourMillis = (consensus.getValidAfterMillis() - / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; - for (String fingerprint : fingerprints) { - if (!this.newRunningRelays.containsKey(fingerprint)) { - this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); - } - this.newRunningRelays.get(fingerprint).add(dateHourMillis); + String fingerprint = entry.getFingerprint(); + if (!this.newRunningRelays.containsKey(fingerprint)) { + this.newRunningRelays.put(fingerprint, + new TreeMap<Long, Flags>()); } - this.newRelayStatuses.add(dateHourMillis); + this.newRunningRelays.get(fingerprint).put(dateHourMillis, + new Flags(entry.getFlags())); } + this.newRelayStatuses.put(dateHourMillis, + new Flags(consensus.getKnownFlags())); }
private void processBridgeNetworkStatus(BridgeNetworkStatus status) { SortedSet<String> fingerprints = new TreeSet<String>(); for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } + fingerprints.add(entry.getFingerprint()); } if (!fingerprints.isEmpty()) { long dateHourMillis = (status.getPublishedMillis() @@ -98,20 +129,30 @@ public class UptimeStatusUpdater implements DescriptorListener, }
public void updateStatuses() { - for (Map.Entry<String, SortedSet<Long>> e : + for (Map.Entry<String, SortedMap<Long, Flags>> e : this.newRunningRelays.entrySet()) { this.updateStatus(true, e.getKey(), e.getValue()); } this.updateStatus(true, null, this.newRelayStatuses); for (Map.Entry<String, SortedSet<Long>> e : this.newRunningBridges.entrySet()) { - this.updateStatus(false, e.getKey(), e.getValue()); + SortedMap<Long, Flags> dateHourMillisNoFlags = + new TreeMap<Long, Flags>(); + for (long dateHourMillis : e.getValue()) { + dateHourMillisNoFlags.put(dateHourMillis, null); + } + this.updateStatus(false, e.getKey(), dateHourMillisNoFlags); + } + SortedMap<Long, Flags> dateHourMillisNoFlags = + new TreeMap<Long, Flags>(); + for (long dateHourMillis : this.newBridgeStatuses) { + dateHourMillisNoFlags.put(dateHourMillis, null); } - this.updateStatus(false, null, this.newBridgeStatuses); + this.updateStatus(false, null, dateHourMillisNoFlags); }
private void updateStatus(boolean relay, String fingerprint, - SortedSet<Long> newUptimeHours) { + SortedMap<Long, Flags> dateHourMillisFlags) { UptimeStatus uptimeStatus = (fingerprint == null) ? this.documentStore.retrieve(UptimeStatus.class, true) : this.documentStore.retrieve(UptimeStatus.class, true, @@ -119,7 +160,10 @@ public class UptimeStatusUpdater implements DescriptorListener, if (uptimeStatus == null) { uptimeStatus = new UptimeStatus(); } - uptimeStatus.addToHistory(relay, newUptimeHours); + for (Map.Entry<Long, Flags> e : dateHourMillisFlags.entrySet()) { + uptimeStatus.addToHistory(relay, e.getKey(), + e.getValue() == null ? null : e.getValue().getFlags()); + } if (uptimeStatus.isDirty()) { uptimeStatus.compressHistory(); if (fingerprint == null) { diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index c5f71a1..d9eb91f 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -6,7 +6,10 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.SortedMap; import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet;
import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,18 +110,45 @@ public class UptimeDocumentWriter implements DocumentWriter { this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses); + graphIntervalIndex, relay, history, knownStatuses, null); if (graphHistory != null) { uptime.put(graphName, graphHistory); } } uptimeDocument.setUptime(uptime); + SortedMap<String, Map<String, GraphHistory>> flags = + new TreeMap<String, Map<String, GraphHistory>>(); + SortedSet<String> allFlags = new TreeSet<String>(); + for (UptimeHistory hist : history) { + if (hist.getFlags() != null) { + allFlags.addAll(hist.getFlags()); + } + } + for (String flag : allFlags) { + Map<String, GraphHistory> graphsForFlags = + new LinkedHashMap<String, GraphHistory>(); + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String graphName = this.graphNames[graphIntervalIndex]; + GraphHistory graphHistory = this.compileUptimeHistory( + graphIntervalIndex, relay, history, knownStatuses, flag); + if (graphHistory != null) { + graphsForFlags.put(graphName, graphHistory); + } + } + if (!graphsForFlags.isEmpty()) { + flags.put(flag, graphsForFlags); + } + } + if (!flags.isEmpty()) { + uptimeDocument.setFlags(flags); + } return uptimeDocument; }
private GraphHistory compileUptimeHistory(int graphIntervalIndex, boolean relay, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { + SortedSet<UptimeHistory> knownStatuses, String flag) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; @@ -130,7 +160,9 @@ public class UptimeDocumentWriter implements DocumentWriter { int uptimeHours = 0; long firstStatusStartMillis = -1L; for (UptimeHistory hist : history) { - if (hist.isRelay() != relay) { + if (hist.isRelay() != relay || + (flag != null && (hist.getFlags() == null || + !hist.getFlags().contains(flag)))) { continue; } if (firstStatusStartMillis < 0L) { @@ -170,7 +202,9 @@ public class UptimeDocumentWriter implements DocumentWriter { / dataPointInterval) * dataPointInterval; int statusHours = -1; for (UptimeHistory hist : knownStatuses) { - if (hist.isRelay() != relay) { + if (hist.isRelay() != relay || + (flag != null && (hist.getFlags() == null || + !hist.getFlags().contains(flag)))) { continue; } long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR diff --git a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java index b4bcdc8..7a48c25 100644 --- a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java @@ -7,12 +7,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue;
import java.util.Arrays; +import java.util.SortedSet; import java.util.TreeSet;
import org.junit.Test; -import org.torproject.onionoo.docs.DateTimeHelper; -import org.torproject.onionoo.docs.UptimeHistory; -import org.torproject.onionoo.docs.UptimeStatus;
public class UptimeStatusTest {
@@ -26,8 +24,8 @@ public class UptimeStatusTest { @Test() public void testSingleHourWriteToDisk() { UptimeStatus uptimeStatus = new UptimeStatus(); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-12-20 00:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 00:00:00"), null); uptimeStatus.compressHistory(); assertTrue("Changed uptime status should say it's dirty.", uptimeStatus.isDirty()); @@ -47,9 +45,10 @@ public class UptimeStatusTest { @Test() public void testTwoConsecutiveHours() { UptimeStatus uptimeStatus = new UptimeStatus(); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-12-20 00:00:00"), - DateTimeHelper.parse("2013-12-20 01:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 00:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 01:00:00"), null); uptimeStatus.compressHistory(); assertEquals("History must contain single entry.", 1, uptimeStatus.getRelayHistory().size()); @@ -73,9 +72,10 @@ public class UptimeStatusTest { public void testGabelmooFillInGaps() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-09-09 02:00:00"), - DateTimeHelper.parse("2013-12-20 00:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 02:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 00:00:00"), null); assertEquals("Uncompressed history must contain five entries.", 5, uptimeStatus.getRelayHistory().size()); uptimeStatus.compressHistory(); @@ -96,8 +96,8 @@ public class UptimeStatusTest { public void testAddExistingHourToIntervalStart() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 17:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 17:00:00"), null); uptimeStatus.compressHistory(); assertFalse("Unchanged history should not make uptime status dirty.", uptimeStatus.isDirty()); @@ -107,8 +107,8 @@ public class UptimeStatusTest { public void testAddExistingHourToIntervalEnd() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-09-09 01:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 01:00:00"), null); uptimeStatus.compressHistory(); assertFalse("Unchanged history should not make uptime status dirty.", uptimeStatus.isDirty()); @@ -118,9 +118,10 @@ public class UptimeStatusTest { public void testTwoHoursOverlappingWithIntervalStart() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"), - DateTimeHelper.parse("2013-07-22 17:00:00")}))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 16:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 17:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed history must still contain three entries.", 3, uptimeStatus.getRelayHistory().size()); @@ -139,9 +140,10 @@ public class UptimeStatusTest { public void testTwoHoursOverlappingWithIntervalEnd() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-09-09 01:00:00"), - DateTimeHelper.parse("2013-09-09 02:00:00")}))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 01:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 02:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed history must now contain two entries.", 2, uptimeStatus.getRelayHistory().size()); @@ -164,9 +166,10 @@ public class UptimeStatusTest { public void testAddRelayUptimeHours() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"), - DateTimeHelper.parse("2014-03-21 20:00:00")}))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 16:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed relay history must still contain one entry.", 1, uptimeStatus.getRelayHistory().size()); @@ -185,9 +188,10 @@ public class UptimeStatusTest { public void testAddBridgeUptimeHours() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); - uptimeStatus.addToHistory(false, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"), - DateTimeHelper.parse("2014-03-21 20:00:00")}))); + uptimeStatus.addToHistory(false, + DateTimeHelper.parse("2013-07-22 16:00:00"), null); + uptimeStatus.addToHistory(false, + DateTimeHelper.parse("2014-03-21 20:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed bridge history must still contain one " + "entry.", 1, uptimeStatus.getBridgeHistory().size()); @@ -201,5 +205,70 @@ public class UptimeStatusTest { assertEquals("History uptime hours not 1+5811+1=5813.", 5813, newUptimeHistory.getUptimeHours()); } + + private static final SortedSet<String> RUNNING_FLAG = + new TreeSet<String>(Arrays.asList(new String[] { "Running" })); + + @Test() + public void testAddFlagsToNoFlagsEnd() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Mixed relay history must not be compressed.", 2, + uptimeStatus.getRelayHistory().size()); + } + + @Test() + public void testAddFlagsToNoFlagsBegin() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 16:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Mixed relay history must not be compressed.", 2, + uptimeStatus.getRelayHistory().size()); + } + + @Test() + public void testAddFlagsToNoFlagsMiddle() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-20 12:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Mixed relay history must not be compressed.", 3, + uptimeStatus.getRelayHistory().size()); + } + + private static final String RELAYS_FLAGS_UPTIME_SAMPLE = + "R 2013-07-22-17 5811 Running\n"; /* ends 2014-03-21 20:00:00 */ + + @Test() + public void testAddFlagsToFlagsEnd() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_FLAGS_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Relay history with flags must be compressed.", 1, + uptimeStatus.getRelayHistory().size()); + } + + private static final SortedSet<String> RUNNING_VALID_FLAGS = + new TreeSet<String>(Arrays.asList(new String[] { "Running", + "Valid" })); + + @Test() + public void testDontCompressDifferentFlags() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_FLAGS_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_VALID_FLAGS); + uptimeStatus.compressHistory(); + assertEquals("Relay history with different flags must not be " + + "compressed.", 2, uptimeStatus.getRelayHistory().size()); + } }
diff --git a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java index 8f164a0..a2d1e3e 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet;
import org.torproject.descriptor.DirSourceEntry; import org.torproject.descriptor.DirectorySignature; @@ -70,8 +71,12 @@ public class DummyConsensus implements RelayNetworkStatusConsensus { return null; }
+ private SortedSet<String> knownFlags = new TreeSet<String>(); + public void addKnownFlag(String flag) { + this.knownFlags.add(flag); + } public SortedSet<String> getKnownFlags() { - return null; + return this.knownFlags; }
public SortedMap<String, Integer> getConsensusParams() { diff --git a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java index e74dc5c..25399ab 100644 --- a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java +++ b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java @@ -52,6 +52,7 @@ public class UptimeStatusUpdaterTest { statusEntry.addFlag("Running"); DummyConsensus consensus = new DummyConsensus(); consensus.setValidAfterMillis(VALID_AFTER_SAMPLE); + consensus.addKnownFlag("Running"); consensus.addStatusEntry(statusEntry); this.descriptorSource.addDescriptor(DescriptorType.RELAY_CONSENSUSES, consensus); @@ -83,7 +84,7 @@ public class UptimeStatusUpdaterTest { private static final String ALL_RELAYS_AND_BRIDGES_FINGERPRINT = null;
private static final String ALL_RELAYS_AND_BRIDGES_UPTIME_SAMPLE = - "r 2013-07-22-17 5811\n" /* ends 2014-03-21 20:00:00 */ + "R 2013-07-22-17 5811 Running\n" /* ends 2014-03-21 20:00:00 */ + "b 2013-07-22-17 5811\n"; /* ends 2014-03-21 20:00:00 */
private void addAllRelaysAndBridgesUptimeSample() { @@ -105,7 +106,7 @@ public class UptimeStatusUpdaterTest { 2, this.documentStore.getPerformedStoreOperations()); UptimeStatus status = this.documentStore.getDocument( UptimeStatus.class, ALL_RELAYS_AND_BRIDGES_FINGERPRINT); - assertEquals("Relay history must contain one entry", 1, + assertEquals("Relay history must contain one entry.", 1, status.getRelayHistory().size()); UptimeHistory history = status.getRelayHistory().first(); assertEquals("History not for relay.", true, history.isRelay()); diff --git a/web/protocol.html b/web/protocol.html index 2a13be8..c1dfa3f 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -174,6 +174,8 @@ field from details documents and optional "advertised_bandwidth" and <li><strong>2.2</strong>: Removed optional "pool_assignment" field and added "transports" field to bridge details documents on December 8, 2014.</li> +<li><strong>2.3</strong>: Added optional "flags" field to uptime +documents on March 22, 2015.</li> </ul>
</div> <!-- box --> @@ -2206,6 +2208,23 @@ of network statuses have been processed for a given time period. </p> </li>
+<li> +<b><font color="blue">flags</font></b> +<code class="typeof">object</code> +<span class="required-false">optional</span> +<p> +Object containing fractional times of this relay having relay flags +assigned. +Keys are flag names like <strong>"Running"</strong> or +<strong>"Exit"</strong>, values are objects similar to the +<strong>uptime</strong> field above, again with keys like +<strong>"1_week"</strong> etc. +If a relay never had a given relay flag assigned, no object is included +for that flag. +<font color="blue">Added on March 22, 2015.</font> +</p> +</li> + </ul>
<h4>Bridge uptime objects</h4>