commit 7c911367930d6aa4183090c13f8c2a9de4842f56 Author: Karsten Loesing karsten.loesing@gmx.net Date: Mon Mar 10 14:24:48 2014 +0100
Add new document with fractional uptimes.
Prepares for providing monthly relay and bridge statistics (#11041). --- etc/web.xml.template | 4 + src/org/torproject/onionoo/DocumentStore.java | 31 +- src/org/torproject/onionoo/Main.java | 4 +- src/org/torproject/onionoo/ResourceServlet.java | 2 + src/org/torproject/onionoo/ResponseBuilder.java | 28 ++ src/org/torproject/onionoo/UptimeDataWriter.java | 533 ++++++++++++++++++++++ src/org/torproject/onionoo/UptimeDocument.java | 8 + src/org/torproject/onionoo/UptimeStatus.java | 5 + web/index.html | 192 ++++++++ 9 files changed, 800 insertions(+), 7 deletions(-)
diff --git a/etc/web.xml.template b/etc/web.xml.template index 53a1878..f69314f 100644 --- a/etc/web.xml.template +++ b/etc/web.xml.template @@ -40,6 +40,10 @@ <servlet-name>Resource</servlet-name> <url-pattern>/clients</url-pattern> </servlet-mapping> + <servlet-mapping> + <servlet-name>Resource</servlet-name> + <url-pattern>/uptime</url-pattern> + </servlet-mapping>
</web-app>
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 1092e25..519b4f2 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -124,6 +124,9 @@ public class DocumentStore { } else if (documentType.equals(ClientsStatus.class)) { directory = this.statusDir; subdirectory = "clients"; + } else if (documentType.equals(UptimeStatus.class)) { + directory = this.statusDir; + subdirectory = "uptimes"; } else if (documentType.equals(DetailsDocument.class)) { directory = this.outDir; subdirectory = "details"; @@ -136,6 +139,9 @@ public class DocumentStore { } else if (documentType.equals(ClientsDocument.class)) { directory = this.outDir; subdirectory = "clients"; + } else if (documentType.equals(UptimeDocument.class)) { + directory = this.outDir; + subdirectory = "uptimes"; } if (directory != null && subdirectory != null) { Stack<File> files = new Stack<File>(); @@ -186,7 +192,8 @@ public class DocumentStore { } else if (document instanceof DetailsDocument || document instanceof BandwidthDocument || document instanceof WeightsDocument || - document instanceof ClientsDocument) { + document instanceof ClientsDocument || + document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); } else { @@ -275,7 +282,8 @@ public class DocumentStore { } else if (documentType.equals(DetailsDocument.class) || documentType.equals(BandwidthDocument.class) || documentType.equals(WeightsDocument.class) || - documentType.equals(ClientsDocument.class)) { + documentType.equals(ClientsDocument.class) || + documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); } else { @@ -351,12 +359,11 @@ public class DocumentStore { private <T extends Document> File getDocumentFile(Class<T> documentType, String fingerprint) { File documentFile = null; - if (fingerprint == null && - !documentType.equals(UpdateStatus.class)) { + if (fingerprint == null && !documentType.equals(UpdateStatus.class) && + !documentType.equals(UptimeStatus.class)) { // TODO Instead of using the update file workaround, add new method // lastModified(Class<T> documentType) that serves a similar - // purpose. Once that's implemented, make fingerprint mandatory for - // all methods. + // purpose. return null; } File directory = null; @@ -381,6 +388,15 @@ public class DocumentStore { fileName = String.format("clients/%s/%s/%s", fingerprint.substring(0, 1), fingerprint.substring(1, 2), fingerprint); + } else if (documentType.equals(UptimeStatus.class)) { + directory = this.statusDir; + if (fingerprint == null) { + fileName = "uptime"; + } else { + fileName = String.format("uptimes/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + } } else if (documentType.equals(UpdateStatus.class)) { directory = this.outDir; fileName = "update"; @@ -401,6 +417,9 @@ public class DocumentStore { } else if (documentType.equals(ClientsDocument.class)) { directory = this.outDir; fileName = String.format("clients/%s", fingerprint); + } else if (documentType.equals(UptimeDocument.class)) { + directory = this.outDir; + fileName = String.format("uptimes/%s", fingerprint); } if (directory != null && fileName != null) { documentFile = new File(directory, fileName); diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 87e7c9a..5631a5e 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -38,7 +38,9 @@ public class Main { Logger.printStatusTime("Initialized weights data writer"); ClientsDataWriter cdw = new ClientsDataWriter(dso, ds, t); Logger.printStatusTime("Initialized clients data writer"); - DataWriter[] dws = new DataWriter[] { ndw, bdw, wdw, cdw }; + UptimeDataWriter udw = new UptimeDataWriter(dso, ds, t); + Logger.printStatusTime("Initialized uptime data writer"); + DataWriter[] dws = new DataWriter[] { ndw, bdw, wdw, cdw, udw };
Logger.printStatus("Reading descriptors."); dso.readRelayNetworkConsensuses(); diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index d04828f..de47d44 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -130,6 +130,8 @@ public class ResourceServlet extends HttpServlet { resourceType = "weights"; } else if (uri.startsWith("/clients")) { resourceType = "clients"; + } else if (uri.startsWith("/uptime")) { + resourceType = "uptime"; } else { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index b7231db..b0775b1 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -776,6 +776,8 @@ public class ResponseBuilder { return this.writeWeightsLines(summaryLine); } else if (this.resourceType.equals("clients")) { return this.writeClientsLines(summaryLine); + } else if (this.resourceType.equals("uptime")) { + return this.writeUptimeLines(summaryLine); } else { return ""; } @@ -928,4 +930,30 @@ public class ResponseBuilder { return ""; } } + + private String writeUptimeLines(String summaryLine) { + String fingerprint = null; + if (summaryLine.contains(""f":"")) { + fingerprint = summaryLine.substring(summaryLine.indexOf( + ""f":"") + ""f":"".length()); + } else if (summaryLine.contains(""h":"")) { + fingerprint = summaryLine.substring(summaryLine.indexOf( + ""h":"") + ""h":"".length()); + } else { + return ""; + } + fingerprint = fingerprint.substring(0, 40); + UptimeDocument uptimeDocument = documentStore.retrieve( + UptimeDocument.class, false, fingerprint); + if (uptimeDocument != null && + uptimeDocument.documentString != null) { + String uptimeLines = uptimeDocument.documentString; + uptimeLines = uptimeLines.substring(0, uptimeLines.length() - 1); + return uptimeLines; + } else { + // TODO We should probably log that we didn't find an uptime + // document that we expected to exist. + return ""; + } + } } diff --git a/src/org/torproject/onionoo/UptimeDataWriter.java b/src/org/torproject/onionoo/UptimeDataWriter.java new file mode 100644 index 0000000..7a8b3af --- /dev/null +++ b/src/org/torproject/onionoo/UptimeDataWriter.java @@ -0,0 +1,533 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; + +public class UptimeDataWriter implements DataWriter, DescriptorListener { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public UptimeDataWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerListener(this, + DescriptorType.BRIDGE_STATUSES); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + this.processBridgeNetworkStatus( + (BridgeNetworkStatus) descriptor); + } + } + + private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), + newBridgeStatuses = new TreeSet<Long>(); + private SortedMap<String, SortedSet<Long>> + newRunningRelays = new TreeMap<String, SortedSet<Long>>(), + newRunningBridges = new TreeMap<String, SortedSet<Long>>(); + + private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L; + + private void processRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) { + SortedSet<String> fingerprints = new TreeSet<String>(); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + fingerprints.add(entry.getFingerprint()); + } + } + if (!fingerprints.isEmpty()) { + long dateHourMillis = (consensus.getValidAfterMillis() + / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS; + for (String fingerprint : fingerprints) { + if (!this.newRunningRelays.containsKey(fingerprint)) { + this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); + } + this.newRunningRelays.get(fingerprint).add(dateHourMillis); + } + this.newRelayStatuses.add(dateHourMillis); + } + } + + private void processBridgeNetworkStatus(BridgeNetworkStatus status) { + SortedSet<String> fingerprints = new TreeSet<String>(); + for (NetworkStatusEntry entry : + status.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + fingerprints.add(entry.getFingerprint()); + } + } + if (!fingerprints.isEmpty()) { + long dateHourMillis = (status.getPublishedMillis() + / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS; + for (String fingerprint : fingerprints) { + if (!this.newRunningBridges.containsKey(fingerprint)) { + this.newRunningBridges.put(fingerprint, new TreeSet<Long>()); + } + this.newRunningBridges.get(fingerprint).add(dateHourMillis); + } + this.newBridgeStatuses.add(dateHourMillis); + } + } + + public void updateStatuses() { + for (Map.Entry<String, SortedSet<Long>> e : + this.newRunningRelays.entrySet()) { + this.updateStatus(true, e.getKey(), e.getValue()); + } + this.updateStatus(true, null, this.newRelayStatuses); + for (Map.Entry<String, SortedSet<Long>> e : + this.newRunningBridges.entrySet()) { + this.updateStatus(false, e.getKey(), e.getValue()); + } + this.updateStatus(false, null, this.newBridgeStatuses); + Logger.printStatusTime("Updated uptime status files"); + } + + private static class UptimeHistory + implements Comparable<UptimeHistory> { + private boolean relay; + private long startMillis; + private int uptimeHours; + private UptimeHistory(boolean relay, long startMillis, + int uptimeHours) { + this.relay = relay; + this.startMillis = startMillis; + this.uptimeHours = uptimeHours; + } + public static UptimeHistory fromString(String uptimeHistoryString) { + String[] parts = uptimeHistoryString.split(" ", 3); + if (parts.length != 3) { + return null; + } + boolean relay = false; + if (parts[0].equals("r")) { + relay = true; + } else if (!parts[0].equals("b")) { + return null; + } + long startMillis = -1L; + SimpleDateFormat dateHourFormat = new SimpleDateFormat( + "yyyy-MM-dd-HH"); + dateHourFormat.setLenient(false); + dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + startMillis = dateHourFormat.parse(parts[1]).getTime(); + } catch (ParseException e) { + return null; + } + int uptimeHours = -1; + try { + uptimeHours = Integer.parseInt(parts[2]); + } catch (NumberFormatException e) { + return null; + } + return new UptimeHistory(relay, startMillis, uptimeHours); + } + public String toString() { + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateHourFormat = new SimpleDateFormat( + "yyyy-MM-dd-HH"); + dateHourFormat.setLenient(false); + dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append(this.relay ? "r" : "b"); + sb.append(" " + dateHourFormat.format(this.startMillis)); + sb.append(" " + String.format("%d", this.uptimeHours)); + return sb.toString(); + } + public void addUptime(UptimeHistory other) { + this.uptimeHours += other.uptimeHours; + if (this.startMillis > other.startMillis) { + this.startMillis = other.startMillis; + } + } + public int compareTo(UptimeHistory other) { + if (this.relay && !other.relay) { + return -1; + } else if (!this.relay && other.relay) { + return 1; + } + return this.startMillis < other.startMillis ? -1 : + this.startMillis > other.startMillis ? 1 : 0; + } + public boolean equals(Object other) { + return other instanceof UptimeHistory && + this.relay == ((UptimeHistory) other).relay && + this.startMillis == ((UptimeHistory) other).startMillis; + } + } + + private void updateStatus(boolean relay, String fingerprint, + SortedSet<Long> newUptimeHours) { + SortedSet<UptimeHistory> history = this.readHistory(fingerprint); + this.addToHistory(history, relay, newUptimeHours); + history = this.compressHistory(history); + this.writeHistory(fingerprint, history); + } + + private SortedSet<UptimeHistory> readHistory(String fingerprint) { + SortedSet<UptimeHistory> history = new TreeSet<UptimeHistory>(); + UptimeStatus uptimeStatus = fingerprint == null ? + documentStore.retrieve(UptimeStatus.class, false) : + documentStore.retrieve(UptimeStatus.class, false, fingerprint); + if (uptimeStatus != null) { + Scanner s = new Scanner(uptimeStatus.documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + UptimeHistory parsedLine = UptimeHistory.fromString(line); + if (parsedLine != null) { + history.add(parsedLine); + } else { + System.err.println("Could not parse uptime history line '" + + line + "' for fingerprint '" + fingerprint + + "'. Skipping."); + } + } + s.close(); + } + return history; + } + + private void addToHistory(SortedSet<UptimeHistory> history, + boolean relay, SortedSet<Long> newIntervals) { + for (long startMillis : newIntervals) { + UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); + if (!history.headSet(interval).isEmpty()) { + UptimeHistory prev = history.headSet(interval).last(); + if (prev.relay == interval.relay && + prev.startMillis + ONE_HOUR_MILLIS * prev.uptimeHours > + interval.startMillis) { + continue; + } + } + if (!history.tailSet(interval).isEmpty()) { + UptimeHistory next = history.tailSet(interval).first(); + if (next.relay == interval.relay && + next.startMillis < interval.startMillis + ONE_HOUR_MILLIS) { + continue; + } + } + history.add(interval); + } + } + + private SortedSet<UptimeHistory> compressHistory( + SortedSet<UptimeHistory> history) { + SortedSet<UptimeHistory> compressedHistory = + new TreeSet<UptimeHistory>(); + UptimeHistory lastInterval = null; + for (UptimeHistory interval : history) { + if (lastInterval != null && + lastInterval.startMillis + ONE_HOUR_MILLIS + * lastInterval.uptimeHours == interval.startMillis && + lastInterval.relay == interval.relay) { + lastInterval.addUptime(interval); + } else { + if (lastInterval != null) { + compressedHistory.add(lastInterval); + } + lastInterval = interval; + } + } + if (lastInterval != null) { + compressedHistory.add(lastInterval); + } + return compressedHistory; + } + + private void writeHistory(String fingerprint, + SortedSet<UptimeHistory> history) { + StringBuilder sb = new StringBuilder(); + for (UptimeHistory interval : history) { + sb.append(interval.toString() + "\n"); + } + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.documentString = sb.toString(); + if (fingerprint == null) { + this.documentStore.store(uptimeStatus); + } else { + this.documentStore.store(uptimeStatus, fingerprint); + } + } + + public void updateDocuments() { + SortedSet<UptimeHistory> + knownRelayStatuses = new TreeSet<UptimeHistory>(), + knownBridgeStatuses = new TreeSet<UptimeHistory>(); + SortedSet<UptimeHistory> knownStatuses = this.readHistory(null); + for (UptimeHistory status : knownStatuses) { + if (status.relay) { + knownRelayStatuses.add(status); + } else { + knownBridgeStatuses.add(status); + } + } + for (String fingerprint : this.newRunningRelays.keySet()) { + this.updateDocument(true, fingerprint, knownRelayStatuses); + } + for (String fingerprint : this.newRunningBridges.keySet()) { + this.updateDocument(false, fingerprint, knownBridgeStatuses); + } + Logger.printStatusTime("Wrote uptime document files"); + } + + private void updateDocument(boolean relay, String fingerprint, + SortedSet<UptimeHistory> knownStatuses) { + SortedSet<UptimeHistory> history = this.readHistory(fingerprint); + UptimeDocument uptimeDocument = new UptimeDocument(); + uptimeDocument.documentString = this.formatHistoryString(relay, + fingerprint, history, knownStatuses); + this.documentStore.store(uptimeDocument, fingerprint); + } + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + 7L * 24L * 60L * 60L * 1000L, + 31L * 24L * 60L * 60L * 1000L, + 92L * 24L * 60L * 60L * 1000L, + 366L * 24L * 60L * 60L * 1000L, + 5L * 366L * 24L * 60L * 60L * 1000L }; + + private long[] dataPointIntervals = new long[] { + 60L * 60L * 1000L, + 4L * 60L * 60L * 1000L, + 12L * 60L * 60L * 1000L, + 2L * 24L * 60L * 60L * 1000L, + 10L * 24L * 60L * 60L * 1000L }; + + private String formatHistoryString(boolean relay, String fingerprint, + SortedSet<UptimeHistory> history, + SortedSet<UptimeHistory> knownStatuses) { + StringBuilder sb = new StringBuilder(); + sb.append("{"fingerprint":"" + fingerprint + """); + sb.append(",\n"uptime":{"); + int graphIntervalsWritten = 0; + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String timeline = this.formatTimeline(graphIntervalIndex, relay, + history, knownStatuses); + if (timeline != null) { + sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" + + timeline); + } + } + sb.append("}"); + sb.append("\n}\n"); + return sb.toString(); + } + + private String formatTimeline(int graphIntervalIndex, boolean relay, + SortedSet<UptimeHistory> history, + SortedSet<UptimeHistory> knownStatuses) { + String graphName = this.graphNames[graphIntervalIndex]; + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + int dataPointIntervalHours = (int) (dataPointInterval + / ONE_HOUR_MILLIS); + List<Integer> statusDataPoints = new ArrayList<Integer>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + int statusHours = 0; + for (UptimeHistory hist : knownStatuses) { + if (hist.relay != relay) { + continue; + } + long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS + * hist.uptimeHours; + if (histEndMillis < intervalStartMillis) { + continue; + } + while (hist.startMillis >= intervalStartMillis + + dataPointInterval) { + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = 0; + intervalStartMillis += dataPointInterval; + } + while (histEndMillis >= intervalStartMillis + dataPointInterval) { + statusHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(hist.startMillis, intervalStartMillis)) + / ONE_HOUR_MILLIS); + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = 0; + intervalStartMillis += dataPointInterval; + } + statusHours += (int) ((histEndMillis - Math.max(hist.startMillis, + intervalStartMillis)) / ONE_HOUR_MILLIS); + } + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + List<Integer> uptimeDataPoints = new ArrayList<Integer>(); + intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + int uptimeHours = 0; + long firstStatusStartMillis = -1L; + for (UptimeHistory hist : history) { + if (hist.relay != relay) { + continue; + } + if (firstStatusStartMillis < 0L) { + firstStatusStartMillis = hist.startMillis; + } + long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS + * hist.uptimeHours; + if (histEndMillis < intervalStartMillis) { + continue; + } + while (hist.startMillis >= intervalStartMillis + + dataPointInterval) { + if (firstStatusStartMillis < intervalStartMillis + + dataPointInterval) { + uptimeDataPoints.add(uptimeHours); + } else { + uptimeDataPoints.add(-1); + } + uptimeHours = 0; + intervalStartMillis += dataPointInterval; + } + while (histEndMillis >= intervalStartMillis + dataPointInterval) { + uptimeHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(hist.startMillis, intervalStartMillis)) + / ONE_HOUR_MILLIS); + uptimeDataPoints.add(uptimeHours); + uptimeHours = 0; + intervalStartMillis += dataPointInterval; + } + uptimeHours += (int) ((histEndMillis - Math.max(hist.startMillis, + intervalStartMillis)) / ONE_HOUR_MILLIS); + } + uptimeDataPoints.add(uptimeHours); + List<Double> dataPoints = new ArrayList<Double>(); + for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); + dataPointIndex++) { + if (dataPointIndex >= uptimeDataPoints.size()) { + dataPoints.add(0.0); + } else if (uptimeDataPoints.get(dataPointIndex) >= 0 && + statusDataPoints.get(dataPointIndex) > 0) { + dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex)) + / ((double) statusDataPoints.get(dataPointIndex))); + } else { + dataPoints.add(-1.0); + } + } + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip uptime history object, because it doesn't contain + * anything new that wasn't already contained in the last + * uptime history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = 1.0 / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append(""" + graphName + "":{" + + ""first":"" + dateTimeFormat.format(firstDataPointMillis) + + "","last":"" + dateTimeFormat.format(lastDataPointMillis) + + "","interval":" + String.valueOf(dataPointInterval / 1000L) + + ","factor":" + String.format(Locale.US, "%.9f", factor) + + ","count":" + String.valueOf(count) + ","values":["); + int dataPointsWritten = 0, previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + sb.append((dataPointsWritten++ > 0 ? "," : "") + + (dataPoint < -0.5 ? "null" : + String.valueOf((long) (dataPoint * 999.0)))); + } + sb.append("]}"); + if (foundTwoAdjacentDataPoints) { + return sb.toString(); + } else { + return null; + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber( + this.newRelayStatuses.size()) + " hours of relay uptimes " + + "processed\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newBridgeStatuses.size()) + " hours of bridge uptimes " + + "processed\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newRunningRelays.size() + this.newRunningBridges.size()) + + " uptime status files updated\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newRunningRelays.size() + this.newRunningBridges.size()) + + " uptime document files updated\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/UptimeDocument.java b/src/org/torproject/onionoo/UptimeDocument.java new file mode 100644 index 0000000..f71cb87 --- /dev/null +++ b/src/org/torproject/onionoo/UptimeDocument.java @@ -0,0 +1,8 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +class UptimeDocument extends Document { + +} + diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java new file mode 100644 index 0000000..d16c8fc --- /dev/null +++ b/src/org/torproject/onionoo/UptimeStatus.java @@ -0,0 +1,5 @@ +package org.torproject.onionoo; + +class UptimeStatus extends Document { +} + diff --git a/web/index.html b/web/index.html index c07ca86..732ff8f 100644 --- a/web/index.html +++ b/web/index.html @@ -58,6 +58,7 @@ h3 .request-response { padding: 0 !important; } <li><a href="#bandwidth">Bandwidth documents</a></li> <li><a href="#weights">Weights documents</a></li> <li><a href="#clients">Clients documents</a></li> + <li><a href="#uptime">Uptime documents</a></li> </ul>
</div> @@ -1955,6 +1956,197 @@ It might be removed in the future without notice.</font>
</div> <!-- box -->
+<div class="box"> +<a name="uptime"></a> +<h3>Uptime documents <a href="#uptime">#</a> +<span class="request-response"> +<a href="uptime?limit=4">example request</a> +</span> +</h3> + +<p> +<font color="blue">Added on March 10, 2014.</font> +Uptime documents contain fractional uptimes of relays and bridges. +Uptime documents contain different time intervals and are available for +all relays and bridges that have been running in the past week. +Uptime documents contain the following fields: +</p> + +<ul class="properties"> + +<li> +<b>relays_published</b> +<code class="typeof">string</code> +<span class="required-true">required</span> +<p> +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known relay network status consensus started being valid. +Indicates how recent the relay uptime documents in this document are. +</p> +</li> + +<li> +<b>relays</b> +<code class="typeof">array of objects</code> +<span class="required-true">required</span> +<p> +Array of objects representing relay uptime documents. +Each array object contains the following key-value pairs: +</p> + +<ul class="properties"> + +<li> +<b>fingerprint</b> +<code class="typeof">string</code> +<span class="required-true">required</span> +<p> +Relay fingerprint consisting of 40 upper-case +hexadecimal characters. +</p> +</li> + +<li> +<b>uptime</b> +<code class="typeof">object</code> +<span class="required-false">optional</span> +<p> +History object containing the fractional uptime of this relay. +Keys are string representation of the time period covered by the uptime +history object. +Keys are fixed strings <strong>"1_week"</strong>, +<strong>"1_month"</strong>, <strong>"3_months"</strong>, +<strong>"1_year"</strong>, and <strong>"5_years"</strong>. +Keys refer to the last known uptime history of a relay, not to the time +when the uptime document was published. +An uptime history object is only contained if the time period it covers +is not already contained in another uptime history object with shorter +time period and higher data resolution. +Each uptime history object contains the following key-value pairs: +</p> + +<ul class="properties"> + +<li> +<b>first</b> +<code class="typeof">string</code> +<span class="required-true">required</span> +<p> +UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data +data point in the uptime history. +</p> +</li> + +<li> +<b>last</b> +<code class="typeof">string</code> +<span class="required-true">required</span> +<p> +UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data +data point in the uptime history. +</p> +</li> + +<li> +<b>interval</b> +<code class="typeof">number</code> +<span class="required-true">required</span> +<p> +Time interval between two data points in seconds. +</p> +</li> + +<li> +<b>factor</b> +<code class="typeof">number</code> +<span class="required-true">required</span> +<p> +Factor by which subsequent uptime values need to +be multiplied to get the fractional uptime. +This is only done for compatibility reasons with the other document types. +</p> +</li> + +<li> +<b>count</b> +<code class="typeof">number</code> +<span class="required-false">optional</span> +<p> +Number of provided data points, included mostly for debugging purposes. +Can also be derived from the number of elements in the subsequent array. +</p> +</li> + +<li> +<b>values</b> +<code class="typeof">array of numbers</code> +<span class="required-true">required</span> +<p> +Array of normalized uptime values. +May contain null values if less than 20% of network statuses have been +processed for a given time period. +Contains at least two subsequent non-null values to enable drawing of line +graphs. +</p> +</li> + +</ul> + +</li> + +</ul> + +</li> + +<li> +<b>bridges_published</b> +<code class="typeof">string</code> +<span class="required-true">required</span> +<p> +UTC timestamp (YYYY-MM-DD hh:mm:ss) when +the last known bridge network status was published. +Indicates how recent the bridge uptime documents in this document are. +</p> +</li> + +<li> +<b>bridges</b> +<code class="typeof">array of objects</code> +<span class="required-true">required</span> +<p> +Array of objects representing bridge uptime documents. +Each array object contains the following key-value pairs: +</p> + +<ul class="properties"> + +<li> +<b>fingerprint</b> +<code class="typeof">string</code> +<span class="required-true">required</span> +<p> +SHA-1 hash of the bridge fingerprint consisting +of 40 upper-case hexadecimal characters. +</p> +</li> + +<li> +<b>uptime</b> +<code class="typeof">object</code> +<span class="required-true">required</span> +<p> +Object containing uptime history objects for different time periods. +The specification of uptime history objects is similar to those in the +<strong>uptime</strong> field of <strong>relays</strong>. +</p> +</li> + +</li> + +</ul> + +</div> <!-- box --> + </body> </html>