[tor-commits] [onionoo/master] Split bandwidth data writer into two classes.
karsten at torproject.org
karsten at torproject.org
Fri Apr 11 07:38:01 UTC 2014
commit 4afcccf3de9d923332d6b043c9dd43bb2ee82b22
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Apr 10 18:35:12 2014 +0200
Split bandwidth data writer into two classes.
---
.../torproject/onionoo/BandwidthDataWriter.java | 395 --------------------
.../onionoo/BandwidthDocumentWriter.java | 199 ++++++++++
src/org/torproject/onionoo/BandwidthStatus.java | 74 +++-
.../torproject/onionoo/BandwidthStatusUpdater.java | 152 ++++++++
src/org/torproject/onionoo/DocumentStore.java | 6 +-
src/org/torproject/onionoo/Main.java | 11 +-
6 files changed, 435 insertions(+), 402 deletions(-)
diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java
deleted file mode 100644
index 227df2b..0000000
--- a/src/org/torproject/onionoo/BandwidthDataWriter.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Scanner;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeMap;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.ExtraInfoDescriptor;
-
-/* Write bandwidth data files to disk and delete bandwidth files of relays
- * or bridges that fell out of the summary list.
- *
- * Bandwidth history data is available in different resolutions, depending
- * on the considered time interval. Data for the past 72 hours is
- * available for 15 minute detail, data for the past week in 1 hour
- * detail, data for the past month in 4 hour detail, data for the past 3
- * months in 12 hour detail, data for the past year in 2 day detail, and
- * earlier data in 10 day detail. These detail levels have been chosen to
- * provide between 92 and 192 data points for graphing the bandwidth of
- * the past day, past week, past month, past three months, past year, and
- * past five years.
- *
- * Only update bandwidth data files for which new bandwidth histories are
- * available. There's no point in updating bandwidth documents when we
- * don't have newer bandwidth data to add. This means that, e.g., the
- * last 3 days in the bandwidth document may not be equivalent to the last
- * 3 days as of publishing the document, but that's something clients can
- * work around. */
-public class BandwidthDataWriter implements DescriptorListener,
- StatusUpdater, FingerprintListener, DocumentWriter {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- private SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
-
- public BandwidthDataWriter(DescriptorSource descriptorSource,
- DocumentStore documentStore, Time time) {
- this.descriptorSource = descriptorSource;
- this.documentStore = documentStore;
- this.now = time.currentTimeMillis();
- this.dateTimeFormat.setLenient(false);
- this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.registerDescriptorListeners();
- this.registerFingerprintListeners();
- }
-
- private void registerDescriptorListeners() {
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_EXTRA_INFOS);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_EXTRA_INFOS);
- }
-
- private void registerFingerprintListeners() {
- /* TODO Not used yet.
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_EXTRA_INFOS);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_EXTRA_INFOS);*/
- }
-
- public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof ExtraInfoDescriptor) {
- this.parseDescriptor((ExtraInfoDescriptor) descriptor);
- }
- }
-
- public void processFingerprints(SortedSet<String> fingerprints,
- boolean relay) {
- /* TODO Not used yet. */
- }
-
- public void updateStatuses() {
- /* Status files are already updated while processing descriptors. */
- }
-
- public void writeDocuments() {
- /* Document files are already updated while processing descriptors. */
- }
-
- private void parseDescriptor(ExtraInfoDescriptor descriptor) {
- String fingerprint = descriptor.getFingerprint();
- boolean updateHistory = false;
- SortedMap<Long, long[]> writeHistory = new TreeMap<Long, long[]>(),
- readHistory = new TreeMap<Long, long[]>();
- if (descriptor.getWriteHistory() != null) {
- parseHistoryLine(descriptor.getWriteHistory().getLine(),
- writeHistory);
- updateHistory = true;
- }
- if (descriptor.getReadHistory() != null) {
- parseHistoryLine(descriptor.getReadHistory().getLine(),
- readHistory);
- updateHistory = true;
- }
- if (updateHistory) {
- this.readHistoryFromDisk(fingerprint, writeHistory, readHistory);
- this.compressHistory(writeHistory);
- this.compressHistory(readHistory);
- this.writeHistoryToDisk(fingerprint, writeHistory, readHistory);
- this.writeBandwidthDataFileToDisk(fingerprint, writeHistory,
- readHistory);
- }
- }
-
- private void parseHistoryLine(String line,
- SortedMap<Long, long[]> history) {
- String[] parts = line.split(" ");
- if (parts.length < 6) {
- return;
- }
- try {
- long endMillis = this.dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L;
- String[] values = parts[5].split(",");
- for (int i = values.length - 1; i >= 0; i--) {
- long bandwidthValue = Long.parseLong(values[i]);
- long startMillis = endMillis - intervalMillis;
- history.put(startMillis, new long[] { startMillis, endMillis,
- bandwidthValue });
- endMillis -= intervalMillis;
- }
- } catch (ParseException e) {
- System.err.println("Could not parse timestamp in line '" + line
- + "'. Skipping.");
- }
- }
-
- private void readHistoryFromDisk(String fingerprint,
- SortedMap<Long, long[]> writeHistory,
- SortedMap<Long, long[]> readHistory) {
- BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
- BandwidthStatus.class, false, fingerprint);
- if (bandwidthStatus == null) {
- return;
- }
- String historyString = bandwidthStatus.documentString;
- try {
- Scanner s = new Scanner(historyString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- String[] parts = line.split(" ");
- if (parts.length != 6) {
- System.err.println("Illegal line '" + line + "' in bandwidth "
- + "history for fingerprint '" + fingerprint + "'. "
- + "Skipping this line.");
- continue;
- }
- SortedMap<Long, long[]> history = parts[0].equals("r")
- ? readHistory : writeHistory;
- long startMillis = this.dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- long endMillis = this.dateTimeFormat.parse(parts[3] + " "
- + parts[4]).getTime();
- long bandwidth = Long.parseLong(parts[5]);
- long previousEndMillis = history.headMap(startMillis).isEmpty()
- ? startMillis
- : history.get(history.headMap(startMillis).lastKey())[1];
- long nextStartMillis = history.tailMap(startMillis).isEmpty()
- ? endMillis : history.tailMap(startMillis).firstKey();
- if (previousEndMillis <= startMillis &&
- nextStartMillis >= endMillis) {
- history.put(startMillis, new long[] { startMillis, endMillis,
- bandwidth });
- }
- }
- s.close();
- } catch (ParseException e) {
- System.err.println("Could not parse timestamp while reading "
- + "bandwidth history for fingerprint '" + fingerprint + "'. "
- + "Skipping.");
- e.printStackTrace();
- }
- }
-
- private void compressHistory(
- SortedMap<Long, long[]> history) {
- SortedMap<Long, long[]> uncompressedHistory =
- new TreeMap<Long, long[]>(history);
- history.clear();
- long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L;
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String lastMonthString = "1970-01";
- for (long[] v : uncompressedHistory.values()) {
- long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
- long intervalLengthMillis;
- if (this.now - endMillis <= 72L * 60L * 60L * 1000L) {
- intervalLengthMillis = 15L * 60L * 1000L;
- } else if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) {
- intervalLengthMillis = 60L * 60L * 1000L;
- } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) {
- intervalLengthMillis = 4L * 60L * 60L * 1000L;
- } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) {
- intervalLengthMillis = 12L * 60L * 60L * 1000L;
- } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) {
- intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L;
- } else {
- intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L;
- }
- String monthString = dateTimeFormat.format(startMillis);
- if (lastEndMillis == startMillis &&
- ((lastEndMillis - 1L) / intervalLengthMillis) ==
- ((endMillis - 1L) / intervalLengthMillis) &&
- lastMonthString.equals(monthString)) {
- lastEndMillis = endMillis;
- lastBandwidth += bandwidth;
- } else {
- if (lastStartMillis > 0L) {
- history.put(lastStartMillis, new long[] { lastStartMillis,
- lastEndMillis, lastBandwidth });
- }
- lastStartMillis = startMillis;
- lastEndMillis = endMillis;
- lastBandwidth = bandwidth;
- }
- lastMonthString = monthString;
- }
- if (lastStartMillis > 0L) {
- history.put(lastStartMillis, new long[] { lastStartMillis,
- lastEndMillis, lastBandwidth });
- }
- }
-
- private void writeHistoryToDisk(String fingerprint,
- SortedMap<Long, long[]> writeHistory,
- SortedMap<Long, long[]> readHistory) {
- StringBuilder sb = new StringBuilder();
- for (long[] v : writeHistory.values()) {
- sb.append("w " + this.dateTimeFormat.format(v[0]) + " "
- + this.dateTimeFormat.format(v[1]) + " "
- + String.valueOf(v[2]) + "\n");
- }
- for (long[] v : readHistory.values()) {
- sb.append("r " + this.dateTimeFormat.format(v[0]) + " "
- + this.dateTimeFormat.format(v[1]) + " "
- + String.valueOf(v[2]) + "\n");
- }
- BandwidthStatus bandwidthStatus = new BandwidthStatus();
- bandwidthStatus.documentString = sb.toString();
- this.documentStore.store(bandwidthStatus, fingerprint);
- }
-
- private void writeBandwidthDataFileToDisk(String fingerprint,
- SortedMap<Long, long[]> writeHistory,
- SortedMap<Long, long[]> readHistory) {
- String writeHistoryString = formatHistoryString(writeHistory);
- String readHistoryString = formatHistoryString(readHistory);
- StringBuilder sb = new StringBuilder();
- sb.append("{\"fingerprint\":\"" + fingerprint + "\",\n"
- + "\"write_history\":{\n" + writeHistoryString + "},\n"
- + "\"read_history\":{\n" + readHistoryString + "}}\n");
- BandwidthDocument bandwidthDocument = new BandwidthDocument();
- bandwidthDocument.documentString = sb.toString();
- this.documentStore.store(bandwidthDocument, fingerprint);
- }
-
- private String[] graphNames = new String[] {
- "3_days",
- "1_week",
- "1_month",
- "3_months",
- "1_year",
- "5_years" };
-
- private long[] graphIntervals = new long[] {
- 72L * 60L * 60L * 1000L,
- 7L * 24L * 60L * 60L * 1000L,
- 31L * 24L * 60L * 60L * 1000L,
- 92L * 24L * 60L * 60L * 1000L,
- 366L * 24L * 60L * 60L * 1000L,
- 5L * 366L * 24L * 60L * 60L * 1000L };
-
- private long[] dataPointIntervals = new long[] {
- 15L * 60L * 1000L,
- 60L * 60L * 1000L,
- 4L * 60L * 60L * 1000L,
- 12L * 60L * 60L * 1000L,
- 2L * 24L * 60L * 60L * 1000L,
- 10L * 24L * 60L * 60L * 1000L };
-
- private String formatHistoryString(SortedMap<Long, long[]> history) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < this.graphIntervals.length; i++) {
- String graphName = this.graphNames[i];
- long graphInterval = this.graphIntervals[i];
- long dataPointInterval = this.dataPointIntervals[i];
- List<Long> dataPoints = new ArrayList<Long>();
- long intervalStartMillis = ((this.now - graphInterval)
- / dataPointInterval) * dataPointInterval;
- long totalMillis = 0L, totalBandwidth = 0L;
- for (long[] v : history.values()) {
- long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
- if (endMillis < intervalStartMillis) {
- continue;
- }
- while ((intervalStartMillis / dataPointInterval) !=
- (endMillis / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * 1000L) / totalMillis);
- totalBandwidth = 0L;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- totalBandwidth += bandwidth;
- totalMillis += (endMillis - startMillis);
- }
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * 1000L) / totalMillis);
- long maxValue = 1L;
- int firstNonNullIndex = -1, lastNonNullIndex = -1;
- for (int j = 0; j < dataPoints.size(); j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = j;
- }
- lastNonNullIndex = j;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- continue;
- }
- long firstDataPointMillis = (((this.now - graphInterval)
- / dataPointInterval) + firstNonNullIndex) * dataPointInterval
- + dataPointInterval / 2L;
- if (i > 0 &&
- firstDataPointMillis >= this.now - graphIntervals[i - 1]) {
- /* Skip bandwidth history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * bandwidth history object(s). */
- continue;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- StringBuilder sb2 = new StringBuilder();
- sb2.append("\"" + graphName + "\":{"
- + "\"first\":\""
- + this.dateTimeFormat.format(firstDataPointMillis) + "\","
- + "\"last\":\""
- + this.dateTimeFormat.format(lastDataPointMillis) + "\","
- +"\"interval\":" + String.valueOf(dataPointInterval / 1000L)
- + ",\"factor\":" + String.format(Locale.US, "%.3f", factor)
- + ",\"count\":" + String.valueOf(count) + ",\"values\":[");
- int written = 0, previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (j - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = j;
- }
- sb2.append((written++ > 0 ? "," : "") + (dataPoint < 0L ? "null" :
- String.valueOf((dataPoint * 999L) / maxValue)));
- }
- sb2.append("]},\n");
- if (foundTwoAdjacentDataPoints) {
- sb.append(sb2.toString());
- }
- }
- String result = sb.toString();
- if (result.length() >= 2) {
- result = result.substring(0, result.length() - 2) + "\n";
- }
- return result;
- }
-
- public String getStatsString() {
- /* TODO Add statistics string. */
- return null;
- }
-}
-
diff --git a/src/org/torproject/onionoo/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/BandwidthDocumentWriter.java
new file mode 100644
index 0000000..754c8f3
--- /dev/null
+++ b/src/org/torproject/onionoo/BandwidthDocumentWriter.java
@@ -0,0 +1,199 @@
+/* Copyright 2011--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+
+public class BandwidthDocumentWriter implements FingerprintListener,
+ DocumentWriter{
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ private SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+
+ public BandwidthDocumentWriter(DescriptorSource descriptorSource,
+ DocumentStore documentStore, Time time) {
+ this.descriptorSource = descriptorSource;
+ this.documentStore = documentStore;
+ this.now = time.currentTimeMillis();
+ this.dateTimeFormat.setLenient(false);
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.registerFingerprintListeners();
+ }
+
+ private void registerFingerprintListeners() {
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_EXTRA_INFOS);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_EXTRA_INFOS);
+ }
+
+ private Set<String> updateBandwidthDocuments = new HashSet<String>();
+
+ public void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay) {
+ this.updateBandwidthDocuments.addAll(fingerprints);
+ }
+
+ public void writeDocuments() {
+ for (String fingerprint : this.updateBandwidthDocuments) {
+ BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
+ BandwidthStatus.class, true, fingerprint);
+ if (bandwidthStatus == null) {
+ continue;
+ }
+ this.writeBandwidthDataFileToDisk(fingerprint,
+ bandwidthStatus.writeHistory, bandwidthStatus.readHistory);
+ }
+ Logger.printStatusTime("Wrote bandwidth document files");
+ }
+
+ private void writeBandwidthDataFileToDisk(String fingerprint,
+ SortedMap<Long, long[]> writeHistory,
+ SortedMap<Long, long[]> readHistory) {
+ String writeHistoryString = formatHistoryString(writeHistory);
+ String readHistoryString = formatHistoryString(readHistory);
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"fingerprint\":\"" + fingerprint + "\",\n"
+ + "\"write_history\":{\n" + writeHistoryString + "},\n"
+ + "\"read_history\":{\n" + readHistoryString + "}}\n");
+ BandwidthDocument bandwidthDocument = new BandwidthDocument();
+ bandwidthDocument.documentString = sb.toString();
+ this.documentStore.store(bandwidthDocument, fingerprint);
+ }
+
+ private String[] graphNames = new String[] {
+ "3_days",
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ 72L * 60L * 60L * 1000L,
+ 7L * 24L * 60L * 60L * 1000L,
+ 31L * 24L * 60L * 60L * 1000L,
+ 92L * 24L * 60L * 60L * 1000L,
+ 366L * 24L * 60L * 60L * 1000L,
+ 5L * 366L * 24L * 60L * 60L * 1000L };
+
+ private long[] dataPointIntervals = new long[] {
+ 15L * 60L * 1000L,
+ 60L * 60L * 1000L,
+ 4L * 60L * 60L * 1000L,
+ 12L * 60L * 60L * 1000L,
+ 2L * 24L * 60L * 60L * 1000L,
+ 10L * 24L * 60L * 60L * 1000L };
+
+ private String formatHistoryString(SortedMap<Long, long[]> history) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ String graphName = this.graphNames[i];
+ long graphInterval = this.graphIntervals[i];
+ long dataPointInterval = this.dataPointIntervals[i];
+ List<Long> dataPoints = new ArrayList<Long>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ long totalMillis = 0L, totalBandwidth = 0L;
+ for (long[] v : history.values()) {
+ long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
+ if (endMillis < intervalStartMillis) {
+ continue;
+ }
+ while ((intervalStartMillis / dataPointInterval) !=
+ (endMillis / dataPointInterval)) {
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1L : (totalBandwidth * 1000L) / totalMillis);
+ totalBandwidth = 0L;
+ totalMillis = 0L;
+ intervalStartMillis += dataPointInterval;
+ }
+ totalBandwidth += bandwidth;
+ totalMillis += (endMillis - startMillis);
+ }
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1L : (totalBandwidth * 1000L) / totalMillis);
+ long maxValue = 1L;
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int j = 0; j < dataPoints.size(); j++) {
+ long dataPoint = dataPoints.get(j);
+ if (dataPoint >= 0L) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = j;
+ }
+ lastNonNullIndex = j;
+ if (dataPoint > maxValue) {
+ maxValue = dataPoint;
+ }
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ continue;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex) * dataPointInterval
+ + dataPointInterval / 2L;
+ if (i > 0 &&
+ firstDataPointMillis >= this.now - graphIntervals[i - 1]) {
+ /* Skip bandwidth history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * bandwidth history object(s). */
+ continue;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ double factor = ((double) maxValue) / 999.0;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ StringBuilder sb2 = new StringBuilder();
+ sb2.append("\"" + graphName + "\":{"
+ + "\"first\":\""
+ + this.dateTimeFormat.format(firstDataPointMillis) + "\","
+ + "\"last\":\""
+ + this.dateTimeFormat.format(lastDataPointMillis) + "\","
+ +"\"interval\":" + String.valueOf(dataPointInterval / 1000L)
+ + ",\"factor\":" + String.format(Locale.US, "%.3f", factor)
+ + ",\"count\":" + String.valueOf(count) + ",\"values\":[");
+ int written = 0, previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) {
+ long dataPoint = dataPoints.get(j);
+ if (dataPoint >= 0L) {
+ if (j - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = j;
+ }
+ sb2.append((written++ > 0 ? "," : "") + (dataPoint < 0L ? "null" :
+ String.valueOf((dataPoint * 999L) / maxValue)));
+ }
+ sb2.append("]},\n");
+ if (foundTwoAdjacentDataPoints) {
+ sb.append(sb2.toString());
+ }
+ }
+ String result = sb.toString();
+ if (result.length() >= 2) {
+ result = result.substring(0, result.length() - 2) + "\n";
+ }
+ return result;
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java
index bf6f504..fd3c36e 100644
--- a/src/org/torproject/onionoo/BandwidthStatus.java
+++ b/src/org/torproject/onionoo/BandwidthStatus.java
@@ -1,7 +1,79 @@
-/* Copyright 2013 The Tor Project
+/* Copyright 2013--2014 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.TimeZone;
+import java.util.TreeMap;
+
class BandwidthStatus extends Document {
+
+ SortedMap<Long, long[]> writeHistory = new TreeMap<Long, long[]>();
+
+ SortedMap<Long, long[]> readHistory = new TreeMap<Long, long[]>();
+
+ public void fromDocumentString(String documentString) {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ try {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ String[] parts = line.split(" ");
+ if (parts.length != 6) {
+ System.err.println("Illegal line '" + line + "' in bandwidth "
+ + "history. Skipping this line.");
+ continue;
+ }
+ SortedMap<Long, long[]> history = parts[0].equals("r")
+ ? readHistory : writeHistory;
+ long startMillis = dateTimeFormat.parse(parts[1] + " "
+ + parts[2]).getTime();
+ long endMillis = dateTimeFormat.parse(parts[3] + " "
+ + parts[4]).getTime();
+ long bandwidth = Long.parseLong(parts[5]);
+ long previousEndMillis = history.headMap(startMillis).isEmpty()
+ ? startMillis
+ : history.get(history.headMap(startMillis).lastKey())[1];
+ long nextStartMillis = history.tailMap(startMillis).isEmpty()
+ ? endMillis : history.tailMap(startMillis).firstKey();
+ if (previousEndMillis <= startMillis &&
+ nextStartMillis >= endMillis) {
+ history.put(startMillis, new long[] { startMillis, endMillis,
+ bandwidth });
+ }
+ }
+ s.close();
+ } catch (ParseException e) {
+ System.err.println("Could not parse timestamp while reading "
+ + "bandwidth history. Skipping.");
+ e.printStackTrace();
+ }
+
+ }
+
+ public String toDocumentString() {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ StringBuilder sb = new StringBuilder();
+ for (long[] v : writeHistory.values()) {
+ sb.append("w " + dateTimeFormat.format(v[0]) + " "
+ + dateTimeFormat.format(v[1]) + " "
+ + String.valueOf(v[2]) + "\n");
+ }
+ for (long[] v : readHistory.values()) {
+ sb.append("r " + dateTimeFormat.format(v[0]) + " "
+ + dateTimeFormat.format(v[1]) + " "
+ + String.valueOf(v[2]) + "\n");
+ }
+ return sb.toString();
+ }
}
diff --git a/src/org/torproject/onionoo/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/BandwidthStatusUpdater.java
new file mode 100644
index 0000000..6254260
--- /dev/null
+++ b/src/org/torproject/onionoo/BandwidthStatusUpdater.java
@@ -0,0 +1,152 @@
+/* Copyright 2011--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.SortedMap;
+import java.util.TimeZone;
+import java.util.TreeMap;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+
+public class BandwidthStatusUpdater implements DescriptorListener,
+ StatusUpdater {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ private SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+
+ public BandwidthStatusUpdater(DescriptorSource descriptorSource,
+ DocumentStore documentStore, Time time) {
+ this.descriptorSource = descriptorSource;
+ this.documentStore = documentStore;
+ this.now = time.currentTimeMillis();
+ this.dateTimeFormat.setLenient(false);
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_EXTRA_INFOS);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_EXTRA_INFOS);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof ExtraInfoDescriptor) {
+ this.parseDescriptor((ExtraInfoDescriptor) descriptor);
+ }
+ }
+
+ public void updateStatuses() {
+ /* Status files are already updated while processing descriptors. */
+ }
+
+ private void parseDescriptor(ExtraInfoDescriptor descriptor) {
+ String fingerprint = descriptor.getFingerprint();
+ BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
+ BandwidthStatus.class, true, fingerprint);
+ if (bandwidthStatus == null) {
+ bandwidthStatus = new BandwidthStatus();
+ }
+ if (descriptor.getWriteHistory() != null) {
+ parseHistoryLine(descriptor.getWriteHistory().getLine(),
+ bandwidthStatus.writeHistory);
+ }
+ if (descriptor.getReadHistory() != null) {
+ parseHistoryLine(descriptor.getReadHistory().getLine(),
+ bandwidthStatus.readHistory);
+ }
+ this.compressHistory(bandwidthStatus.writeHistory);
+ this.compressHistory(bandwidthStatus.readHistory);
+ this.documentStore.store(bandwidthStatus, fingerprint);
+ }
+
+ private void parseHistoryLine(String line,
+ SortedMap<Long, long[]> history) {
+ String[] parts = line.split(" ");
+ if (parts.length < 6) {
+ return;
+ }
+ try {
+ long endMillis = this.dateTimeFormat.parse(parts[1] + " "
+ + parts[2]).getTime();
+ long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L;
+ String[] values = parts[5].split(",");
+ for (int i = values.length - 1; i >= 0; i--) {
+ long bandwidthValue = Long.parseLong(values[i]);
+ long startMillis = endMillis - intervalMillis;
+ /* TODO Should we first check whether an interval is already
+ * contained in history? */
+ history.put(startMillis, new long[] { startMillis, endMillis,
+ bandwidthValue });
+ endMillis -= intervalMillis;
+ }
+ } catch (ParseException e) {
+ System.err.println("Could not parse timestamp in line '" + line
+ + "'. Skipping.");
+ }
+ }
+
+ private void compressHistory(SortedMap<Long, long[]> history) {
+ SortedMap<Long, long[]> uncompressedHistory =
+ new TreeMap<Long, long[]>(history);
+ history.clear();
+ long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L;
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String lastMonthString = "1970-01";
+ for (long[] v : uncompressedHistory.values()) {
+ long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
+ long intervalLengthMillis;
+ if (this.now - endMillis <= 72L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 15L * 60L * 1000L;
+ } else if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 60L * 60L * 1000L;
+ } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 4L * 60L * 60L * 1000L;
+ } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 12L * 60L * 60L * 1000L;
+ } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L;
+ } else {
+ intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L;
+ }
+ String monthString = dateTimeFormat.format(startMillis);
+ if (lastEndMillis == startMillis &&
+ ((lastEndMillis - 1L) / intervalLengthMillis) ==
+ ((endMillis - 1L) / intervalLengthMillis) &&
+ lastMonthString.equals(monthString)) {
+ lastEndMillis = endMillis;
+ lastBandwidth += bandwidth;
+ } else {
+ if (lastStartMillis > 0L) {
+ history.put(lastStartMillis, new long[] { lastStartMillis,
+ lastEndMillis, lastBandwidth });
+ }
+ lastStartMillis = startMillis;
+ lastEndMillis = endMillis;
+ lastBandwidth = bandwidth;
+ }
+ lastMonthString = monthString;
+ }
+ if (lastStartMillis > 0L) {
+ history.put(lastStartMillis, new long[] { lastStartMillis,
+ lastEndMillis, lastBandwidth });
+ }
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
index 5da7267..be6abd5 100644
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ b/src/org/torproject/onionoo/DocumentStore.java
@@ -196,7 +196,8 @@ public class DocumentStore {
document instanceof UptimeDocument) {
Gson gson = new Gson();
documentString = gson.toJson(this);
- } else if (document instanceof WeightsStatus ||
+ } else if (document instanceof BandwidthStatus ||
+ document instanceof WeightsStatus ||
document instanceof ClientsStatus ||
document instanceof UptimeStatus) {
documentString = document.toDocumentString();
@@ -290,7 +291,8 @@ public class DocumentStore {
documentType.equals(UptimeDocument.class)) {
return this.retrieveParsedDocumentFile(documentType,
documentString);
- } else if (documentType.equals(WeightsStatus.class) ||
+ } else if (documentType.equals(BandwidthStatus.class) ||
+ documentType.equals(WeightsStatus.class) ||
documentType.equals(ClientsStatus.class) ||
documentType.equals(UptimeStatus.class)) {
return this.retrieveParsedStatusFile(documentType, documentString);
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 434d90c..60db116 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -32,16 +32,18 @@ public class Main {
Logger.printStatusTime("Initialized reverse domain name resolver");
NodeDataWriter ndw = new NodeDataWriter(dso, rdnr, ls, ds, t);
Logger.printStatusTime("Initialized node data writer");
- BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds, t);
- Logger.printStatusTime("Initialized bandwidth data writer");
+ BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(dso, ds, t);
+ Logger.printStatusTime("Initialized bandwidth status updater");
WeightsStatusUpdater wsu = new WeightsStatusUpdater(dso, ds, t);
Logger.printStatusTime("Initialized weights status updater");
ClientsStatusUpdater csu = new ClientsStatusUpdater(dso, ds, t);
Logger.printStatusTime("Initialized clients status updater");
UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds);
Logger.printStatusTime("Initialized uptime status updater");
- StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wsu, csu, usu };
+ StatusUpdater[] sus = new StatusUpdater[] { ndw, bsu, wsu, csu, usu };
+ BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(dso, ds, t);
+ Logger.printStatusTime("Initialized bandwidth document writer");
WeightsDocumentWriter wdw = new WeightsDocumentWriter(dso, ds, t);
Logger.printStatusTime("Initialized weights document writer");
ClientsDocumentWriter cdw = new ClientsDocumentWriter(dso, ds, t);
@@ -95,7 +97,8 @@ public class Main {
}
/* TODO Print status updater statistics for *all* status updaters once
* all data writers have been separated. */
- for (DocumentWriter dw : new DocumentWriter[] { wdw, cdw, udw }) {
+ for (DocumentWriter dw : new DocumentWriter[] { bdw, wdw, cdw,
+ udw }) {
String statsString = dw.getStatsString();
if (statsString != null) {
Logger.printStatistics(dw.getClass().getSimpleName(),
More information about the tor-commits
mailing list