commit 4afcccf3de9d923332d6b043c9dd43bb2ee82b22 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Apr 10 18:35:12 2014 +0200

Split bandwidth data writer into two classes. --- .../torproject/onionoo/BandwidthDataWriter.java | 395 -------------------- .../onionoo/BandwidthDocumentWriter.java | 199 ++++++++++ src/org/torproject/onionoo/BandwidthStatus.java | 74 +++- .../torproject/onionoo/BandwidthStatusUpdater.java | 152 ++++++++ src/org/torproject/onionoo/DocumentStore.java | 6 +- src/org/torproject/onionoo/Main.java | 11 +- 6 files changed, 435 insertions(+), 402 deletions(-)

diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java deleted file mode 100644 index 227df2b..0000000 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ /dev/null @@ -1,395 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Scanner; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExtraInfoDescriptor; - -/* Write bandwidth data files to disk and delete bandwidth files of relays - * or bridges that fell out of the summary list. - * - * Bandwidth history data is available in different resolutions, depending - * on the considered time interval. Data for the past 72 hours is - * available for 15 minute detail, data for the past week in 1 hour - * detail, data for the past month in 4 hour detail, data for the past 3 - * months in 12 hour detail, data for the past year in 2 day detail, and - * earlier data in 10 day detail. These detail levels have been chosen to - * provide between 92 and 192 data points for graphing the bandwidth of - * the past day, past week, past month, past three months, past year, and - * past five years. - * - * Only update bandwidth data files for which new bandwidth histories are - * available. There's no point in updating bandwidth documents when we - * don't have newer bandwidth data to add. This means that, e.g., the - * last 3 days in the bandwidth document may not be equivalent to the last - * 3 days as of publishing the document, but that's something clients can - * work around. */ -public class BandwidthDataWriter implements DescriptorListener, - StatusUpdater, FingerprintListener, DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - - public BandwidthDataWriter(DescriptorSource descriptorSource, - DocumentStore documentStore, Time time) { - this.descriptorSource = descriptorSource; - this.documentStore = documentStore; - this.now = time.currentTimeMillis(); - this.dateTimeFormat.setLenient(false); - this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - this.registerDescriptorListeners(); - this.registerFingerprintListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private void registerFingerprintListeners() { - /* TODO Not used yet. - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS);*/ - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ExtraInfoDescriptor) { - this.parseDescriptor((ExtraInfoDescriptor) descriptor); - } - } - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - /* TODO Not used yet. */ - } - - public void updateStatuses() { - /* Status files are already updated while processing descriptors. */ - } - - public void writeDocuments() { - /* Document files are already updated while processing descriptors. */ - } - - private void parseDescriptor(ExtraInfoDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - boolean updateHistory = false; - SortedMap<Long, long[]> writeHistory = new TreeMap<Long, long[]>(), - readHistory = new TreeMap<Long, long[]>(); - if (descriptor.getWriteHistory() != null) { - parseHistoryLine(descriptor.getWriteHistory().getLine(), - writeHistory); - updateHistory = true; - } - if (descriptor.getReadHistory() != null) { - parseHistoryLine(descriptor.getReadHistory().getLine(), - readHistory); - updateHistory = true; - } - if (updateHistory) { - this.readHistoryFromDisk(fingerprint, writeHistory, readHistory); - this.compressHistory(writeHistory); - this.compressHistory(readHistory); - this.writeHistoryToDisk(fingerprint, writeHistory, readHistory); - this.writeBandwidthDataFileToDisk(fingerprint, writeHistory, - readHistory); - } - } - - private void parseHistoryLine(String line, - SortedMap<Long, long[]> history) { - String[] parts = line.split(" "); - if (parts.length < 6) { - return; - } - try { - long endMillis = this.dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L; - String[] values = parts[5].split(","); - for (int i = values.length - 1; i >= 0; i--) { - long bandwidthValue = Long.parseLong(values[i]); - long startMillis = endMillis - intervalMillis; - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidthValue }); - endMillis -= intervalMillis; - } - } catch (ParseException e) { - System.err.println("Could not parse timestamp in line '" + line - + "'. Skipping."); - } - } - - private void readHistoryFromDisk(String fingerprint, - SortedMap<Long, long[]> writeHistory, - SortedMap<Long, long[]> readHistory) { - BandwidthStatus bandwidthStatus = this.documentStore.retrieve( - BandwidthStatus.class, false, fingerprint); - if (bandwidthStatus == null) { - return; - } - String historyString = bandwidthStatus.documentString; - try { - Scanner s = new Scanner(historyString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length != 6) { - System.err.println("Illegal line '" + line + "' in bandwidth " - + "history for fingerprint '" + fingerprint + "'. " - + "Skipping this line."); - continue; - } - SortedMap<Long, long[]> history = parts[0].equals("r") - ? readHistory : writeHistory; - long startMillis = this.dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long endMillis = this.dateTimeFormat.parse(parts[3] + " " - + parts[4]).getTime(); - long bandwidth = Long.parseLong(parts[5]); - long previousEndMillis = history.headMap(startMillis).isEmpty() - ? startMillis - : history.get(history.headMap(startMillis).lastKey())[1]; - long nextStartMillis = history.tailMap(startMillis).isEmpty() - ? endMillis : history.tailMap(startMillis).firstKey(); - if (previousEndMillis <= startMillis && - nextStartMillis >= endMillis) { - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidth }); - } - } - s.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "bandwidth history for fingerprint '" + fingerprint + "'. " - + "Skipping."); - e.printStackTrace(); - } - } - - private void compressHistory( - SortedMap<Long, long[]> history) { - SortedMap<Long, long[]> uncompressedHistory = - new TreeMap<Long, long[]>(history); - history.clear(); - long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String lastMonthString = "1970-01"; - for (long[] v : uncompressedHistory.values()) { - long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; - long intervalLengthMillis; - if (this.now - endMillis <= 72L * 60L * 60L * 1000L) { - intervalLengthMillis = 15L * 60L * 1000L; - } else if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 60L * 60L * 1000L; - } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 4L * 60L * 60L * 1000L; - } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 12L * 60L * 60L * 1000L; - } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) { - intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; - } else { - intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; - } - String monthString = dateTimeFormat.format(startMillis); - if (lastEndMillis == startMillis && - ((lastEndMillis - 1L) / intervalLengthMillis) == - ((endMillis - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString)) { - lastEndMillis = endMillis; - lastBandwidth += bandwidth; - } else { - if (lastStartMillis > 0L) { - history.put(lastStartMillis, new long[] { lastStartMillis, - lastEndMillis, lastBandwidth }); - } - lastStartMillis = startMillis; - lastEndMillis = endMillis; - lastBandwidth = bandwidth; - } - lastMonthString = monthString; - } - if (lastStartMillis > 0L) { - history.put(lastStartMillis, new long[] { lastStartMillis, - lastEndMillis, lastBandwidth }); - } - } - - private void writeHistoryToDisk(String fingerprint, - SortedMap<Long, long[]> writeHistory, - SortedMap<Long, long[]> readHistory) { - StringBuilder sb = new StringBuilder(); - for (long[] v : writeHistory.values()) { - sb.append("w " + this.dateTimeFormat.format(v[0]) + " " - + this.dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); - } - for (long[] v : readHistory.values()) { - sb.append("r " + this.dateTimeFormat.format(v[0]) + " " - + this.dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); - } - BandwidthStatus bandwidthStatus = new BandwidthStatus(); - bandwidthStatus.documentString = sb.toString(); - this.documentStore.store(bandwidthStatus, fingerprint); - } - - private void writeBandwidthDataFileToDisk(String fingerprint, - SortedMap<Long, long[]> writeHistory, - SortedMap<Long, long[]> readHistory) { - String writeHistoryString = formatHistoryString(writeHistory); - String readHistoryString = formatHistoryString(readHistory); - StringBuilder sb = new StringBuilder(); - sb.append("{"fingerprint":"" + fingerprint + "",\n" - + ""write_history":{\n" + writeHistoryString + "},\n" - + ""read_history":{\n" + readHistoryString + "}}\n"); - BandwidthDocument bandwidthDocument = new BandwidthDocument(); - bandwidthDocument.documentString = sb.toString(); - this.documentStore.store(bandwidthDocument, fingerprint); - } - - private String[] graphNames = new String[] { - "3_days", - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - 72L * 60L * 60L * 1000L, - 7L * 24L * 60L * 60L * 1000L, - 31L * 24L * 60L * 60L * 1000L, - 92L * 24L * 60L * 60L * 1000L, - 366L * 24L * 60L * 60L * 1000L, - 5L * 366L * 24L * 60L * 60L * 1000L }; - - private long[] dataPointIntervals = new long[] { - 15L * 60L * 1000L, - 60L * 60L * 1000L, - 4L * 60L * 60L * 1000L, - 12L * 60L * 60L * 1000L, - 2L * 24L * 60L * 60L * 1000L, - 10L * 24L * 60L * 60L * 1000L }; - - private String formatHistoryString(SortedMap<Long, long[]> history) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < this.graphIntervals.length; i++) { - String graphName = this.graphNames[i]; - long graphInterval = this.graphIntervals[i]; - long dataPointInterval = this.dataPointIntervals[i]; - List<Long> dataPoints = new ArrayList<Long>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long totalMillis = 0L, totalBandwidth = 0L; - for (long[] v : history.values()) { - long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; - if (endMillis < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (endMillis / dataPointInterval)) { - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1L : (totalBandwidth * 1000L) / totalMillis); - totalBandwidth = 0L; - totalMillis = 0L; - intervalStartMillis += dataPointInterval; - } - totalBandwidth += bandwidth; - totalMillis += (endMillis - startMillis); - } - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1L : (totalBandwidth * 1000L) / totalMillis); - long maxValue = 1L; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int j = 0; j < dataPoints.size(); j++) { - long dataPoint = dataPoints.get(j); - if (dataPoint >= 0L) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = j; - } - lastNonNullIndex = j; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - continue; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (i > 0 && - firstDataPointMillis >= this.now - graphIntervals[i - 1]) { - /* Skip bandwidth history object, because it doesn't contain - * anything new that wasn't already contained in the last - * bandwidth history object(s). */ - continue; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - StringBuilder sb2 = new StringBuilder(); - sb2.append(""" + graphName + "":{" - + ""first":"" - + this.dateTimeFormat.format(firstDataPointMillis) + ""," - + ""last":"" - + this.dateTimeFormat.format(lastDataPointMillis) + ""," - +""interval":" + String.valueOf(dataPointInterval / 1000L) - + ","factor":" + String.format(Locale.US, "%.3f", factor) - + ","count":" + String.valueOf(count) + ","values":["); - int written = 0, previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) { - long dataPoint = dataPoints.get(j); - if (dataPoint >= 0L) { - if (j - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = j; - } - sb2.append((written++ > 0 ? "," : "") + (dataPoint < 0L ? "null" : - String.valueOf((dataPoint * 999L) / maxValue))); - } - sb2.append("]},\n"); - if (foundTwoAdjacentDataPoints) { - sb.append(sb2.toString()); - } - } - String result = sb.toString(); - if (result.length() >= 2) { - result = result.substring(0, result.length() - 2) + "\n"; - } - return result; - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} - diff --git a/src/org/torproject/onionoo/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/BandwidthDocumentWriter.java new file mode 100644 index 0000000..754c8f3 --- /dev/null +++ b/src/org/torproject/onionoo/BandwidthDocumentWriter.java @@ -0,0 +1,199 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; + +public class BandwidthDocumentWriter implements FingerprintListener, + DocumentWriter{ + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + + public BandwidthDocumentWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.dateTimeFormat.setLenient(false); + this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + private Set<String> updateBandwidthDocuments = new HashSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + this.updateBandwidthDocuments.addAll(fingerprints); + } + + public void writeDocuments() { + for (String fingerprint : this.updateBandwidthDocuments) { + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, true, fingerprint); + if (bandwidthStatus == null) { + continue; + } + this.writeBandwidthDataFileToDisk(fingerprint, + bandwidthStatus.writeHistory, bandwidthStatus.readHistory); + } + Logger.printStatusTime("Wrote bandwidth document files"); + } + + private void writeBandwidthDataFileToDisk(String fingerprint, + SortedMap<Long, long[]> writeHistory, + SortedMap<Long, long[]> readHistory) { + String writeHistoryString = formatHistoryString(writeHistory); + String readHistoryString = formatHistoryString(readHistory); + StringBuilder sb = new StringBuilder(); + sb.append("{"fingerprint":"" + fingerprint + "",\n" + + ""write_history":{\n" + writeHistoryString + "},\n" + + ""read_history":{\n" + readHistoryString + "}}\n"); + BandwidthDocument bandwidthDocument = new BandwidthDocument(); + bandwidthDocument.documentString = sb.toString(); + this.documentStore.store(bandwidthDocument, fingerprint); + } + + private String[] graphNames = new String[] { + "3_days", + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + 72L * 60L * 60L * 1000L, + 7L * 24L * 60L * 60L * 1000L, + 31L * 24L * 60L * 60L * 1000L, + 92L * 24L * 60L * 60L * 1000L, + 366L * 24L * 60L * 60L * 1000L, + 5L * 366L * 24L * 60L * 60L * 1000L }; + + private long[] dataPointIntervals = new long[] { + 15L * 60L * 1000L, + 60L * 60L * 1000L, + 4L * 60L * 60L * 1000L, + 12L * 60L * 60L * 1000L, + 2L * 24L * 60L * 60L * 1000L, + 10L * 24L * 60L * 60L * 1000L }; + + private String formatHistoryString(SortedMap<Long, long[]> history) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < this.graphIntervals.length; i++) { + String graphName = this.graphNames[i]; + long graphInterval = this.graphIntervals[i]; + long dataPointInterval = this.dataPointIntervals[i]; + List<Long> dataPoints = new ArrayList<Long>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long totalMillis = 0L, totalBandwidth = 0L; + for (long[] v : history.values()) { + long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; + if (endMillis < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (endMillis / dataPointInterval)) { + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1L : (totalBandwidth * 1000L) / totalMillis); + totalBandwidth = 0L; + totalMillis = 0L; + intervalStartMillis += dataPointInterval; + } + totalBandwidth += bandwidth; + totalMillis += (endMillis - startMillis); + } + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1L : (totalBandwidth * 1000L) / totalMillis); + long maxValue = 1L; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int j = 0; j < dataPoints.size(); j++) { + long dataPoint = dataPoints.get(j); + if (dataPoint >= 0L) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = j; + } + lastNonNullIndex = j; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + continue; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (i > 0 && + firstDataPointMillis >= this.now - graphIntervals[i - 1]) { + /* Skip bandwidth history object, because it doesn't contain + * anything new that wasn't already contained in the last + * bandwidth history object(s). */ + continue; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + StringBuilder sb2 = new StringBuilder(); + sb2.append(""" + graphName + "":{" + + ""first":"" + + this.dateTimeFormat.format(firstDataPointMillis) + ""," + + ""last":"" + + this.dateTimeFormat.format(lastDataPointMillis) + ""," + +""interval":" + String.valueOf(dataPointInterval / 1000L) + + ","factor":" + String.format(Locale.US, "%.3f", factor) + + ","count":" + String.valueOf(count) + ","values":["); + int written = 0, previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) { + long dataPoint = dataPoints.get(j); + if (dataPoint >= 0L) { + if (j - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = j; + } + sb2.append((written++ > 0 ? "," : "") + (dataPoint < 0L ? "null" : + String.valueOf((dataPoint * 999L) / maxValue))); + } + sb2.append("]},\n"); + if (foundTwoAdjacentDataPoints) { + sb.append(sb2.toString()); + } + } + String result = sb.toString(); + if (result.length() >= 2) { + result = result.substring(0, result.length() - 2) + "\n"; + } + return result; + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java index bf6f504..fd3c36e 100644 --- a/src/org/torproject/onionoo/BandwidthStatus.java +++ b/src/org/torproject/onionoo/BandwidthStatus.java @@ -1,7 +1,79 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo;

+import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + class BandwidthStatus extends Document { + + SortedMap<Long, long[]> writeHistory = new TreeMap<Long, long[]>(); + + SortedMap<Long, long[]> readHistory = new TreeMap<Long, long[]>(); + + public void fromDocumentString(String documentString) { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length != 6) { + System.err.println("Illegal line '" + line + "' in bandwidth " + + "history. Skipping this line."); + continue; + } + SortedMap<Long, long[]> history = parts[0].equals("r") + ? readHistory : writeHistory; + long startMillis = dateTimeFormat.parse(parts[1] + " " + + parts[2]).getTime(); + long endMillis = dateTimeFormat.parse(parts[3] + " " + + parts[4]).getTime(); + long bandwidth = Long.parseLong(parts[5]); + long previousEndMillis = history.headMap(startMillis).isEmpty() + ? startMillis + : history.get(history.headMap(startMillis).lastKey())[1]; + long nextStartMillis = history.tailMap(startMillis).isEmpty() + ? endMillis : history.tailMap(startMillis).firstKey(); + if (previousEndMillis <= startMillis && + nextStartMillis >= endMillis) { + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidth }); + } + } + s.close(); + } catch (ParseException e) { + System.err.println("Could not parse timestamp while reading " + + "bandwidth history. Skipping."); + e.printStackTrace(); + } + + } + + public String toDocumentString() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + StringBuilder sb = new StringBuilder(); + for (long[] v : writeHistory.values()) { + sb.append("w " + dateTimeFormat.format(v[0]) + " " + + dateTimeFormat.format(v[1]) + " " + + String.valueOf(v[2]) + "\n"); + } + for (long[] v : readHistory.values()) { + sb.append("r " + dateTimeFormat.format(v[0]) + " " + + dateTimeFormat.format(v[1]) + " " + + String.valueOf(v[2]) + "\n"); + } + return sb.toString(); + } }

diff --git a/src/org/torproject/onionoo/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/BandwidthStatusUpdater.java new file mode 100644 index 0000000..6254260 --- /dev/null +++ b/src/org/torproject/onionoo/BandwidthStatusUpdater.java @@ -0,0 +1,152 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExtraInfoDescriptor; + +public class BandwidthStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + + public BandwidthStatusUpdater(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.dateTimeFormat.setLenient(false); + this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ExtraInfoDescriptor) { + this.parseDescriptor((ExtraInfoDescriptor) descriptor); + } + } + + public void updateStatuses() { + /* Status files are already updated while processing descriptors. */ + } + + private void parseDescriptor(ExtraInfoDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, true, fingerprint); + if (bandwidthStatus == null) { + bandwidthStatus = new BandwidthStatus(); + } + if (descriptor.getWriteHistory() != null) { + parseHistoryLine(descriptor.getWriteHistory().getLine(), + bandwidthStatus.writeHistory); + } + if (descriptor.getReadHistory() != null) { + parseHistoryLine(descriptor.getReadHistory().getLine(), + bandwidthStatus.readHistory); + } + this.compressHistory(bandwidthStatus.writeHistory); + this.compressHistory(bandwidthStatus.readHistory); + this.documentStore.store(bandwidthStatus, fingerprint); + } + + private void parseHistoryLine(String line, + SortedMap<Long, long[]> history) { + String[] parts = line.split(" "); + if (parts.length < 6) { + return; + } + try { + long endMillis = this.dateTimeFormat.parse(parts[1] + " " + + parts[2]).getTime(); + long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L; + String[] values = parts[5].split(","); + for (int i = values.length - 1; i >= 0; i--) { + long bandwidthValue = Long.parseLong(values[i]); + long startMillis = endMillis - intervalMillis; + /* TODO Should we first check whether an interval is already + * contained in history? */ + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidthValue }); + endMillis -= intervalMillis; + } + } catch (ParseException e) { + System.err.println("Could not parse timestamp in line '" + line + + "'. Skipping."); + } + } + + private void compressHistory(SortedMap<Long, long[]> history) { + SortedMap<Long, long[]> uncompressedHistory = + new TreeMap<Long, long[]>(history); + history.clear(); + long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; + SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String lastMonthString = "1970-01"; + for (long[] v : uncompressedHistory.values()) { + long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; + long intervalLengthMillis; + if (this.now - endMillis <= 72L * 60L * 60L * 1000L) { + intervalLengthMillis = 15L * 60L * 1000L; + } else if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 60L * 60L * 1000L; + } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 4L * 60L * 60L * 1000L; + } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 12L * 60L * 60L * 1000L; + } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) { + intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L; + } else { + intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; + } + String monthString = dateTimeFormat.format(startMillis); + if (lastEndMillis == startMillis && + ((lastEndMillis - 1L) / intervalLengthMillis) == + ((endMillis - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString)) { + lastEndMillis = endMillis; + lastBandwidth += bandwidth; + } else { + if (lastStartMillis > 0L) { + history.put(lastStartMillis, new long[] { lastStartMillis, + lastEndMillis, lastBandwidth }); + } + lastStartMillis = startMillis; + lastEndMillis = endMillis; + lastBandwidth = bandwidth; + } + lastMonthString = monthString; + } + if (lastStartMillis > 0L) { + history.put(lastStartMillis, new long[] { lastStartMillis, + lastEndMillis, lastBandwidth }); + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} + diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 5da7267..be6abd5 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -196,7 +196,8 @@ public class DocumentStore { document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); - } else if (document instanceof WeightsStatus || + } else if (document instanceof BandwidthStatus || + document instanceof WeightsStatus || document instanceof ClientsStatus || document instanceof UptimeStatus) { documentString = document.toDocumentString(); @@ -290,7 +291,8 @@ public class DocumentStore { documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); - } else if (documentType.equals(WeightsStatus.class) || + } else if (documentType.equals(BandwidthStatus.class) || + documentType.equals(WeightsStatus.class) || documentType.equals(ClientsStatus.class) || documentType.equals(UptimeStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 434d90c..60db116 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -32,16 +32,18 @@ public class Main { Logger.printStatusTime("Initialized reverse domain name resolver"); NodeDataWriter ndw = new NodeDataWriter(dso, rdnr, ls, ds, t); Logger.printStatusTime("Initialized node data writer"); - BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds, t); - Logger.printStatusTime("Initialized bandwidth data writer"); + BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(dso, ds, t); + Logger.printStatusTime("Initialized bandwidth status updater"); WeightsStatusUpdater wsu = new WeightsStatusUpdater(dso, ds, t); Logger.printStatusTime("Initialized weights status updater"); ClientsStatusUpdater csu = new ClientsStatusUpdater(dso, ds, t); Logger.printStatusTime("Initialized clients status updater"); UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds); Logger.printStatusTime("Initialized uptime status updater"); - StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wsu, csu, usu }; + StatusUpdater[] sus = new StatusUpdater[] { ndw, bsu, wsu, csu, usu };

+ BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(dso, ds, t); + Logger.printStatusTime("Initialized bandwidth document writer"); WeightsDocumentWriter wdw = new WeightsDocumentWriter(dso, ds, t); Logger.printStatusTime("Initialized weights document writer"); ClientsDocumentWriter cdw = new ClientsDocumentWriter(dso, ds, t); @@ -95,7 +97,8 @@ public class Main { } /* TODO Print status updater statistics for *all* status updaters once * all data writers have been separated. */ - for (DocumentWriter dw : new DocumentWriter[] { wdw, cdw, udw }) { + for (DocumentWriter dw : new DocumentWriter[] { bdw, wdw, cdw, + udw }) { String statsString = dw.getStatsString(); if (statsString != null) { Logger.printStatistics(dw.getClass().getSimpleName(),