[tor-commits] [onionoo/master] Move (de-)serialization code into WeightsStatus.
karsten at torproject.org
karsten at torproject.org
Fri Apr 11 07:38:01 UTC 2014
commit d3638d4c9cf350ab5eb6f319e453a94163838236
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Apr 8 23:58:18 2014 +0200
Move (de-)serialization code into WeightsStatus.
---
src/org/torproject/onionoo/DocumentStore.java | 6 +-
src/org/torproject/onionoo/WeightsDataWriter.java | 161 ++++++---------------
src/org/torproject/onionoo/WeightsStatus.java | 91 ++++++++++++
3 files changed, 143 insertions(+), 115 deletions(-)
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
index 387f34c..5da7267 100644
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ b/src/org/torproject/onionoo/DocumentStore.java
@@ -196,7 +196,8 @@ public class DocumentStore {
document instanceof UptimeDocument) {
Gson gson = new Gson();
documentString = gson.toJson(this);
- } else if (document instanceof ClientsStatus ||
+ } else if (document instanceof WeightsStatus ||
+ document instanceof ClientsStatus ||
document instanceof UptimeStatus) {
documentString = document.toDocumentString();
} else {
@@ -289,7 +290,8 @@ public class DocumentStore {
documentType.equals(UptimeDocument.class)) {
return this.retrieveParsedDocumentFile(documentType,
documentString);
- } else if (documentType.equals(ClientsStatus.class) ||
+ } else if (documentType.equals(WeightsStatus.class) ||
+ documentType.equals(ClientsStatus.class) ||
documentType.equals(UptimeStatus.class)) {
return this.retrieveParsedStatusFile(documentType, documentString);
} else {
diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java
index d230662..0d7b815 100644
--- a/src/org/torproject/onionoo/WeightsDataWriter.java
+++ b/src/org/torproject/onionoo/WeightsDataWriter.java
@@ -2,17 +2,14 @@
* See LICENSE for licensing information */
package org.torproject.onionoo;
-import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Scanner;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -230,7 +227,19 @@ public class WeightsDataWriter implements DescriptorListener,
double advertisedBandwidth = 0.0;
if (!this.advertisedBandwidths.containsKey(
serverDescriptorDigest)) {
- this.readHistoryFromDisk(fingerprint);
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus != null) {
+ if (!this.descriptorDigestsByFingerprint.containsKey(
+ fingerprint)) {
+ this.descriptorDigestsByFingerprint.put(fingerprint,
+ new HashSet<String>());
+ }
+ this.descriptorDigestsByFingerprint.get(fingerprint).addAll(
+ weightsStatus.advertisedBandwidths.keySet());
+ this.advertisedBandwidths.putAll(
+ weightsStatus.advertisedBandwidths);
+ }
}
if (this.advertisedBandwidths.containsKey(
serverDescriptorDigest)) {
@@ -288,93 +297,42 @@ public class WeightsDataWriter implements DescriptorListener,
private void addToHistory(String fingerprint, long validAfterMillis,
long freshUntilMillis, double[] weights) {
- SortedMap<long[], double[]> history =
- this.readHistoryFromDisk(fingerprint);
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus == null) {
+ weightsStatus = new WeightsStatus();
+ }
+ SortedMap<long[], double[]> history = weightsStatus.history;
long[] interval = new long[] { validAfterMillis, freshUntilMillis };
if ((history.headMap(interval).isEmpty() ||
history.headMap(interval).lastKey()[1] <= validAfterMillis) &&
(history.tailMap(interval).isEmpty() ||
history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) {
history.put(interval, weights);
- history = this.compressHistory(history);
- this.writeHistoryToDisk(fingerprint, history);
+ this.compressHistory(weightsStatus);
+ this.addAdvertisedBandwidths(weightsStatus, fingerprint);
+ this.documentStore.store(weightsStatus, fingerprint);
this.updateWeightsStatuses.remove(fingerprint);
}
}
- private SortedMap<long[], double[]> readHistoryFromDisk(
+ private void addAdvertisedBandwidths(WeightsStatus weightsStatus,
String fingerprint) {
- SortedMap<long[], double[]> history =
- new TreeMap<long[], double[]>(new Comparator<long[]>() {
- public int compare(long[] a, long[] b) {
- return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
- }
- });
- WeightsStatus weightsStatus = this.documentStore.retrieve(
- WeightsStatus.class, false, fingerprint);
- if (weightsStatus != null) {
- String historyString = weightsStatus.documentString;
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setLenient(false);
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- try {
- Scanner s = new Scanner(historyString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- String[] parts = line.split(" ");
- if (parts.length == 2) {
- String descriptorDigest = parts[0];
- int advertisedBandwidth = Integer.parseInt(parts[1]);
- if (!this.descriptorDigestsByFingerprint.containsKey(
- fingerprint)) {
- this.descriptorDigestsByFingerprint.put(fingerprint,
- new HashSet<String>());
- }
- this.descriptorDigestsByFingerprint.get(fingerprint).add(
- descriptorDigest);
- this.advertisedBandwidths.put(descriptorDigest,
- advertisedBandwidth);
- continue;
- }
- if (parts.length != 9) {
- System.err.println("Illegal line '" + line + "' in weights "
- + "history for fingerprint '" + fingerprint + "'. "
- + "Skipping this line.");
- continue;
- }
- if (parts[4].equals("NaN")) {
- /* Remove corrupt lines written on 2013-07-07 and the days
- * after. */
- continue;
- }
- long validAfterMillis = dateTimeFormat.parse(parts[0]
- + " " + parts[1]).getTime();
- long freshUntilMillis = dateTimeFormat.parse(parts[2]
- + " " + parts[3]).getTime();
- long[] interval = new long[] { validAfterMillis,
- freshUntilMillis };
- double[] weights = new double[] {
- Double.parseDouble(parts[4]),
- Double.parseDouble(parts[5]),
- Double.parseDouble(parts[6]),
- Double.parseDouble(parts[7]),
- Double.parseDouble(parts[8]) };
- history.put(interval, weights);
+ if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) {
+ for (String descriptorDigest :
+ this.descriptorDigestsByFingerprint.get(fingerprint)) {
+ if (this.advertisedBandwidths.containsKey(descriptorDigest)) {
+ int advertisedBandwidth =
+ this.advertisedBandwidths.get(descriptorDigest);
+ weightsStatus.advertisedBandwidths.put(descriptorDigest,
+ advertisedBandwidth);
}
- s.close();
- } catch (ParseException e) {
- System.err.println("Could not parse timestamp while reading "
- + "weights history for fingerprint '" + fingerprint + "'. "
- + "Skipping.");
- e.printStackTrace();
}
}
- return history;
}
- private SortedMap<long[], double[]> compressHistory(
- SortedMap<long[], double[]> history) {
+ private void compressHistory(WeightsStatus weightsStatus) {
+ SortedMap<long[], double[]> history = weightsStatus.history;
SortedMap<long[], double[]> compressedHistory =
new TreeMap<long[], double[]>(history.comparator());
long lastStartMillis = 0L, lastEndMillis = 0L;
@@ -429,39 +387,7 @@ public class WeightsDataWriter implements DescriptorListener,
compressedHistory.put(new long[] { lastStartMillis, lastEndMillis },
lastWeights);
}
- return compressedHistory;
- }
-
- private void writeHistoryToDisk(String fingerprint,
- SortedMap<long[], double[]> history) {
- StringBuilder sb = new StringBuilder();
- if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) {
- for (String descriptorDigest :
- this.descriptorDigestsByFingerprint.get(fingerprint)) {
- if (this.advertisedBandwidths.containsKey(descriptorDigest)) {
- int advertisedBandwidth =
- this.advertisedBandwidths.get(descriptorDigest);
- sb.append(descriptorDigest + " "
- + String.valueOf(advertisedBandwidth) + "\n");
- }
- }
- }
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- for (Map.Entry<long[], double[]> e : history.entrySet()) {
- long[] fresh = e.getKey();
- double[] weights = e.getValue();
- sb.append(dateTimeFormat.format(fresh[0]) + " "
- + dateTimeFormat.format(fresh[1]));
- for (double weight : weights) {
- sb.append(String.format(" %.12f", weight));
- }
- sb.append("\n");
- }
- WeightsStatus weightsStatus = new WeightsStatus();
- weightsStatus.documentString = sb.toString();
- this.documentStore.store(weightsStatus, fingerprint);
+ weightsStatus.history = compressedHistory;
}
public void processFingerprints(SortedSet<String> fingerprints,
@@ -473,13 +399,18 @@ public class WeightsDataWriter implements DescriptorListener,
private void writeWeightsDataFiles() {
for (String fingerprint : this.updateWeightsDocuments) {
- SortedMap<long[], double[]> history =
- this.readHistoryFromDisk(fingerprint);
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus == null) {
+ continue;
+ }
+ SortedMap<long[], double[]> history = weightsStatus.history;
WeightsDocument weightsDocument = new WeightsDocument();
weightsDocument.documentString = this.formatHistoryString(
fingerprint, history);
this.documentStore.store(weightsDocument, fingerprint);
}
+ Logger.printStatusTime("Wrote weights document files");
}
private String[] graphTypes = new String[] {
@@ -633,9 +564,13 @@ public class WeightsDataWriter implements DescriptorListener,
private void updateWeightsStatuses() {
for (String fingerprint : this.updateWeightsStatuses) {
- SortedMap<long[], double[]> history =
- this.readHistoryFromDisk(fingerprint);
- this.writeHistoryToDisk(fingerprint, history);
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus == null) {
+ weightsStatus = new WeightsStatus();
+ }
+ this.addAdvertisedBandwidths(weightsStatus, fingerprint);
+ this.documentStore.store(weightsStatus, fingerprint);
}
}
diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java
index f8f78ad..4d92f30 100644
--- a/src/org/torproject/onionoo/WeightsStatus.java
+++ b/src/org/torproject/onionoo/WeightsStatus.java
@@ -1,5 +1,96 @@
package org.torproject.onionoo;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.TimeZone;
+import java.util.TreeMap;
+
class WeightsStatus extends Document {
+
+ SortedMap<long[], double[]> history = new TreeMap<long[], double[]>(
+ new Comparator<long[]>() {
+ public int compare(long[] a, long[] b) {
+ return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
+ }
+ });
+
+ Map<String, Integer> advertisedBandwidths =
+ new HashMap<String, Integer>();
+
+ public void fromDocumentString(String documentString) {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ try {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ String[] parts = line.split(" ");
+ if (parts.length == 2) {
+ String descriptorDigest = parts[0];
+ int advertisedBandwidth = Integer.parseInt(parts[1]);
+ this.advertisedBandwidths.put(descriptorDigest,
+ advertisedBandwidth);
+ continue;
+ }
+ if (parts.length != 9) {
+ System.err.println("Illegal line '" + line + "' in weights "
+ + "status file. Skipping this line.");
+ continue;
+ }
+ if (parts[4].equals("NaN")) {
+ /* Remove corrupt lines written on 2013-07-07 and the days
+ * after. */
+ continue;
+ }
+ long validAfterMillis = dateTimeFormat.parse(parts[0]
+ + " " + parts[1]).getTime();
+ long freshUntilMillis = dateTimeFormat.parse(parts[2]
+ + " " + parts[3]).getTime();
+ long[] interval = new long[] { validAfterMillis,
+ freshUntilMillis };
+ double[] weights = new double[] {
+ Double.parseDouble(parts[4]),
+ Double.parseDouble(parts[5]),
+ Double.parseDouble(parts[6]),
+ Double.parseDouble(parts[7]),
+ Double.parseDouble(parts[8]) };
+ this.history.put(interval, weights);
+ }
+ s.close();
+ } catch (ParseException e) {
+ System.err.println("Could not parse timestamp while reading "
+ + "weights status file. Skipping.");
+ e.printStackTrace();
+ }
+ }
+
+ public String toDocumentString() {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<String, Integer> e :
+ this.advertisedBandwidths.entrySet()) {
+ sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n");
+ }
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long[] fresh = e.getKey();
+ double[] weights = e.getValue();
+ sb.append(dateTimeFormat.format(fresh[0]) + " "
+ + dateTimeFormat.format(fresh[1]));
+ for (double weight : weights) {
+ sb.append(String.format(" %.12f", weight));
+ }
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
}
More information about the tor-commits
mailing list