commit d3638d4c9cf350ab5eb6f319e453a94163838236 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Apr 8 23:58:18 2014 +0200
Move (de-)serialization code into WeightsStatus. --- src/org/torproject/onionoo/DocumentStore.java | 6 +- src/org/torproject/onionoo/WeightsDataWriter.java | 161 ++++++--------------- src/org/torproject/onionoo/WeightsStatus.java | 91 ++++++++++++ 3 files changed, 143 insertions(+), 115 deletions(-)
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 387f34c..5da7267 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -196,7 +196,8 @@ public class DocumentStore { document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); - } else if (document instanceof ClientsStatus || + } else if (document instanceof WeightsStatus || + document instanceof ClientsStatus || document instanceof UptimeStatus) { documentString = document.toDocumentString(); } else { @@ -289,7 +290,8 @@ public class DocumentStore { documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); - } else if (documentType.equals(ClientsStatus.class) || + } else if (documentType.equals(WeightsStatus.class) || + documentType.equals(ClientsStatus.class) || documentType.equals(UptimeStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); } else { diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index d230662..0d7b815 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -2,17 +2,14 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;
-import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -230,7 +227,19 @@ public class WeightsDataWriter implements DescriptorListener, double advertisedBandwidth = 0.0; if (!this.advertisedBandwidths.containsKey( serverDescriptorDigest)) { - this.readHistoryFromDisk(fingerprint); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus != null) { + if (!this.descriptorDigestsByFingerprint.containsKey( + fingerprint)) { + this.descriptorDigestsByFingerprint.put(fingerprint, + new HashSet<String>()); + } + this.descriptorDigestsByFingerprint.get(fingerprint).addAll( + weightsStatus.advertisedBandwidths.keySet()); + this.advertisedBandwidths.putAll( + weightsStatus.advertisedBandwidths); + } } if (this.advertisedBandwidths.containsKey( serverDescriptorDigest)) { @@ -288,93 +297,42 @@ public class WeightsDataWriter implements DescriptorListener,
private void addToHistory(String fingerprint, long validAfterMillis, long freshUntilMillis, double[] weights) { - SortedMap<long[], double[]> history = - this.readHistoryFromDisk(fingerprint); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + SortedMap<long[], double[]> history = weightsStatus.history; long[] interval = new long[] { validAfterMillis, freshUntilMillis }; if ((history.headMap(interval).isEmpty() || history.headMap(interval).lastKey()[1] <= validAfterMillis) && (history.tailMap(interval).isEmpty() || history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) { history.put(interval, weights); - history = this.compressHistory(history); - this.writeHistoryToDisk(fingerprint, history); + this.compressHistory(weightsStatus); + this.addAdvertisedBandwidths(weightsStatus, fingerprint); + this.documentStore.store(weightsStatus, fingerprint); this.updateWeightsStatuses.remove(fingerprint); } }
- private SortedMap<long[], double[]> readHistoryFromDisk( + private void addAdvertisedBandwidths(WeightsStatus weightsStatus, String fingerprint) { - SortedMap<long[], double[]> history = - new TreeMap<long[], double[]>(new Comparator<long[]>() { - public int compare(long[] a, long[] b) { - return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; - } - }); - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, false, fingerprint); - if (weightsStatus != null) { - String historyString = weightsStatus.documentString; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - Scanner s = new Scanner(historyString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length == 2) { - String descriptorDigest = parts[0]; - int advertisedBandwidth = Integer.parseInt(parts[1]); - if (!this.descriptorDigestsByFingerprint.containsKey( - fingerprint)) { - this.descriptorDigestsByFingerprint.put(fingerprint, - new HashSet<String>()); - } - this.descriptorDigestsByFingerprint.get(fingerprint).add( - descriptorDigest); - this.advertisedBandwidths.put(descriptorDigest, - advertisedBandwidth); - continue; - } - if (parts.length != 9) { - System.err.println("Illegal line '" + line + "' in weights " - + "history for fingerprint '" + fingerprint + "'. " - + "Skipping this line."); - continue; - } - if (parts[4].equals("NaN")) { - /* Remove corrupt lines written on 2013-07-07 and the days - * after. */ - continue; - } - long validAfterMillis = dateTimeFormat.parse(parts[0] - + " " + parts[1]).getTime(); - long freshUntilMillis = dateTimeFormat.parse(parts[2] - + " " + parts[3]).getTime(); - long[] interval = new long[] { validAfterMillis, - freshUntilMillis }; - double[] weights = new double[] { - Double.parseDouble(parts[4]), - Double.parseDouble(parts[5]), - Double.parseDouble(parts[6]), - Double.parseDouble(parts[7]), - Double.parseDouble(parts[8]) }; - history.put(interval, weights); + if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) { + for (String descriptorDigest : + this.descriptorDigestsByFingerprint.get(fingerprint)) { + if (this.advertisedBandwidths.containsKey(descriptorDigest)) { + int advertisedBandwidth = + this.advertisedBandwidths.get(descriptorDigest); + weightsStatus.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); } - s.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "weights history for fingerprint '" + fingerprint + "'. " - + "Skipping."); - e.printStackTrace(); } } - return history; }
- private SortedMap<long[], double[]> compressHistory( - SortedMap<long[], double[]> history) { + private void compressHistory(WeightsStatus weightsStatus) { + SortedMap<long[], double[]> history = weightsStatus.history; SortedMap<long[], double[]> compressedHistory = new TreeMap<long[], double[]>(history.comparator()); long lastStartMillis = 0L, lastEndMillis = 0L; @@ -429,39 +387,7 @@ public class WeightsDataWriter implements DescriptorListener, compressedHistory.put(new long[] { lastStartMillis, lastEndMillis }, lastWeights); } - return compressedHistory; - } - - private void writeHistoryToDisk(String fingerprint, - SortedMap<long[], double[]> history) { - StringBuilder sb = new StringBuilder(); - if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) { - for (String descriptorDigest : - this.descriptorDigestsByFingerprint.get(fingerprint)) { - if (this.advertisedBandwidths.containsKey(descriptorDigest)) { - int advertisedBandwidth = - this.advertisedBandwidths.get(descriptorDigest); - sb.append(descriptorDigest + " " - + String.valueOf(advertisedBandwidth) + "\n"); - } - } - } - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long[] fresh = e.getKey(); - double[] weights = e.getValue(); - sb.append(dateTimeFormat.format(fresh[0]) + " " - + dateTimeFormat.format(fresh[1])); - for (double weight : weights) { - sb.append(String.format(" %.12f", weight)); - } - sb.append("\n"); - } - WeightsStatus weightsStatus = new WeightsStatus(); - weightsStatus.documentString = sb.toString(); - this.documentStore.store(weightsStatus, fingerprint); + weightsStatus.history = compressedHistory; }
public void processFingerprints(SortedSet<String> fingerprints, @@ -473,13 +399,18 @@ public class WeightsDataWriter implements DescriptorListener,
private void writeWeightsDataFiles() { for (String fingerprint : this.updateWeightsDocuments) { - SortedMap<long[], double[]> history = - this.readHistoryFromDisk(fingerprint); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + continue; + } + SortedMap<long[], double[]> history = weightsStatus.history; WeightsDocument weightsDocument = new WeightsDocument(); weightsDocument.documentString = this.formatHistoryString( fingerprint, history); this.documentStore.store(weightsDocument, fingerprint); } + Logger.printStatusTime("Wrote weights document files"); }
private String[] graphTypes = new String[] { @@ -633,9 +564,13 @@ public class WeightsDataWriter implements DescriptorListener,
private void updateWeightsStatuses() { for (String fingerprint : this.updateWeightsStatuses) { - SortedMap<long[], double[]> history = - this.readHistoryFromDisk(fingerprint); - this.writeHistoryToDisk(fingerprint, history); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + this.addAdvertisedBandwidths(weightsStatus, fingerprint); + this.documentStore.store(weightsStatus, fingerprint); } }
diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java index f8f78ad..4d92f30 100644 --- a/src/org/torproject/onionoo/WeightsStatus.java +++ b/src/org/torproject/onionoo/WeightsStatus.java @@ -1,5 +1,96 @@ package org.torproject.onionoo;
+import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeMap; + class WeightsStatus extends Document { + + SortedMap<long[], double[]> history = new TreeMap<long[], double[]>( + new Comparator<long[]>() { + public int compare(long[] a, long[] b) { + return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; + } + }); + + Map<String, Integer> advertisedBandwidths = + new HashMap<String, Integer>(); + + public void fromDocumentString(String documentString) { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length == 2) { + String descriptorDigest = parts[0]; + int advertisedBandwidth = Integer.parseInt(parts[1]); + this.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); + continue; + } + if (parts.length != 9) { + System.err.println("Illegal line '" + line + "' in weights " + + "status file. Skipping this line."); + continue; + } + if (parts[4].equals("NaN")) { + /* Remove corrupt lines written on 2013-07-07 and the days + * after. */ + continue; + } + long validAfterMillis = dateTimeFormat.parse(parts[0] + + " " + parts[1]).getTime(); + long freshUntilMillis = dateTimeFormat.parse(parts[2] + + " " + parts[3]).getTime(); + long[] interval = new long[] { validAfterMillis, + freshUntilMillis }; + double[] weights = new double[] { + Double.parseDouble(parts[4]), + Double.parseDouble(parts[5]), + Double.parseDouble(parts[6]), + Double.parseDouble(parts[7]), + Double.parseDouble(parts[8]) }; + this.history.put(interval, weights); + } + s.close(); + } catch (ParseException e) { + System.err.println("Could not parse timestamp while reading " + + "weights status file. Skipping."); + e.printStackTrace(); + } + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, Integer> e : + this.advertisedBandwidths.entrySet()) { + sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n"); + } + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long[] fresh = e.getKey(); + double[] weights = e.getValue(); + sb.append(dateTimeFormat.format(fresh[0]) + " " + + dateTimeFormat.format(fresh[1])); + for (double weight : weights) { + sb.append(String.format(" %.12f", weight)); + } + sb.append("\n"); + } + return sb.toString(); + } }