[tor-commits] [onionoo/master] Move (de-)serialization code into ClientsStatus.
karsten at torproject.org
karsten at torproject.org
Fri Apr 11 07:38:01 UTC 2014
commit bf01739df763e75030aa8731bf91ca7add4f8b83
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Apr 8 23:57:54 2014 +0200
Move (de-)serialization code into ClientsStatus.
---
src/org/torproject/onionoo/ClientsDataWriter.java | 237 +++------------------
src/org/torproject/onionoo/ClientsStatus.java | 192 +++++++++++++++++
src/org/torproject/onionoo/DocumentStore.java | 6 +-
3 files changed, 230 insertions(+), 205 deletions(-)
diff --git a/src/org/torproject/onionoo/ClientsDataWriter.java b/src/org/torproject/onionoo/ClientsDataWriter.java
index 2ccdff7..7662f54 100644
--- a/src/org/torproject/onionoo/ClientsDataWriter.java
+++ b/src/org/torproject/onionoo/ClientsDataWriter.java
@@ -2,13 +2,11 @@
* See LICENSE for licensing information */
package org.torproject.onionoo;
-import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Scanner;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TimeZone;
@@ -54,147 +52,6 @@ import org.torproject.descriptor.ExtraInfoDescriptor;
public class ClientsDataWriter implements DescriptorListener,
StatusUpdater, FingerprintListener, DocumentWriter {
- private static class ResponseHistory
- implements Comparable<ResponseHistory> {
- private long startMillis;
- private long endMillis;
- private double totalResponses;
- private SortedMap<String, Double> responsesByCountry;
- private SortedMap<String, Double> responsesByTransport;
- private SortedMap<String, Double> responsesByVersion;
- private ResponseHistory(long startMillis, long endMillis,
- double totalResponses,
- SortedMap<String, Double> responsesByCountry,
- SortedMap<String, Double> responsesByTransport,
- SortedMap<String, Double> responsesByVersion) {
- this.startMillis = startMillis;
- this.endMillis = endMillis;
- this.totalResponses = totalResponses;
- this.responsesByCountry = responsesByCountry;
- this.responsesByTransport = responsesByTransport;
- this.responsesByVersion = responsesByVersion;
- }
- public static ResponseHistory fromString(
- String responseHistoryString) {
- String[] parts = responseHistoryString.split(" ", 8);
- if (parts.length != 8) {
- return null;
- }
- long startMillis = -1L, endMillis = -1L;
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setLenient(false);
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- try {
- startMillis = dateTimeFormat.parse(parts[0] + " " + parts[1]).
- getTime();
- endMillis = dateTimeFormat.parse(parts[2] + " " + parts[3]).
- getTime();
- } catch (ParseException e) {
- return null;
- }
- if (startMillis >= endMillis) {
- return null;
- }
- double totalResponses = 0.0;
- try {
- totalResponses = Double.parseDouble(parts[4]);
- } catch (NumberFormatException e) {
- return null;
- }
- SortedMap<String, Double> responsesByCountry =
- parseResponses(parts[5]);
- SortedMap<String, Double> responsesByTransport =
- parseResponses(parts[6]);
- SortedMap<String, Double> responsesByVersion =
- parseResponses(parts[7]);
- if (responsesByCountry == null || responsesByTransport == null ||
- responsesByVersion == null) {
- return null;
- }
- return new ResponseHistory(startMillis, endMillis, totalResponses,
- responsesByCountry, responsesByTransport, responsesByVersion);
- }
- private static SortedMap<String, Double> parseResponses(
- String responsesString) {
- SortedMap<String, Double> responses = new TreeMap<String, Double>();
- if (responsesString.length() > 0) {
- for (String pair : responsesString.split(",")) {
- String[] keyValue = pair.split("=");
- if (keyValue.length != 2) {
- return null;
- }
- double value = 0.0;
- try {
- value = Double.parseDouble(keyValue[1]);
- } catch (NumberFormatException e) {
- return null;
- }
- responses.put(keyValue[0], value);
- }
- }
- return responses;
- }
- public String toString() {
- StringBuilder sb = new StringBuilder();
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setLenient(false);
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- sb.append(dateTimeFormat.format(startMillis));
- sb.append(" " + dateTimeFormat.format(endMillis));
- sb.append(" " + String.format("%.3f", this.totalResponses));
- this.appendResponses(sb, this.responsesByCountry);
- this.appendResponses(sb, this.responsesByTransport);
- this.appendResponses(sb, this.responsesByVersion);
- return sb.toString();
- }
- private void appendResponses(StringBuilder sb,
- SortedMap<String, Double> responses) {
- sb.append(" ");
- int written = 0;
- for (Map.Entry<String, Double> e : responses.entrySet()) {
- sb.append((written++ > 0 ? "," : "") + e.getKey() + "="
- + String.format("%.3f", e.getValue()));
- }
- }
- public void addResponses(ResponseHistory other) {
- this.totalResponses += other.totalResponses;
- this.addResponsesByCategory(this.responsesByCountry,
- other.responsesByCountry);
- this.addResponsesByCategory(this.responsesByTransport,
- other.responsesByTransport);
- this.addResponsesByCategory(this.responsesByVersion,
- other.responsesByVersion);
- if (this.startMillis > other.startMillis) {
- this.startMillis = other.startMillis;
- }
- if (this.endMillis < other.endMillis) {
- this.endMillis = other.endMillis;
- }
- }
- private void addResponsesByCategory(
- SortedMap<String, Double> thisResponses,
- SortedMap<String, Double> otherResponses) {
- for (Map.Entry<String, Double> e : otherResponses.entrySet()) {
- if (thisResponses.containsKey(e.getKey())) {
- thisResponses.put(e.getKey(), thisResponses.get(e.getKey())
- + e.getValue());
- } else {
- thisResponses.put(e.getKey(), e.getValue());
- }
- }
- }
- public int compareTo(ResponseHistory other) {
- return this.startMillis < other.startMillis ? -1 :
- this.startMillis > other.startMillis ? 1 : 0;
- }
- public boolean equals(Object other) {
- return other instanceof ResponseHistory &&
- this.startMillis == ((ResponseHistory) other).startMillis;
- }
- }
-
private DescriptorSource descriptorSource;
private DocumentStore documentStore;
@@ -230,8 +87,8 @@ public class ClientsDataWriter implements DescriptorListener,
private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L,
ONE_DAY_MILLIS = 24L * ONE_HOUR_MILLIS;
- private SortedMap<String, SortedSet<ResponseHistory>> newResponses =
- new TreeMap<String, SortedSet<ResponseHistory>>();
+ private SortedMap<String, SortedSet<ClientsHistory>> newResponses =
+ new TreeMap<String, SortedSet<ClientsHistory>>();
private void processBridgeExtraInfoDescriptor(
ExtraInfoDescriptor descriptor) {
@@ -271,12 +128,12 @@ public class ClientsDataWriter implements DescriptorListener,
SortedMap<String, Double> responsesByVersion =
this.weightResponsesWithUniqueIps(totalResponses,
descriptor.getBridgeIpVersions(), "");
- ResponseHistory newResponseHistory = new ResponseHistory(
+ ClientsHistory newResponseHistory = new ClientsHistory(
startMillis, endMillis, totalResponses, responsesByCountry,
responsesByTransport, responsesByVersion);
if (!this.newResponses.containsKey(hashedFingerprint)) {
this.newResponses.put(hashedFingerprint,
- new TreeSet<ResponseHistory>());
+ new TreeSet<ClientsHistory>());
}
this.newResponses.get(hashedFingerprint).add(
newResponseHistory);
@@ -309,44 +166,25 @@ public class ClientsDataWriter implements DescriptorListener,
}
public void updateStatuses() {
- for (Map.Entry<String, SortedSet<ResponseHistory>> e :
+ for (Map.Entry<String, SortedSet<ClientsHistory>> e :
this.newResponses.entrySet()) {
String hashedFingerprint = e.getKey();
- SortedSet<ResponseHistory> history =
- this.readHistory(hashedFingerprint);
- this.addToHistory(history, e.getValue());
- history = this.compressHistory(history);
- this.writeHistory(hashedFingerprint, history);
- }
- Logger.printStatusTime("Updated clients status files");
- }
-
- private SortedSet<ResponseHistory> readHistory(
- String hashedFingerprint) {
- SortedSet<ResponseHistory> history = new TreeSet<ResponseHistory>();
- ClientsStatus clientsStatus = this.documentStore.retrieve(
- ClientsStatus.class, false, hashedFingerprint);
- if (clientsStatus != null) {
- Scanner s = new Scanner(clientsStatus.documentString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- ResponseHistory parsedLine = ResponseHistory.fromString(line);
- if (parsedLine != null) {
- history.add(parsedLine);
- } else {
- System.err.println("Could not parse clients history line '"
- + line + "' for fingerprint '" + hashedFingerprint
- + "'. Skipping.");
- }
+ ClientsStatus clientsStatus = this.documentStore.retrieve(
+ ClientsStatus.class, true, hashedFingerprint);
+ if (clientsStatus == null) {
+ clientsStatus = new ClientsStatus();
}
- s.close();
+ this.addToHistory(clientsStatus, e.getValue());
+ this.compressHistory(clientsStatus);
+ this.documentStore.store(clientsStatus, hashedFingerprint);
}
- return history;
+ Logger.printStatusTime("Updated clients status files");
}
- private void addToHistory(SortedSet<ResponseHistory> history,
- SortedSet<ResponseHistory> newIntervals) {
- for (ResponseHistory interval : newIntervals) {
+ private void addToHistory(ClientsStatus clientsStatus,
+ SortedSet<ClientsHistory> newIntervals) {
+ SortedSet<ClientsHistory> history = clientsStatus.history;
+ for (ClientsHistory interval : newIntervals) {
if ((history.headSet(interval).isEmpty() ||
history.headSet(interval).last().endMillis <=
interval.startMillis) &&
@@ -358,15 +196,15 @@ public class ClientsDataWriter implements DescriptorListener,
}
}
- private SortedSet<ResponseHistory> compressHistory(
- SortedSet<ResponseHistory> history) {
- SortedSet<ResponseHistory> compressedHistory =
- new TreeSet<ResponseHistory>();
- ResponseHistory lastResponses = null;
+ private void compressHistory(ClientsStatus clientsStatus) {
+ SortedSet<ClientsHistory> history = clientsStatus.history;
+ SortedSet<ClientsHistory> compressedHistory =
+ new TreeSet<ClientsHistory>();
+ ClientsHistory lastResponses = null;
SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String lastMonthString = "1970-01";
- for (ResponseHistory responses : history) {
+ for (ClientsHistory responses : history) {
long intervalLengthMillis;
if (this.now - responses.endMillis <=
92L * 24L * 60L * 60L * 1000L) {
@@ -395,18 +233,7 @@ public class ClientsDataWriter implements DescriptorListener,
if (lastResponses != null) {
compressedHistory.add(lastResponses);
}
- return compressedHistory;
- }
-
- private void writeHistory(String hashedFingerprint,
- SortedSet<ResponseHistory> history) {
- StringBuilder sb = new StringBuilder();
- for (ResponseHistory responses : history) {
- sb.append(responses.toString() + "\n");
- }
- ClientsStatus clientsStatus = new ClientsStatus();
- clientsStatus.documentString = sb.toString();
- this.documentStore.store(clientsStatus, hashedFingerprint);
+ clientsStatus.history = compressedHistory;
}
public void processFingerprints(SortedSet<String> fingerprints,
@@ -420,8 +247,12 @@ public class ClientsDataWriter implements DescriptorListener,
public void writeDocuments() {
for (String hashedFingerprint : this.updateDocuments) {
- SortedSet<ResponseHistory> history =
- this.readHistory(hashedFingerprint);
+ ClientsStatus clientsStatus = this.documentStore.retrieve(
+ ClientsStatus.class, true, hashedFingerprint);
+ if (clientsStatus == null) {
+ continue;
+ }
+ SortedSet<ClientsHistory> history = clientsStatus.history;
ClientsDocument clientsDocument = new ClientsDocument();
clientsDocument.documentString = this.formatHistoryString(
hashedFingerprint, history);
@@ -452,7 +283,7 @@ public class ClientsDataWriter implements DescriptorListener,
10L * 24L * 60L * 60L * 1000L };
private String formatHistoryString(String hashedFingerprint,
- SortedSet<ResponseHistory> history) {
+ SortedSet<ClientsHistory> history) {
StringBuilder sb = new StringBuilder();
sb.append("{\"fingerprint\":\"" + hashedFingerprint + "\"");
sb.append(",\n\"average_clients\":{");
@@ -471,7 +302,7 @@ public class ClientsDataWriter implements DescriptorListener,
}
private String formatTimeline(int graphIntervalIndex,
- SortedSet<ResponseHistory> history) {
+ SortedSet<ClientsHistory> history) {
String graphName = this.graphNames[graphIntervalIndex];
long graphInterval = this.graphIntervals[graphIntervalIndex];
long dataPointInterval =
@@ -485,7 +316,7 @@ public class ClientsDataWriter implements DescriptorListener,
totalResponsesByCountry = new TreeMap<String, Double>(),
totalResponsesByTransport = new TreeMap<String, Double>(),
totalResponsesByVersion = new TreeMap<String, Double>();
- for (ResponseHistory hist : history) {
+ for (ClientsHistory hist : history) {
if (hist.endMillis < intervalStartMillis) {
continue;
}
@@ -636,7 +467,7 @@ public class ClientsDataWriter implements DescriptorListener,
public String getStatsString() {
int newIntervals = 0;
- for (SortedSet<ResponseHistory> hist : this.newResponses.values()) {
+ for (SortedSet<ClientsHistory> hist : this.newResponses.values()) {
newIntervals += hist.size();
}
StringBuilder sb = new StringBuilder();
diff --git a/src/org/torproject/onionoo/ClientsStatus.java b/src/org/torproject/onionoo/ClientsStatus.java
index 65fb341..e315fc3 100644
--- a/src/org/torproject/onionoo/ClientsStatus.java
+++ b/src/org/torproject/onionoo/ClientsStatus.java
@@ -1,5 +1,197 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
package org.torproject.onionoo;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+class ClientsHistory implements Comparable<ClientsHistory> {
+
+ long startMillis;
+
+ long endMillis;
+
+ double totalResponses;
+
+ SortedMap<String, Double> responsesByCountry;
+
+ SortedMap<String, Double> responsesByTransport;
+
+ SortedMap<String, Double> responsesByVersion;
+
+ ClientsHistory(long startMillis, long endMillis,
+ double totalResponses,
+ SortedMap<String, Double> responsesByCountry,
+ SortedMap<String, Double> responsesByTransport,
+ SortedMap<String, Double> responsesByVersion) {
+ this.startMillis = startMillis;
+ this.endMillis = endMillis;
+ this.totalResponses = totalResponses;
+ this.responsesByCountry = responsesByCountry;
+ this.responsesByTransport = responsesByTransport;
+ this.responsesByVersion = responsesByVersion;
+ }
+
+ public static ClientsHistory fromString(
+ String responseHistoryString) {
+ String[] parts = responseHistoryString.split(" ", 8);
+ if (parts.length != 8) {
+ return null;
+ }
+ long startMillis = -1L, endMillis = -1L;
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ try {
+ startMillis = dateTimeFormat.parse(parts[0] + " " + parts[1]).
+ getTime();
+ endMillis = dateTimeFormat.parse(parts[2] + " " + parts[3]).
+ getTime();
+ } catch (ParseException e) {
+ return null;
+ }
+ if (startMillis >= endMillis) {
+ return null;
+ }
+ double totalResponses = 0.0;
+ try {
+ totalResponses = Double.parseDouble(parts[4]);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ SortedMap<String, Double> responsesByCountry =
+ parseResponses(parts[5]);
+ SortedMap<String, Double> responsesByTransport =
+ parseResponses(parts[6]);
+ SortedMap<String, Double> responsesByVersion =
+ parseResponses(parts[7]);
+ if (responsesByCountry == null || responsesByTransport == null ||
+ responsesByVersion == null) {
+ return null;
+ }
+ return new ClientsHistory(startMillis, endMillis, totalResponses,
+ responsesByCountry, responsesByTransport, responsesByVersion);
+ }
+
+ private static SortedMap<String, Double> parseResponses(
+ String responsesString) {
+ SortedMap<String, Double> responses = new TreeMap<String, Double>();
+ if (responsesString.length() > 0) {
+ for (String pair : responsesString.split(",")) {
+ String[] keyValue = pair.split("=");
+ if (keyValue.length != 2) {
+ return null;
+ }
+ double value = 0.0;
+ try {
+ value = Double.parseDouble(keyValue[1]);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ responses.put(keyValue[0], value);
+ }
+ }
+ return responses;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ sb.append(dateTimeFormat.format(startMillis));
+ sb.append(" " + dateTimeFormat.format(endMillis));
+ sb.append(" " + String.format("%.3f", this.totalResponses));
+ this.appendResponses(sb, this.responsesByCountry);
+ this.appendResponses(sb, this.responsesByTransport);
+ this.appendResponses(sb, this.responsesByVersion);
+ return sb.toString();
+ }
+
+ private void appendResponses(StringBuilder sb,
+ SortedMap<String, Double> responses) {
+ sb.append(" ");
+ int written = 0;
+ for (Map.Entry<String, Double> e : responses.entrySet()) {
+ sb.append((written++ > 0 ? "," : "") + e.getKey() + "="
+ + String.format("%.3f", e.getValue()));
+ }
+ }
+
+ public void addResponses(ClientsHistory other) {
+ this.totalResponses += other.totalResponses;
+ this.addResponsesByCategory(this.responsesByCountry,
+ other.responsesByCountry);
+ this.addResponsesByCategory(this.responsesByTransport,
+ other.responsesByTransport);
+ this.addResponsesByCategory(this.responsesByVersion,
+ other.responsesByVersion);
+ if (this.startMillis > other.startMillis) {
+ this.startMillis = other.startMillis;
+ }
+ if (this.endMillis < other.endMillis) {
+ this.endMillis = other.endMillis;
+ }
+ }
+
+ private void addResponsesByCategory(
+ SortedMap<String, Double> thisResponses,
+ SortedMap<String, Double> otherResponses) {
+ for (Map.Entry<String, Double> e : otherResponses.entrySet()) {
+ if (thisResponses.containsKey(e.getKey())) {
+ thisResponses.put(e.getKey(), thisResponses.get(e.getKey())
+ + e.getValue());
+ } else {
+ thisResponses.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+
+ public int compareTo(ClientsHistory other) {
+ return this.startMillis < other.startMillis ? -1 :
+ this.startMillis > other.startMillis ? 1 : 0;
+ }
+
+ public boolean equals(Object other) {
+ return other instanceof ClientsHistory &&
+ this.startMillis == ((ClientsHistory) other).startMillis;
+ }
+}
+
class ClientsStatus extends Document {
+
+ SortedSet<ClientsHistory> history = new TreeSet<ClientsHistory>();
+
+ public void fromDocumentString(String documentString) {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ ClientsHistory parsedLine = ClientsHistory.fromString(line);
+ if (parsedLine != null) {
+ this.history.add(parsedLine);
+ } else {
+ System.err.println("Could not parse clients history line '"
+ + line + "'. Skipping.");
+ }
+ }
+ s.close();
+ }
+
+ public String toDocumentString() {
+ StringBuilder sb = new StringBuilder();
+ for (ClientsHistory interval : this.history) {
+ sb.append(interval.toString() + "\n");
+ }
+ return sb.toString();
+ }
}
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
index f786d01..387f34c 100644
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ b/src/org/torproject/onionoo/DocumentStore.java
@@ -196,7 +196,8 @@ public class DocumentStore {
document instanceof UptimeDocument) {
Gson gson = new Gson();
documentString = gson.toJson(this);
- } else if (document instanceof UptimeStatus) {
+ } else if (document instanceof ClientsStatus ||
+ document instanceof UptimeStatus) {
documentString = document.toDocumentString();
} else {
System.err.println("Serializing is not supported for type "
@@ -288,7 +289,8 @@ public class DocumentStore {
documentType.equals(UptimeDocument.class)) {
return this.retrieveParsedDocumentFile(documentType,
documentString);
- } else if (documentType.equals(UptimeStatus.class)) {
+ } else if (documentType.equals(ClientsStatus.class) ||
+ documentType.equals(UptimeStatus.class)) {
return this.retrieveParsedStatusFile(documentType, documentString);
} else {
System.err.println("Parsing is not supported for type "
More information about the tor-commits
mailing list