[tor-commits] [onionoo/master] Tweak node indexer to be more efficient.
karsten at torproject.org
karsten at torproject.org
Sat Apr 19 22:48:51 UTC 2014
commit f3991be5df78d32268c87b31f0a413724d370879
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sat Apr 19 15:18:16 2014 +0200
Tweak node indexer to be more efficient.
---
src/org/torproject/onionoo/NodeIndexer.java | 67 +++--------
src/org/torproject/onionoo/RequestHandler.java | 91 +++++++--------
src/org/torproject/onionoo/ResponseBuilder.java | 138 +++++++++++------------
3 files changed, 127 insertions(+), 169 deletions(-)
diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java
index 5dc0e0a..b1935e1 100644
--- a/src/org/torproject/onionoo/NodeIndexer.java
+++ b/src/org/torproject/onionoo/NodeIndexer.java
@@ -43,21 +43,22 @@ class NodeIndex {
return relaysByConsensusWeight;
}
- private Map<String, String> relayFingerprintSummaryLines;
+
+ private Map<String, NodeStatus> relayFingerprintSummaryLines;
public void setRelayFingerprintSummaryLines(
- Map<String, String> relayFingerprintSummaryLines) {
+ Map<String, NodeStatus> relayFingerprintSummaryLines) {
this.relayFingerprintSummaryLines = relayFingerprintSummaryLines;
}
- public Map<String, String> getRelayFingerprintSummaryLines() {
+ public Map<String, NodeStatus> getRelayFingerprintSummaryLines() {
return this.relayFingerprintSummaryLines;
}
- private Map<String, String> bridgeFingerprintSummaryLines;
+ private Map<String, NodeStatus> bridgeFingerprintSummaryLines;
public void setBridgeFingerprintSummaryLines(
- Map<String, String> bridgeFingerprintSummaryLines) {
+ Map<String, NodeStatus> bridgeFingerprintSummaryLines) {
this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines;
}
- public Map<String, String> getBridgeFingerprintSummaryLines() {
+ public Map<String, NodeStatus> getBridgeFingerprintSummaryLines() {
return this.bridgeFingerprintSummaryLines;
}
@@ -230,9 +231,11 @@ public class NodeIndexer implements ServletContextListener, Runnable {
}
}
List<String> newRelaysByConsensusWeight = new ArrayList<String>();
- Map<String, String>
- newRelayFingerprintSummaryLines = new HashMap<String, String>(),
- newBridgeFingerprintSummaryLines = new HashMap<String, String>();
+ Map<String, NodeStatus>
+ newRelayFingerprintSummaryLines =
+ new HashMap<String, NodeStatus>(),
+ newBridgeFingerprintSummaryLines =
+ new HashMap<String, NodeStatus>();
Map<String, Set<String>>
newRelaysByCountryCode = new HashMap<String, Set<String>>(),
newRelaysByASNumber = new HashMap<String, Set<String>>(),
@@ -270,9 +273,8 @@ public class NodeIndexer implements ServletContextListener, Runnable {
toUpperCase();
entry.setRunning(entry.getLastSeenMillis() ==
relaysLastValidAfterMillis);
- String line = formatRelaySummaryLine(entry);
- newRelayFingerprintSummaryLines.put(fingerprint, line);
- newRelayFingerprintSummaryLines.put(hashedFingerprint, line);
+ newRelayFingerprintSummaryLines.put(fingerprint, entry);
+ newRelayFingerprintSummaryLines.put(hashedFingerprint, entry);
long consensusWeight = entry.getConsensusWeight();
orderRelaysByConsensusWeight.add(String.format("%020d %s",
consensusWeight, fingerprint));
@@ -339,10 +341,9 @@ public class NodeIndexer implements ServletContextListener, Runnable {
toUpperCase();
entry.setRunning(entry.getRelayFlags().contains("Running") &&
entry.getLastSeenMillis() == bridgesLastPublishedMillis);
- String line = formatBridgeSummaryLine(entry);
- newBridgeFingerprintSummaryLines.put(hashedFingerprint, line);
+ newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry);
newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint,
- line);
+ entry);
for (String flag : entry.getRelayFlags()) {
String flagLowerCase = flag.toLowerCase();
if (!newBridgesByFlag.containsKey(flagLowerCase)) {
@@ -398,41 +399,5 @@ public class NodeIndexer implements ServletContextListener, Runnable {
this.notifyAll();
}
}
-
- private String formatRelaySummaryLine(NodeStatus entry) {
- String nickname = !entry.getNickname().equals("Unnamed") ?
- entry.getNickname() : null;
- String fingerprint = entry.getFingerprint();
- String running = entry.getRunning() ? "true" : "false";
- List<String> addresses = new ArrayList<String>();
- addresses.add(entry.getAddress());
- for (String orAddress : entry.getOrAddresses()) {
- addresses.add(orAddress);
- }
- for (String exitAddress : entry.getExitAddresses()) {
- if (!addresses.contains(exitAddress)) {
- addresses.add(exitAddress);
- }
- }
- StringBuilder addressesBuilder = new StringBuilder();
- int written = 0;
- for (String address : addresses) {
- addressesBuilder.append((written++ > 0 ? "," : "") + "\""
- + address.toLowerCase() + "\"");
- }
- return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}",
- (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
- fingerprint, addressesBuilder.toString(), running);
- }
-
- private String formatBridgeSummaryLine(NodeStatus entry) {
- String nickname = !entry.getNickname().equals("Unnamed") ?
- entry.getNickname() : null;
- String hashedFingerprint = entry.getFingerprint();
- String running = entry.getRunning() ? "true" : "false";
- return String.format("{%s\"h\":\"%s\",\"r\":%s}",
- (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
- hashedFingerprint, running);
- }
}
diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java
index f0e58da..1efa5a0 100644
--- a/src/org/torproject/onionoo/RequestHandler.java
+++ b/src/org/torproject/onionoo/RequestHandler.java
@@ -96,11 +96,11 @@ public class RequestHandler {
lastSeenDays.length);
}
- private Map<String, String> filteredRelays =
- new HashMap<String, String>();
+ private Map<String, NodeStatus> filteredRelays =
+ new HashMap<String, NodeStatus>();
- private Map<String, String> filteredBridges =
- new HashMap<String, String>();
+ private Map<String, NodeStatus> filteredBridges =
+ new HashMap<String, NodeStatus>();
public void handleRequest() {
this.filteredRelays.putAll(
@@ -149,8 +149,8 @@ public class RequestHandler {
}
boolean runningRequested = this.running.equals("true");
Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, String> e : filteredRelays.entrySet()) {
- if (e.getValue().contains("\"r\":true") != runningRequested) {
+ for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) {
+ if (e.getValue().getRunning() != runningRequested) {
removeRelays.add(e.getKey());
}
}
@@ -158,8 +158,8 @@ public class RequestHandler {
this.filteredRelays.remove(fingerprint);
}
Set<String> removeBridges = new HashSet<String>();
- for (Map.Entry<String, String> e : filteredBridges.entrySet()) {
- if (e.getValue().contains("\"r\":true") != runningRequested) {
+ for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) {
+ if (e.getValue().getRunning() != runningRequested) {
removeBridges.add(e.getKey());
}
}
@@ -179,15 +179,12 @@ public class RequestHandler {
private void filterBySearchTerm(String searchTerm) {
Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, String> e : filteredRelays.entrySet()) {
+ for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) {
String fingerprint = e.getKey();
- String line = e.getValue();
+ NodeStatus entry = e.getValue();
boolean lineMatches = false;
- String nickname = "unnamed";
- if (line.contains("\"n\":\"")) {
- nickname = line.substring(line.indexOf("\"n\":\"") + 5).
- split("\"")[0].toLowerCase();
- }
+ String nickname = entry.getNickname() != null ?
+ entry.getNickname().toLowerCase() : "unnamed";
if (searchTerm.startsWith("$")) {
/* Search is for $-prefixed fingerprint. */
if (fingerprint.startsWith(
@@ -201,10 +198,22 @@ public class RequestHandler {
} else if (fingerprint.startsWith(searchTerm.toUpperCase())) {
/* Non-$-prefixed fingerprint matches. */
lineMatches = true;
- } else if (line.substring(line.indexOf("\"a\":[")).contains("\""
- + searchTerm.toLowerCase())) {
- /* Address matches. */
- lineMatches = true;
+ } else {
+ List<String> addresses = new ArrayList<String>();
+ addresses.add(entry.getAddress().toLowerCase());
+ for (String orAddress : entry.getOrAddresses()) {
+ addresses.add(orAddress.toLowerCase());
+ }
+ for (String exitAddress : entry.getExitAddresses()) {
+ addresses.add(exitAddress.toLowerCase());
+ }
+ for (String address : addresses) {
+ if (address.startsWith(searchTerm.toLowerCase())) {
+ /* Address matches. */
+ lineMatches = true;
+ break;
+ }
+ }
}
if (!lineMatches) {
removeRelays.add(e.getKey());
@@ -214,15 +223,12 @@ public class RequestHandler {
this.filteredRelays.remove(fingerprint);
}
Set<String> removeBridges = new HashSet<String>();
- for (Map.Entry<String, String> e : filteredBridges.entrySet()) {
+ for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) {
String hashedFingerprint = e.getKey();
- String line = e.getValue();
+ NodeStatus entry = e.getValue();
boolean lineMatches = false;
- String nickname = "unnamed";
- if (line.contains("\"n\":\"")) {
- nickname = line.substring(line.indexOf("\"n\":\"") + 5).
- split("\"")[0].toLowerCase();
- }
+ String nickname = entry.getNickname() != null ?
+ entry.getNickname().toLowerCase() : "unnamed";
if (searchTerm.startsWith("$")) {
/* Search is for $-prefixed hashed fingerprint. */
if (hashedFingerprint.startsWith(
@@ -251,12 +257,12 @@ public class RequestHandler {
return;
}
String fingerprint = this.lookup;
- String relayLine = this.filteredRelays.get(fingerprint);
+ NodeStatus relayLine = this.filteredRelays.get(fingerprint);
this.filteredRelays.clear();
if (relayLine != null) {
this.filteredRelays.put(fingerprint, relayLine);
}
- String bridgeLine = this.filteredBridges.get(fingerprint);
+ NodeStatus bridgeLine = this.filteredBridges.get(fingerprint);
this.filteredBridges.clear();
if (bridgeLine != null) {
this.filteredBridges.put(fingerprint, bridgeLine);
@@ -275,8 +281,7 @@ public class RequestHandler {
Set<String> relaysWithCountryCode =
this.nodeIndex.getRelaysByCountryCode().get(countryCode);
Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) {
- String fingerprint = e.getKey();
+ for (String fingerprint : this.filteredRelays.keySet()) {
if (!relaysWithCountryCode.contains(fingerprint)) {
removeRelays.add(fingerprint);
}
@@ -302,8 +307,7 @@ public class RequestHandler {
Set<String> relaysWithASNumber =
this.nodeIndex.getRelaysByASNumber().get(aSNumber);
Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) {
- String fingerprint = e.getKey();
+ for (String fingerprint : this.filteredRelays.keySet()) {
if (!relaysWithASNumber.contains(fingerprint)) {
removeRelays.add(fingerprint);
}
@@ -326,8 +330,7 @@ public class RequestHandler {
Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get(
flag);
Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) {
- String fingerprint = e.getKey();
+ for (String fingerprint : this.filteredRelays.keySet()) {
if (!relaysWithFlag.contains(fingerprint)) {
removeRelays.add(fingerprint);
}
@@ -342,9 +345,7 @@ public class RequestHandler {
Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get(
flag);
Set<String> removeBridges = new HashSet<String>();
- for (Map.Entry<String, String> e :
- this.filteredBridges.entrySet()) {
- String fingerprint = e.getKey();
+ for (String fingerprint : this.filteredBridges.keySet()) {
if (!bridgesWithFlag.contains(fingerprint)) {
removeBridges.add(fingerprint);
}
@@ -375,7 +376,7 @@ public class RequestHandler {
this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays);
}
- private void filterNodesByDays(Map<String, String> filteredNodes,
+ private void filterNodesByDays(Map<String, NodeStatus> filteredNodes,
SortedMap<Integer, Set<String>> nodesByDays, int[] days) {
Set<String> removeNodes = new HashSet<String>();
for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) {
@@ -432,14 +433,14 @@ public class RequestHandler {
this.orderedRelays.add(this.filteredRelays.remove(relay));
}
}
- Set<String> uniqueBridges = new HashSet<String>(
+ Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>(
this.filteredBridges.values());
this.orderedBridges.addAll(uniqueBridges);
} else {
- Set<String> uniqueRelays = new HashSet<String>(
+ Set<NodeStatus> uniqueRelays = new HashSet<NodeStatus>(
this.filteredRelays.values());
this.orderedRelays.addAll(uniqueRelays);
- Set<String> uniqueBridges = new HashSet<String>(
+ Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>(
this.filteredBridges.values());
this.orderedBridges.addAll(uniqueBridges);
}
@@ -477,13 +478,13 @@ public class RequestHandler {
}
}
- private List<String> orderedRelays = new ArrayList<String>();
- public List<String> getOrderedRelays() {
+ private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>();
+ public List<NodeStatus> getOrderedRelays() {
return this.orderedRelays;
}
- private List<String> orderedBridges = new ArrayList<String>();
- public List<String> getOrderedBridges() {
+ private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>();
+ public List<NodeStatus> getOrderedBridges() {
return this.orderedBridges;
}
diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java
index d14ee5b..2b22c59 100644
--- a/src/org/torproject/onionoo/ResponseBuilder.java
+++ b/src/org/torproject/onionoo/ResponseBuilder.java
@@ -30,13 +30,13 @@ public class ResponseBuilder {
this.bridgesPublishedString = bridgesPublishedString;
}
- private List<String> orderedRelays = new ArrayList<String>();
- public void setOrderedRelays(List<String> orderedRelays) {
+ private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>();
+ public void setOrderedRelays(List<NodeStatus> orderedRelays) {
this.orderedRelays = orderedRelays;
}
- private List<String> orderedBridges = new ArrayList<String>();
- public void setOrderedBridges(List<String> orderedBridges) {
+ private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>();
+ public void setOrderedBridges(List<NodeStatus> orderedBridges) {
this.orderedBridges = orderedBridges;
}
@@ -51,12 +51,12 @@ public class ResponseBuilder {
writeBridges(orderedBridges, pw);
}
- private void writeRelays(List<String> relays, PrintWriter pw) {
+ private void writeRelays(List<NodeStatus> relays, PrintWriter pw) {
pw.write("{\"relays_published\":\"" + relaysPublishedString
+ "\",\n\"relays\":[");
int written = 0;
- for (String line : relays) {
- String lines = this.getFromSummaryLine(line);
+ for (NodeStatus entry : relays) {
+ String lines = this.formatNodeStatus(entry);
if (lines.length() > 0) {
pw.print((written++ > 0 ? ",\n" : "\n") + lines);
}
@@ -64,12 +64,12 @@ public class ResponseBuilder {
pw.print("\n],\n");
}
- private void writeBridges(List<String> bridges, PrintWriter pw) {
+ private void writeBridges(List<NodeStatus> bridges, PrintWriter pw) {
pw.write("\"bridges_published\":\"" + bridgesPublishedString
+ "\",\n\"bridges\":[");
int written = 0;
- for (String line : bridges) {
- String lines = this.getFromSummaryLine(line);
+ for (NodeStatus entry : bridges) {
+ String lines = this.formatNodeStatus(entry);
if (lines.length() > 0) {
pw.print((written++ > 0 ? ",\n" : "\n") + lines);
}
@@ -77,43 +77,69 @@ public class ResponseBuilder {
pw.print("\n]}\n");
}
- private String getFromSummaryLine(String summaryLine) {
+ private String formatNodeStatus(NodeStatus entry) {
if (this.resourceType == null) {
return "";
} else if (this.resourceType.equals("summary")) {
- return this.writeSummaryLine(summaryLine);
+ return this.writeSummaryLine(entry);
} else if (this.resourceType.equals("details")) {
- return this.writeDetailsLines(summaryLine);
+ return this.writeDetailsLines(entry);
} else if (this.resourceType.equals("bandwidth")) {
- return this.writeBandwidthLines(summaryLine);
+ return this.writeBandwidthLines(entry);
} else if (this.resourceType.equals("weights")) {
- return this.writeWeightsLines(summaryLine);
+ return this.writeWeightsLines(entry);
} else if (this.resourceType.equals("clients")) {
- return this.writeClientsLines(summaryLine);
+ return this.writeClientsLines(entry);
} else if (this.resourceType.equals("uptime")) {
- return this.writeUptimeLines(summaryLine);
+ return this.writeUptimeLines(entry);
} else {
return "";
}
}
- private String writeSummaryLine(String summaryLine) {
- return (summaryLine.endsWith(",") ? summaryLine.substring(0,
- summaryLine.length() - 1) : summaryLine);
+ private String writeSummaryLine(NodeStatus entry) {
+ return entry.isRelay() ? writeRelaySummaryLine(entry)
+ : writeBridgeSummaryLine(entry);
}
- private String writeDetailsLines(String summaryLine) {
- String fingerprint = null;
- if (summaryLine.contains("\"f\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"f\":\"") + "\"f\":\"".length());
- } else if (summaryLine.contains("\"h\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"h\":\"") + "\"h\":\"".length());
- } else {
- return "";
+ private String writeRelaySummaryLine(NodeStatus entry) {
+ String nickname = !entry.getNickname().equals("Unnamed") ?
+ entry.getNickname() : null;
+ String fingerprint = entry.getFingerprint();
+ String running = entry.getRunning() ? "true" : "false";
+ List<String> addresses = new ArrayList<String>();
+ addresses.add(entry.getAddress());
+ for (String orAddress : entry.getOrAddresses()) {
+ addresses.add(orAddress);
+ }
+ for (String exitAddress : entry.getExitAddresses()) {
+ if (!addresses.contains(exitAddress)) {
+ addresses.add(exitAddress);
+ }
+ }
+ StringBuilder addressesBuilder = new StringBuilder();
+ int written = 0;
+ for (String address : addresses) {
+ addressesBuilder.append((written++ > 0 ? "," : "") + "\""
+ + address.toLowerCase() + "\"");
}
- fingerprint = fingerprint.substring(0, 40);
+ return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}",
+ (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
+ fingerprint, addressesBuilder.toString(), running);
+ }
+
+ private String writeBridgeSummaryLine(NodeStatus entry) {
+ String nickname = !entry.getNickname().equals("Unnamed") ?
+ entry.getNickname() : null;
+ String hashedFingerprint = entry.getFingerprint();
+ String running = entry.getRunning() ? "true" : "false";
+ return String.format("{%s\"h\":\"%s\",\"r\":%s}",
+ (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
+ hashedFingerprint, running);
+ }
+
+ private String writeDetailsLines(NodeStatus entry) {
+ String fingerprint = entry.getFingerprint();
DetailsDocument detailsDocument = this.documentStore.retrieve(
DetailsDocument.class, false, fingerprint);
if (detailsDocument != null &&
@@ -172,18 +198,8 @@ public class ResponseBuilder {
}
}
- private String writeBandwidthLines(String summaryLine) {
- String fingerprint = null;
- if (summaryLine.contains("\"f\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"f\":\"") + "\"f\":\"".length());
- } else if (summaryLine.contains("\"h\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"h\":\"") + "\"h\":\"".length());
- } else {
- return "";
- }
- fingerprint = fingerprint.substring(0, 40);
+ private String writeBandwidthLines(NodeStatus entry) {
+ String fingerprint = entry.getFingerprint();
BandwidthDocument bandwidthDocument = this.documentStore.retrieve(
BandwidthDocument.class, false, fingerprint);
if (bandwidthDocument != null &&
@@ -199,15 +215,8 @@ public class ResponseBuilder {
}
}
- private String writeWeightsLines(String summaryLine) {
- String fingerprint = null;
- if (summaryLine.contains("\"f\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"f\":\"") + "\"f\":\"".length());
- } else {
- return "";
- }
- fingerprint = fingerprint.substring(0, 40);
+ private String writeWeightsLines(NodeStatus entry) {
+ String fingerprint = entry.getFingerprint();
WeightsDocument weightsDocument = this.documentStore.retrieve(
WeightsDocument.class, false, fingerprint);
if (weightsDocument != null &&
@@ -222,15 +231,8 @@ public class ResponseBuilder {
}
}
- private String writeClientsLines(String summaryLine) {
- String fingerprint = null;
- if (summaryLine.contains("\"h\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"h\":\"") + "\"h\":\"".length());
- } else {
- return "";
- }
- fingerprint = fingerprint.substring(0, 40);
+ private String writeClientsLines(NodeStatus entry) {
+ String fingerprint = entry.getFingerprint();
ClientsDocument clientsDocument = this.documentStore.retrieve(
ClientsDocument.class, false, fingerprint);
if (clientsDocument != null &&
@@ -245,18 +247,8 @@ public class ResponseBuilder {
}
}
- private String writeUptimeLines(String summaryLine) {
- String fingerprint = null;
- if (summaryLine.contains("\"f\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"f\":\"") + "\"f\":\"".length());
- } else if (summaryLine.contains("\"h\":\"")) {
- fingerprint = summaryLine.substring(summaryLine.indexOf(
- "\"h\":\"") + "\"h\":\"".length());
- } else {
- return "";
- }
- fingerprint = fingerprint.substring(0, 40);
+ private String writeUptimeLines(NodeStatus entry) {
+ String fingerprint = entry.getFingerprint();
UptimeDocument uptimeDocument = this.documentStore.retrieve(
UptimeDocument.class, false, fingerprint);
if (uptimeDocument != null &&
More information about the tor-commits
mailing list