commit f3991be5df78d32268c87b31f0a413724d370879 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Apr 19 15:18:16 2014 +0200

Tweak node indexer to be more efficient. --- src/org/torproject/onionoo/NodeIndexer.java | 67 +++-------- src/org/torproject/onionoo/RequestHandler.java | 91 +++++++-------- src/org/torproject/onionoo/ResponseBuilder.java | 138 +++++++++++------------ 3 files changed, 127 insertions(+), 169 deletions(-)

diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java index 5dc0e0a..b1935e1 100644 --- a/src/org/torproject/onionoo/NodeIndexer.java +++ b/src/org/torproject/onionoo/NodeIndexer.java @@ -43,21 +43,22 @@ class NodeIndex { return relaysByConsensusWeight; }

- private Map<String, String> relayFingerprintSummaryLines; + + private Map<String, NodeStatus> relayFingerprintSummaryLines; public void setRelayFingerprintSummaryLines( - Map<String, String> relayFingerprintSummaryLines) { + Map<String, NodeStatus> relayFingerprintSummaryLines) { this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; } - public Map<String, String> getRelayFingerprintSummaryLines() { + public Map<String, NodeStatus> getRelayFingerprintSummaryLines() { return this.relayFingerprintSummaryLines; }

- private Map<String, String> bridgeFingerprintSummaryLines; + private Map<String, NodeStatus> bridgeFingerprintSummaryLines; public void setBridgeFingerprintSummaryLines( - Map<String, String> bridgeFingerprintSummaryLines) { + Map<String, NodeStatus> bridgeFingerprintSummaryLines) { this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; } - public Map<String, String> getBridgeFingerprintSummaryLines() { + public Map<String, NodeStatus> getBridgeFingerprintSummaryLines() { return this.bridgeFingerprintSummaryLines; }

@@ -230,9 +231,11 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } List<String> newRelaysByConsensusWeight = new ArrayList<String>(); - Map<String, String> - newRelayFingerprintSummaryLines = new HashMap<String, String>(), - newBridgeFingerprintSummaryLines = new HashMap<String, String>(); + Map<String, NodeStatus> + newRelayFingerprintSummaryLines = + new HashMap<String, NodeStatus>(), + newBridgeFingerprintSummaryLines = + new HashMap<String, NodeStatus>(); Map<String, Set<String>> newRelaysByCountryCode = new HashMap<String, Set<String>>(), newRelaysByASNumber = new HashMap<String, Set<String>>(), @@ -270,9 +273,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { toUpperCase(); entry.setRunning(entry.getLastSeenMillis() == relaysLastValidAfterMillis); - String line = formatRelaySummaryLine(entry); - newRelayFingerprintSummaryLines.put(fingerprint, line); - newRelayFingerprintSummaryLines.put(hashedFingerprint, line); + newRelayFingerprintSummaryLines.put(fingerprint, entry); + newRelayFingerprintSummaryLines.put(hashedFingerprint, entry); long consensusWeight = entry.getConsensusWeight(); orderRelaysByConsensusWeight.add(String.format("%020d %s", consensusWeight, fingerprint)); @@ -339,10 +341,9 @@ public class NodeIndexer implements ServletContextListener, Runnable { toUpperCase(); entry.setRunning(entry.getRelayFlags().contains("Running") && entry.getLastSeenMillis() == bridgesLastPublishedMillis); - String line = formatBridgeSummaryLine(entry); - newBridgeFingerprintSummaryLines.put(hashedFingerprint, line); + newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry); newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, - line); + entry); for (String flag : entry.getRelayFlags()) { String flagLowerCase = flag.toLowerCase(); if (!newBridgesByFlag.containsKey(flagLowerCase)) { @@ -398,41 +399,5 @@ public class NodeIndexer implements ServletContextListener, Runnable { this.notifyAll(); } } - - private String formatRelaySummaryLine(NodeStatus entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String fingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress); - } - for (String exitAddress : entry.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } - StringBuilder addressesBuilder = new StringBuilder(); - int written = 0; - for (String address : addresses) { - addressesBuilder.append((written++ > 0 ? "," : "") + """ - + address.toLowerCase() + """); - } - return String.format("{%s"f":"%s","a":[%s],"r":%s}", - (nickname == null ? "" : ""n":"" + nickname + "","), - fingerprint, addressesBuilder.toString(), running); - } - - private String formatBridgeSummaryLine(NodeStatus entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String hashedFingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - return String.format("{%s"h":"%s","r":%s}", - (nickname == null ? "" : ""n":"" + nickname + "","), - hashedFingerprint, running); - } }

diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java index f0e58da..1efa5a0 100644 --- a/src/org/torproject/onionoo/RequestHandler.java +++ b/src/org/torproject/onionoo/RequestHandler.java @@ -96,11 +96,11 @@ public class RequestHandler { lastSeenDays.length); }

- private Map<String, String> filteredRelays = - new HashMap<String, String>(); + private Map<String, NodeStatus> filteredRelays = + new HashMap<String, NodeStatus>();

- private Map<String, String> filteredBridges = - new HashMap<String, String>(); + private Map<String, NodeStatus> filteredBridges = + new HashMap<String, NodeStatus>();

public void handleRequest() { this.filteredRelays.putAll( @@ -149,8 +149,8 @@ public class RequestHandler { } boolean runningRequested = this.running.equals("true"); Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { - if (e.getValue().contains(""r":true") != runningRequested) { + for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) { + if (e.getValue().getRunning() != runningRequested) { removeRelays.add(e.getKey()); } } @@ -158,8 +158,8 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredBridges.entrySet()) { - if (e.getValue().contains(""r":true") != runningRequested) { + for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) { + if (e.getValue().getRunning() != runningRequested) { removeBridges.add(e.getKey()); } } @@ -179,15 +179,12 @@ public class RequestHandler {

private void filterBySearchTerm(String searchTerm) { Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { + for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) { String fingerprint = e.getKey(); - String line = e.getValue(); + NodeStatus entry = e.getValue(); boolean lineMatches = false; - String nickname = "unnamed"; - if (line.contains(""n":"")) { - nickname = line.substring(line.indexOf(""n":"") + 5). - split(""")[0].toLowerCase(); - } + String nickname = entry.getNickname() != null ? + entry.getNickname().toLowerCase() : "unnamed"; if (searchTerm.startsWith("$")) { /* Search is for $-prefixed fingerprint. */ if (fingerprint.startsWith( @@ -201,10 +198,22 @@ public class RequestHandler { } else if (fingerprint.startsWith(searchTerm.toUpperCase())) { /* Non-$-prefixed fingerprint matches. */ lineMatches = true; - } else if (line.substring(line.indexOf(""a":[")).contains(""" - + searchTerm.toLowerCase())) { - /* Address matches. */ - lineMatches = true; + } else { + List<String> addresses = new ArrayList<String>(); + addresses.add(entry.getAddress().toLowerCase()); + for (String orAddress : entry.getOrAddresses()) { + addresses.add(orAddress.toLowerCase()); + } + for (String exitAddress : entry.getExitAddresses()) { + addresses.add(exitAddress.toLowerCase()); + } + for (String address : addresses) { + if (address.startsWith(searchTerm.toLowerCase())) { + /* Address matches. */ + lineMatches = true; + break; + } + } } if (!lineMatches) { removeRelays.add(e.getKey()); @@ -214,15 +223,12 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredBridges.entrySet()) { + for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) { String hashedFingerprint = e.getKey(); - String line = e.getValue(); + NodeStatus entry = e.getValue(); boolean lineMatches = false; - String nickname = "unnamed"; - if (line.contains(""n":"")) { - nickname = line.substring(line.indexOf(""n":"") + 5). - split(""")[0].toLowerCase(); - } + String nickname = entry.getNickname() != null ? + entry.getNickname().toLowerCase() : "unnamed"; if (searchTerm.startsWith("$")) { /* Search is for $-prefixed hashed fingerprint. */ if (hashedFingerprint.startsWith( @@ -251,12 +257,12 @@ public class RequestHandler { return; } String fingerprint = this.lookup; - String relayLine = this.filteredRelays.get(fingerprint); + NodeStatus relayLine = this.filteredRelays.get(fingerprint); this.filteredRelays.clear(); if (relayLine != null) { this.filteredRelays.put(fingerprint, relayLine); } - String bridgeLine = this.filteredBridges.get(fingerprint); + NodeStatus bridgeLine = this.filteredBridges.get(fingerprint); this.filteredBridges.clear(); if (bridgeLine != null) { this.filteredBridges.put(fingerprint, bridgeLine); @@ -275,8 +281,7 @@ public class RequestHandler { Set<String> relaysWithCountryCode = this.nodeIndex.getRelaysByCountryCode().get(countryCode); Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { - String fingerprint = e.getKey(); + for (String fingerprint : this.filteredRelays.keySet()) { if (!relaysWithCountryCode.contains(fingerprint)) { removeRelays.add(fingerprint); } @@ -302,8 +307,7 @@ public class RequestHandler { Set<String> relaysWithASNumber = this.nodeIndex.getRelaysByASNumber().get(aSNumber); Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { - String fingerprint = e.getKey(); + for (String fingerprint : this.filteredRelays.keySet()) { if (!relaysWithASNumber.contains(fingerprint)) { removeRelays.add(fingerprint); } @@ -326,8 +330,7 @@ public class RequestHandler { Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get( flag); Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { - String fingerprint = e.getKey(); + for (String fingerprint : this.filteredRelays.keySet()) { if (!relaysWithFlag.contains(fingerprint)) { removeRelays.add(fingerprint); } @@ -342,9 +345,7 @@ public class RequestHandler { Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get( flag); Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, String> e : - this.filteredBridges.entrySet()) { - String fingerprint = e.getKey(); + for (String fingerprint : this.filteredBridges.keySet()) { if (!bridgesWithFlag.contains(fingerprint)) { removeBridges.add(fingerprint); } @@ -375,7 +376,7 @@ public class RequestHandler { this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays); }

- private void filterNodesByDays(Map<String, String> filteredNodes, + private void filterNodesByDays(Map<String, NodeStatus> filteredNodes, SortedMap<Integer, Set<String>> nodesByDays, int[] days) { Set<String> removeNodes = new HashSet<String>(); for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { @@ -432,14 +433,14 @@ public class RequestHandler { this.orderedRelays.add(this.filteredRelays.remove(relay)); } } - Set<String> uniqueBridges = new HashSet<String>( + Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>( this.filteredBridges.values()); this.orderedBridges.addAll(uniqueBridges); } else { - Set<String> uniqueRelays = new HashSet<String>( + Set<NodeStatus> uniqueRelays = new HashSet<NodeStatus>( this.filteredRelays.values()); this.orderedRelays.addAll(uniqueRelays); - Set<String> uniqueBridges = new HashSet<String>( + Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>( this.filteredBridges.values()); this.orderedBridges.addAll(uniqueBridges); } @@ -477,13 +478,13 @@ public class RequestHandler { } }

- private List<String> orderedRelays = new ArrayList<String>(); - public List<String> getOrderedRelays() { + private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>(); + public List<NodeStatus> getOrderedRelays() { return this.orderedRelays; }

- private List<String> orderedBridges = new ArrayList<String>(); - public List<String> getOrderedBridges() { + private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>(); + public List<NodeStatus> getOrderedBridges() { return this.orderedBridges; }

diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index d14ee5b..2b22c59 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -30,13 +30,13 @@ public class ResponseBuilder { this.bridgesPublishedString = bridgesPublishedString; }

- private List<String> orderedRelays = new ArrayList<String>(); - public void setOrderedRelays(List<String> orderedRelays) { + private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>(); + public void setOrderedRelays(List<NodeStatus> orderedRelays) { this.orderedRelays = orderedRelays; }

- private List<String> orderedBridges = new ArrayList<String>(); - public void setOrderedBridges(List<String> orderedBridges) { + private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>(); + public void setOrderedBridges(List<NodeStatus> orderedBridges) { this.orderedBridges = orderedBridges; }

@@ -51,12 +51,12 @@ public class ResponseBuilder { writeBridges(orderedBridges, pw); }

- private void writeRelays(List<String> relays, PrintWriter pw) { + private void writeRelays(List<NodeStatus> relays, PrintWriter pw) { pw.write("{"relays_published":"" + relaysPublishedString + "",\n"relays":["); int written = 0; - for (String line : relays) { - String lines = this.getFromSummaryLine(line); + for (NodeStatus entry : relays) { + String lines = this.formatNodeStatus(entry); if (lines.length() > 0) { pw.print((written++ > 0 ? ",\n" : "\n") + lines); } @@ -64,12 +64,12 @@ public class ResponseBuilder { pw.print("\n],\n"); }

- private void writeBridges(List<String> bridges, PrintWriter pw) { + private void writeBridges(List<NodeStatus> bridges, PrintWriter pw) { pw.write(""bridges_published":"" + bridgesPublishedString + "",\n"bridges":["); int written = 0; - for (String line : bridges) { - String lines = this.getFromSummaryLine(line); + for (NodeStatus entry : bridges) { + String lines = this.formatNodeStatus(entry); if (lines.length() > 0) { pw.print((written++ > 0 ? ",\n" : "\n") + lines); } @@ -77,43 +77,69 @@ public class ResponseBuilder { pw.print("\n]}\n"); }

- private String getFromSummaryLine(String summaryLine) { + private String formatNodeStatus(NodeStatus entry) { if (this.resourceType == null) { return ""; } else if (this.resourceType.equals("summary")) { - return this.writeSummaryLine(summaryLine); + return this.writeSummaryLine(entry); } else if (this.resourceType.equals("details")) { - return this.writeDetailsLines(summaryLine); + return this.writeDetailsLines(entry); } else if (this.resourceType.equals("bandwidth")) { - return this.writeBandwidthLines(summaryLine); + return this.writeBandwidthLines(entry); } else if (this.resourceType.equals("weights")) { - return this.writeWeightsLines(summaryLine); + return this.writeWeightsLines(entry); } else if (this.resourceType.equals("clients")) { - return this.writeClientsLines(summaryLine); + return this.writeClientsLines(entry); } else if (this.resourceType.equals("uptime")) { - return this.writeUptimeLines(summaryLine); + return this.writeUptimeLines(entry); } else { return ""; } }

- private String writeSummaryLine(String summaryLine) { - return (summaryLine.endsWith(",") ? summaryLine.substring(0, - summaryLine.length() - 1) : summaryLine); + private String writeSummaryLine(NodeStatus entry) { + return entry.isRelay() ? writeRelaySummaryLine(entry) + : writeBridgeSummaryLine(entry); }

- private String writeDetailsLines(String summaryLine) { - String fingerprint = null; - if (summaryLine.contains(""f":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""f":"") + ""f":"".length()); - } else if (summaryLine.contains(""h":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""h":"") + ""h":"".length()); - } else { - return ""; + private String writeRelaySummaryLine(NodeStatus entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String fingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + List<String> addresses = new ArrayList<String>(); + addresses.add(entry.getAddress()); + for (String orAddress : entry.getOrAddresses()) { + addresses.add(orAddress); + } + for (String exitAddress : entry.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + StringBuilder addressesBuilder = new StringBuilder(); + int written = 0; + for (String address : addresses) { + addressesBuilder.append((written++ > 0 ? "," : "") + """ + + address.toLowerCase() + """); } - fingerprint = fingerprint.substring(0, 40); + return String.format("{%s"f":"%s","a":[%s],"r":%s}", + (nickname == null ? "" : ""n":"" + nickname + "","), + fingerprint, addressesBuilder.toString(), running); + } + + private String writeBridgeSummaryLine(NodeStatus entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String hashedFingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + return String.format("{%s"h":"%s","r":%s}", + (nickname == null ? "" : ""n":"" + nickname + "","), + hashedFingerprint, running); + } + + private String writeDetailsLines(NodeStatus entry) { + String fingerprint = entry.getFingerprint(); DetailsDocument detailsDocument = this.documentStore.retrieve( DetailsDocument.class, false, fingerprint); if (detailsDocument != null && @@ -172,18 +198,8 @@ public class ResponseBuilder { } }

- private String writeBandwidthLines(String summaryLine) { - String fingerprint = null; - if (summaryLine.contains(""f":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""f":"") + ""f":"".length()); - } else if (summaryLine.contains(""h":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""h":"") + ""h":"".length()); - } else { - return ""; - } - fingerprint = fingerprint.substring(0, 40); + private String writeBandwidthLines(NodeStatus entry) { + String fingerprint = entry.getFingerprint(); BandwidthDocument bandwidthDocument = this.documentStore.retrieve( BandwidthDocument.class, false, fingerprint); if (bandwidthDocument != null && @@ -199,15 +215,8 @@ public class ResponseBuilder { } }

- private String writeWeightsLines(String summaryLine) { - String fingerprint = null; - if (summaryLine.contains(""f":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""f":"") + ""f":"".length()); - } else { - return ""; - } - fingerprint = fingerprint.substring(0, 40); + private String writeWeightsLines(NodeStatus entry) { + String fingerprint = entry.getFingerprint(); WeightsDocument weightsDocument = this.documentStore.retrieve( WeightsDocument.class, false, fingerprint); if (weightsDocument != null && @@ -222,15 +231,8 @@ public class ResponseBuilder { } }

- private String writeClientsLines(String summaryLine) { - String fingerprint = null; - if (summaryLine.contains(""h":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""h":"") + ""h":"".length()); - } else { - return ""; - } - fingerprint = fingerprint.substring(0, 40); + private String writeClientsLines(NodeStatus entry) { + String fingerprint = entry.getFingerprint(); ClientsDocument clientsDocument = this.documentStore.retrieve( ClientsDocument.class, false, fingerprint); if (clientsDocument != null && @@ -245,18 +247,8 @@ public class ResponseBuilder { } }

- private String writeUptimeLines(String summaryLine) { - String fingerprint = null; - if (summaryLine.contains(""f":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""f":"") + ""f":"".length()); - } else if (summaryLine.contains(""h":"")) { - fingerprint = summaryLine.substring(summaryLine.indexOf( - ""h":"") + ""h":"".length()); - } else { - return ""; - } - fingerprint = fingerprint.substring(0, 40); + private String writeUptimeLines(NodeStatus entry) { + String fingerprint = entry.getFingerprint(); UptimeDocument uptimeDocument = this.documentStore.retrieve( UptimeDocument.class, false, fingerprint); if (uptimeDocument != null &&