commit 46743dc511827a12199a6d8177ff2d5591d7dd61 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu Mar 20 18:36:53 2014 +0100 Split up ResponseBuilder class. Suggested by SonarQube. --- src/org/torproject/onionoo/RequestHandler.java | 757 +++++++++++++++++++++++ src/org/torproject/onionoo/ResourceServlet.java | 49 +- src/org/torproject/onionoo/ResponseBuilder.java | 724 +--------------------- 3 files changed, 800 insertions(+), 730 deletions(-) diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java new file mode 100644 index 0000000..6a25549 --- /dev/null +++ b/src/org/torproject/onionoo/RequestHandler.java @@ -0,0 +1,757 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; + +public class RequestHandler { + + private static long summaryFileLastModified = -1L; + private static DocumentStore documentStore; + private static Time time; + private static boolean successfullyReadSummaryFile = false; + private static String relaysPublishedString, bridgesPublishedString; + private static List<String> relaysByConsensusWeight = null; + private static Map<String, String> relayFingerprintSummaryLines = null, + bridgeFingerprintSummaryLines = null; + private static Map<String, Set<String>> relaysByCountryCode = null, + relaysByASNumber = null, relaysByFlag = null, bridgesByFlag = null, + relaysByContact = null; + private static SortedMap<Integer, Set<String>> + relaysByFirstSeenDays = null, bridgesByFirstSeenDays = null, + relaysByLastSeenDays = null, bridgesByLastSeenDays = null; + private static final long SUMMARY_MAX_AGE = DateTimeHelper.SIX_HOURS; + + public static void initialize(DocumentStore documentStoreParam, + Time timeParam) { + documentStore = documentStoreParam; + time = timeParam; + readSummaryFile(); + } + + public static boolean update() { + readSummaryFile(); + return successfullyReadSummaryFile; + } + + private static void readSummaryFile() { + long newSummaryFileLastModified = -1L; + UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, + false); + if (updateStatus != null && + updateStatus.getDocumentString() != null) { + String updateString = updateStatus.getDocumentString(); + try { + newSummaryFileLastModified = Long.parseLong(updateString.trim()); + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (newSummaryFileLastModified < 0L) { + // TODO Does this actually solve anything? Should we instead + // switch to a variant of the maintenance mode and re-check when + // the next requests comes in that happens x seconds after this one? + successfullyReadSummaryFile = false; + return; + } + if (newSummaryFileLastModified + SUMMARY_MAX_AGE + < time.currentTimeMillis()) { + // TODO Does this actually solve anything? Should we instead + // switch to a variant of the maintenance mode and re-check when + // the next requests comes in that happens x seconds after this one? + successfullyReadSummaryFile = false; + return; + } + if (newSummaryFileLastModified > summaryFileLastModified) { + List<String> newRelaysByConsensusWeight = new ArrayList<String>(); + Map<String, String> + newRelayFingerprintSummaryLines = new HashMap<String, String>(), + newBridgeFingerprintSummaryLines = + new HashMap<String, String>(); + Map<String, Set<String>> + newRelaysByCountryCode = new HashMap<String, Set<String>>(), + newRelaysByASNumber = new HashMap<String, Set<String>>(), + newRelaysByFlag = new HashMap<String, Set<String>>(), + newBridgesByFlag = new HashMap<String, Set<String>>(), + newRelaysByContact = new HashMap<String, Set<String>>(); + SortedMap<Integer, Set<String>> + newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), + newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), + newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), + newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); + long relaysLastValidAfterMillis = -1L, + bridgesLastPublishedMillis = -1L; + String newRelaysPublishedString, newBridgesPublishedString; + Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), + currentBridges = new HashSet<NodeStatus>(); + SortedSet<String> fingerprints = documentStore.list( + NodeStatus.class, false); + // TODO We should be able to learn if something goes wrong when + // reading the summary file, rather than silently having an empty + // list of fingerprints. + for (String fingerprint : fingerprints) { + NodeStatus node = documentStore.retrieve(NodeStatus.class, true, + fingerprint); + if (node.isRelay()) { + relaysLastValidAfterMillis = Math.max( + relaysLastValidAfterMillis, node.getLastSeenMillis()); + currentRelays.add(node); + } else { + bridgesLastPublishedMillis = Math.max( + bridgesLastPublishedMillis, node.getLastSeenMillis()); + currentBridges.add(node); + } + } + newRelaysPublishedString = DateTimeHelper.format( + relaysLastValidAfterMillis); + newBridgesPublishedString = DateTimeHelper.format( + bridgesLastPublishedMillis); + List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); + for (NodeStatus entry : currentRelays) { + String fingerprint = entry.getFingerprint().toUpperCase(); + String hashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + entry.setRunning(entry.getLastSeenMillis() == + relaysLastValidAfterMillis); + String line = formatRelaySummaryLine(entry); + newRelayFingerprintSummaryLines.put(fingerprint, line); + newRelayFingerprintSummaryLines.put(hashedFingerprint, line); + long consensusWeight = entry.getConsensusWeight(); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, fingerprint)); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, hashedFingerprint)); + if (entry.getCountryCode() != null) { + String countryCode = entry.getCountryCode(); + if (!newRelaysByCountryCode.containsKey(countryCode)) { + newRelaysByCountryCode.put(countryCode, + new HashSet<String>()); + } + newRelaysByCountryCode.get(countryCode).add(fingerprint); + newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); + } + if (entry.getASNumber() != null) { + String aSNumber = entry.getASNumber(); + if (!newRelaysByASNumber.containsKey(aSNumber)) { + newRelaysByASNumber.put(aSNumber, new HashSet<String>()); + } + newRelaysByASNumber.get(aSNumber).add(fingerprint); + newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); + } + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!newRelaysByFlag.containsKey(flagLowerCase)) { + newRelaysByFlag.put(flagLowerCase, new HashSet<String>()); + } + newRelaysByFlag.get(flagLowerCase).add(fingerprint); + newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); + } + int daysSinceFirstSeen = (int) ((newSummaryFileLastModified + - entry.getFirstSeenMillis()) / 86400000L); + if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { + newRelaysByFirstSeenDays.put(daysSinceFirstSeen, + new HashSet<String>()); + } + newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint); + newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedFingerprint); + int daysSinceLastSeen = (int) ((newSummaryFileLastModified + - entry.getLastSeenMillis()) / 86400000L); + if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) { + newRelaysByLastSeenDays.put(daysSinceLastSeen, + new HashSet<String>()); + } + newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); + newRelaysByLastSeenDays.get(daysSinceLastSeen).add( + hashedFingerprint); + String contact = entry.getContact(); + if (!newRelaysByContact.containsKey(contact)) { + newRelaysByContact.put(contact, new HashSet<String>()); + } + newRelaysByContact.get(contact).add(fingerprint); + newRelaysByContact.get(contact).add(hashedFingerprint); + } + Collections.sort(orderRelaysByConsensusWeight); + newRelaysByConsensusWeight = new ArrayList<String>(); + for (String relay : orderRelaysByConsensusWeight) { + newRelaysByConsensusWeight.add(relay.split(" ")[1]); + } + for (NodeStatus entry : currentBridges) { + String hashedFingerprint = entry.getFingerprint().toUpperCase(); + String hashedHashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + entry.setRunning(entry.getRelayFlags().contains("Running") && + entry.getLastSeenMillis() == bridgesLastPublishedMillis); + String line = formatBridgeSummaryLine(entry); + newBridgeFingerprintSummaryLines.put(hashedFingerprint, line); + newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, + line); + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!newBridgesByFlag.containsKey(flagLowerCase)) { + newBridgesByFlag.put(flagLowerCase, new HashSet<String>()); + } + newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint); + newBridgesByFlag.get(flagLowerCase).add( + hashedHashedFingerprint); + } + int daysSinceFirstSeen = (int) ((newSummaryFileLastModified + - entry.getFirstSeenMillis()) / 86400000L); + if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) { + newBridgesByFirstSeenDays.put(daysSinceFirstSeen, + new HashSet<String>()); + } + newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedFingerprint); + newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedHashedFingerprint); + int daysSinceLastSeen = (int) ((newSummaryFileLastModified + - entry.getLastSeenMillis()) / 86400000L); + if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) { + newBridgesByLastSeenDays.put(daysSinceLastSeen, + new HashSet<String>()); + } + newBridgesByLastSeenDays.get(daysSinceLastSeen).add( + hashedFingerprint); + newBridgesByLastSeenDays.get(daysSinceLastSeen).add( + hashedHashedFingerprint); + } + relaysByConsensusWeight = newRelaysByConsensusWeight; + relayFingerprintSummaryLines = newRelayFingerprintSummaryLines; + bridgeFingerprintSummaryLines = newBridgeFingerprintSummaryLines; + relaysByCountryCode = newRelaysByCountryCode; + relaysByASNumber = newRelaysByASNumber; + relaysByFlag = newRelaysByFlag; + bridgesByFlag = newBridgesByFlag; + relaysByContact = newRelaysByContact; + relaysByFirstSeenDays = newRelaysByFirstSeenDays; + relaysByLastSeenDays = newRelaysByLastSeenDays; + bridgesByFirstSeenDays = newBridgesByFirstSeenDays; + bridgesByLastSeenDays = newBridgesByLastSeenDays; + relaysPublishedString = newRelaysPublishedString; + bridgesPublishedString = newBridgesPublishedString; + } + summaryFileLastModified = newSummaryFileLastModified; + successfullyReadSummaryFile = true; + } + + private static String formatRelaySummaryLine(NodeStatus entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String fingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + List<String> addresses = new ArrayList<String>(); + addresses.add(entry.getAddress()); + for (String orAddress : entry.getOrAddresses()) { + addresses.add(orAddress); + } + for (String exitAddress : entry.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + StringBuilder addressesBuilder = new StringBuilder(); + int written = 0; + for (String address : addresses) { + addressesBuilder.append((written++ > 0 ? "," : "") + "\"" + + address.toLowerCase() + "\""); + } + return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}", + (nickname == null ? "" : "\"n\":\"" + nickname + "\","), + fingerprint, addressesBuilder.toString(), running); + } + + private static String formatBridgeSummaryLine(NodeStatus entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String hashedFingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + return String.format("{%s\"h\":\"%s\",\"r\":%s}", + (nickname == null ? "" : "\"n\":\"" + nickname + "\","), + hashedFingerprint, running); + } + + public static long getLastModified() { + readSummaryFile(); + return summaryFileLastModified; + } + + private String resourceType; + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + private String type; + public void setType(String type) { + this.type = type; + } + + private String running; + public void setRunning(String running) { + this.running = running; + } + + private String[] search; + public void setSearch(String[] search) { + this.search = new String[search.length]; + System.arraycopy(search, 0, this.search, 0, search.length); + } + + private String lookup; + public void setLookup(String lookup) { + this.lookup = lookup; + } + + private String country; + public void setCountry(String country) { + this.country = country; + } + + private String as; + public void setAs(String as) { + this.as = as; + } + + private String flag; + public void setFlag(String flag) { + this.flag = flag; + } + + private String[] contact; + public void setContact(String[] contact) { + this.contact = new String[contact.length]; + System.arraycopy(contact, 0, this.contact, 0, contact.length); + } + + private String[] order; + public void setOrder(String[] order) { + this.order = new String[order.length]; + System.arraycopy(order, 0, this.order, 0, order.length); + } + + private String offset; + public void setOffset(String offset) { + this.offset = offset; + } + + private String limit; + public void setLimit(String limit) { + this.limit = limit; + } + + private int[] firstSeenDays; + public void setFirstSeenDays(int[] firstSeenDays) { + this.firstSeenDays = new int[firstSeenDays.length]; + System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, + firstSeenDays.length); + } + + private int[] lastSeenDays; + public void setLastSeenDays(int[] lastSeenDays) { + this.lastSeenDays = new int[lastSeenDays.length]; + System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, + lastSeenDays.length); + } + + private Map<String, String> filteredRelays = + new HashMap<String, String>(); + + private Map<String, String> filteredBridges = + new HashMap<String, String>(); + + public void handleRequest() { + this.filteredRelays.putAll(relayFingerprintSummaryLines); + this.filteredBridges.putAll(bridgeFingerprintSummaryLines); + this.filterByResourceType(); + this.filterByType(); + this.filterByRunning(); + this.filterBySearchTerms(); + this.filterByFingerprint(); + this.filterByCountryCode(); + this.filterByASNumber(); + this.filterByFlag(); + this.filterNodesByFirstSeenDays(); + this.filterNodesByLastSeenDays(); + this.filterByContact(); + this.order(); + this.offset(); + this.limit(); + } + + + private void filterByResourceType() { + if (this.resourceType.equals("clients")) { + this.filteredRelays.clear(); + } + if (this.resourceType.equals("weights")) { + this.filteredBridges.clear(); + } + } + + private void filterByType() { + if (this.type == null) { + return; + } else if (this.type.equals("relay")) { + this.filteredBridges.clear(); + } else { + this.filteredRelays.clear(); + } + } + + private void filterByRunning() { + if (this.running == null) { + return; + } + boolean runningRequested = this.running.equals("true"); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : filteredRelays.entrySet()) { + if (e.getValue().contains("\"r\":true") != runningRequested) { + removeRelays.add(e.getKey()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + Set<String> removeBridges = new HashSet<String>(); + for (Map.Entry<String, String> e : filteredBridges.entrySet()) { + if (e.getValue().contains("\"r\":true") != runningRequested) { + removeBridges.add(e.getKey()); + } + } + for (String fingerprint : removeBridges) { + this.filteredBridges.remove(fingerprint); + } + } + + private void filterBySearchTerms() { + if (this.search == null) { + return; + } + for (String searchTerm : this.search) { + filterBySearchTerm(searchTerm); + } + } + + private void filterBySearchTerm(String searchTerm) { + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + String line = e.getValue(); + boolean lineMatches = false; + String nickname = "unnamed"; + if (line.contains("\"n\":\"")) { + nickname = line.substring(line.indexOf("\"n\":\"") + 5). + split("\"")[0].toLowerCase(); + } + if (searchTerm.startsWith("$")) { + /* Search is for $-prefixed fingerprint. */ + if (fingerprint.startsWith( + searchTerm.substring(1).toUpperCase())) { + /* $-prefixed fingerprint matches. */ + lineMatches = true; + } + } else if (nickname.contains(searchTerm.toLowerCase())) { + /* Nickname matches. */ + lineMatches = true; + } else if (fingerprint.startsWith(searchTerm.toUpperCase())) { + /* Non-$-prefixed fingerprint matches. */ + lineMatches = true; + } else if (line.substring(line.indexOf("\"a\":[")).contains("\"" + + searchTerm.toLowerCase())) { + /* Address matches. */ + lineMatches = true; + } + if (!lineMatches) { + removeRelays.add(e.getKey()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + Set<String> removeBridges = new HashSet<String>(); + for (Map.Entry<String, String> e : filteredBridges.entrySet()) { + String hashedFingerprint = e.getKey(); + String line = e.getValue(); + boolean lineMatches = false; + String nickname = "unnamed"; + if (line.contains("\"n\":\"")) { + nickname = line.substring(line.indexOf("\"n\":\"") + 5). + split("\"")[0].toLowerCase(); + } + if (searchTerm.startsWith("$")) { + /* Search is for $-prefixed hashed fingerprint. */ + if (hashedFingerprint.startsWith( + searchTerm.substring(1).toUpperCase())) { + /* $-prefixed hashed fingerprint matches. */ + lineMatches = true; + } + } else if (nickname.contains(searchTerm.toLowerCase())) { + /* Nickname matches. */ + lineMatches = true; + } else if (hashedFingerprint.startsWith(searchTerm.toUpperCase())) { + /* Non-$-prefixed hashed fingerprint matches. */ + lineMatches = true; + } + if (!lineMatches) { + removeBridges.add(e.getKey()); + } + } + for (String fingerprint : removeBridges) { + this.filteredBridges.remove(fingerprint); + } + } + + private void filterByFingerprint() { + if (this.lookup == null) { + return; + } + String fingerprint = this.lookup; + String relayLine = this.filteredRelays.get(fingerprint); + this.filteredRelays.clear(); + if (relayLine != null) { + this.filteredRelays.put(fingerprint, relayLine); + } + String bridgeLine = this.filteredBridges.get(fingerprint); + this.filteredBridges.clear(); + if (bridgeLine != null) { + this.filteredBridges.put(fingerprint, bridgeLine); + } + } + + private void filterByCountryCode() { + if (this.country == null) { + return; + } + String countryCode = this.country.toLowerCase(); + if (!relaysByCountryCode.containsKey(countryCode)) { + this.filteredRelays.clear(); + } else { + Set<String> relaysWithCountryCode = + relaysByCountryCode.get(countryCode); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + if (!relaysWithCountryCode.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + } + this.filteredBridges.clear(); + } + + private void filterByASNumber() { + if (this.as == null) { + return; + } + String aSNumber = this.as.toUpperCase(); + if (!aSNumber.startsWith("AS")) { + aSNumber = "AS" + aSNumber; + } + if (!relaysByASNumber.containsKey(aSNumber)) { + this.filteredRelays.clear(); + } else { + Set<String> relaysWithASNumber = + relaysByASNumber.get(aSNumber); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + if (!relaysWithASNumber.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + } + this.filteredBridges.clear(); + } + + private void filterByFlag() { + if (this.flag == null) { + return; + } + String flag = this.flag.toLowerCase(); + if (!relaysByFlag.containsKey(flag)) { + this.filteredRelays.clear(); + } else { + Set<String> relaysWithFlag = relaysByFlag.get(flag); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + if (!relaysWithFlag.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + } + if (!bridgesByFlag.containsKey(flag)) { + this.filteredBridges.clear(); + } else { + Set<String> bridgesWithFlag = bridgesByFlag.get(flag); + Set<String> removeBridges = new HashSet<String>(); + for (Map.Entry<String, String> e : + this.filteredBridges.entrySet()) { + String fingerprint = e.getKey(); + if (!bridgesWithFlag.contains(fingerprint)) { + removeBridges.add(fingerprint); + } + } + for (String fingerprint : removeBridges) { + this.filteredBridges.remove(fingerprint); + } + } + } + + private void filterNodesByFirstSeenDays() { + if (this.firstSeenDays == null) { + return; + } + filterNodesByDays(this.filteredRelays, relaysByFirstSeenDays, + this.firstSeenDays); + filterNodesByDays(this.filteredBridges, bridgesByFirstSeenDays, + this.firstSeenDays); + } + + private void filterNodesByLastSeenDays() { + if (this.lastSeenDays == null) { + return; + } + filterNodesByDays(this.filteredRelays, relaysByLastSeenDays, + this.lastSeenDays); + filterNodesByDays(this.filteredBridges, bridgesByLastSeenDays, + this.lastSeenDays); + } + + private void filterNodesByDays(Map<String, String> filteredNodes, + SortedMap<Integer, Set<String>> nodesByDays, int[] days) { + Set<String> removeNodes = new HashSet<String>(); + for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { + removeNodes.addAll(nodes); + } + if (days[1] < Integer.MAX_VALUE) { + for (Set<String> nodes : + nodesByDays.tailMap(days[1] + 1).values()) { + removeNodes.addAll(nodes); + } + } + for (String fingerprint : removeNodes) { + filteredNodes.remove(fingerprint); + } + } + + private void filterByContact() { + if (this.contact == null) { + return; + } + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, Set<String>> e : relaysByContact.entrySet()) { + String contact = e.getKey(); + for (String contactPart : this.contact) { + if (contact == null || + !contact.contains(contactPart.toLowerCase())) { + removeRelays.addAll(e.getValue()); + break; + } + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + + private void order() { + if (this.order != null && this.order.length == 1) { + List<String> orderBy = new ArrayList<String>( + relaysByConsensusWeight); + if (this.order[0].startsWith("-")) { + Collections.reverse(orderBy); + } + for (String relay : orderBy) { + if (this.filteredRelays.containsKey(relay) && + !this.orderedRelays.contains(filteredRelays.get(relay))) { + this.orderedRelays.add(this.filteredRelays.remove(relay)); + } + } + for (String relay : this.filteredRelays.keySet()) { + if (!this.orderedRelays.contains(this.filteredRelays.get(relay))) { + this.orderedRelays.add(this.filteredRelays.remove(relay)); + } + } + Set<String> uniqueBridges = new HashSet<String>( + this.filteredBridges.values()); + this.orderedBridges.addAll(uniqueBridges); + } else { + Set<String> uniqueRelays = new HashSet<String>( + this.filteredRelays.values()); + this.orderedRelays.addAll(uniqueRelays); + Set<String> uniqueBridges = new HashSet<String>( + this.filteredBridges.values()); + this.orderedBridges.addAll(uniqueBridges); + } + } + + private void offset() { + if (this.offset == null) { + return; + } + int offsetValue = Integer.parseInt(this.offset); + while (offsetValue-- > 0 && + (!this.orderedRelays.isEmpty() || + !this.orderedBridges.isEmpty())) { + if (!this.orderedRelays.isEmpty()) { + this.orderedRelays.remove(0); + } else { + this.orderedBridges.remove(0); + } + } + } + + private void limit() { + if (this.limit == null) { + return; + } + int limitValue = Integer.parseInt(this.limit); + while (!this.orderedRelays.isEmpty() && + limitValue < this.orderedRelays.size()) { + this.orderedRelays.remove(this.orderedRelays.size() - 1); + } + limitValue -= this.orderedRelays.size(); + while (!this.orderedBridges.isEmpty() && + limitValue < this.orderedBridges.size()) { + this.orderedBridges.remove(this.orderedBridges.size() - 1); + } + } + + private List<String> orderedRelays = new ArrayList<String>(); + public List<String> getOrderedRelays() { + return this.orderedRelays; + } + + private List<String> orderedBridges = new ArrayList<String>(); + public List<String> getOrderedBridges() { + return this.orderedBridges; + } + + public String getRelaysPublishedString() { + return relaysPublishedString; + } + + public String getBridgesPublishedString() { + return bridgesPublishedString; + } +} diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 27ff414..0c01d9f 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -42,7 +42,8 @@ public class ResourceServlet extends HttpServlet { DocumentStore documentStore, Time time) { this.maintenanceMode = maintenanceMode; if (!maintenanceMode) { - ResponseBuilder.initialize(documentStore, time); + RequestHandler.initialize(documentStore, time); + ResponseBuilder.initialize(documentStore); } } @@ -50,7 +51,7 @@ public class ResourceServlet extends HttpServlet { if (this.maintenanceMode) { return super.getLastModified(request); } else { - return ResponseBuilder.getLastModified(); + return RequestHandler.getLastModified(); } } @@ -109,12 +110,11 @@ public class ResourceServlet extends HttpServlet { return; } - if (!ResponseBuilder.update()) { + if (!RequestHandler.update()) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; } - ResponseBuilder rb = new ResponseBuilder(); String uri = request.getRequestURI(); if (uri.startsWith("/onionoo/")) { uri = uri.substring("/onionoo".length()); @@ -136,7 +136,8 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setResourceType(resourceType); + RequestHandler rh = new RequestHandler(); + rh.setResourceType(resourceType); /* Extract parameters either from the old-style URI or from request * parameters. */ @@ -170,7 +171,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setType(relaysRequested ? "relay" : "bridge"); + rh.setType(relaysRequested ? "relay" : "bridge"); } if (parameterMap.containsKey("running")) { String runningParameterValue = @@ -182,7 +183,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setRunning(runningRequested ? "true" : "false"); + rh.setRunning(runningRequested ? "true" : "false"); } if (parameterMap.containsKey("search")) { String[] searchTerms = this.parseSearchParameters( @@ -191,7 +192,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setSearch(searchTerms); + rh.setSearch(searchTerms); } if (parameterMap.containsKey("lookup")) { String fingerprintParameter = this.parseFingerprintParameter( @@ -201,7 +202,7 @@ public class ResourceServlet extends HttpServlet { return; } String fingerprint = fingerprintParameter.toUpperCase(); - rb.setLookup(fingerprint); + rh.setLookup(fingerprint); } if (parameterMap.containsKey("country")) { String countryCodeParameter = this.parseCountryCodeParameter( @@ -210,7 +211,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setCountry(countryCodeParameter); + rh.setCountry(countryCodeParameter); } if (parameterMap.containsKey("as")) { String aSNumberParameter = this.parseASNumberParameter( @@ -219,7 +220,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setAs(aSNumberParameter); + rh.setAs(aSNumberParameter); } if (parameterMap.containsKey("flag")) { String flagParameter = this.parseFlagParameter( @@ -228,7 +229,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setFlag(flagParameter); + rh.setFlag(flagParameter); } if (parameterMap.containsKey("first_seen_days")) { int[] days = this.parseDaysParameter( @@ -237,7 +238,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setFirstSeenDays(days); + rh.setFirstSeenDays(days); } if (parameterMap.containsKey("last_seen_days")) { int[] days = this.parseDaysParameter( @@ -246,7 +247,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setLastSeenDays(days); + rh.setLastSeenDays(days); } if (parameterMap.containsKey("contact")) { String[] contactParts = this.parseContactParameter( @@ -255,10 +256,8 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setContact(contactParts); + rh.setContact(contactParts); } - - /* Re-order and limit results. */ if (parameterMap.containsKey("order")) { String orderParameter = parameterMap.get("order").toLowerCase(); String orderByField = orderParameter; @@ -269,7 +268,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setOrder(new String[] { orderParameter }); + rh.setOrder(new String[] { orderParameter }); } if (parameterMap.containsKey("offset")) { String offsetParameter = parameterMap.get("offset"); @@ -283,7 +282,7 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setOffset(offsetParameter); + rh.setOffset(offsetParameter); } if (parameterMap.containsKey("limit")) { String limitParameter = parameterMap.get("limit"); @@ -297,11 +296,16 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rb.setLimit(limitParameter); + rh.setLimit(limitParameter); } + rh.handleRequest(); - /* Possibly include only a subset of fields in the response - * document. */ + ResponseBuilder rb = new ResponseBuilder(); + rb.setResourceType(resourceType); + rb.setRelaysPublishedString(rh.getRelaysPublishedString()); + rb.setBridgesPublishedString(rh.getBridgesPublishedString()); + rb.setOrderedRelays(rh.getOrderedRelays()); + rb.setOrderedBridges(rh.getOrderedBridges()); String[] fields = null; if (parameterMap.containsKey("fields")) { fields = this.parseFieldsParameter(parameterMap.get("fields")); @@ -312,7 +316,6 @@ public class ResourceServlet extends HttpServlet { rb.setFields(fields); } - /* Set response headers and write the response. */ response.setHeader("Access-Control-Allow-Origin", "*"); response.setContentType("application/json"); response.setCharacterEncoding("utf-8"); diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index 080af52..69b5c62 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -4,743 +4,53 @@ package org.torproject.onionoo; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Scanner; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; public class ResponseBuilder { - private static long summaryFileLastModified = -1L; private static DocumentStore documentStore; - private static Time time; - private static boolean successfullyReadSummaryFile = false; - private static String relaysPublishedString, bridgesPublishedString; - private static List<String> relaysByConsensusWeight = null; - private static Map<String, String> relayFingerprintSummaryLines = null, - bridgeFingerprintSummaryLines = null; - private static Map<String, Set<String>> relaysByCountryCode = null, - relaysByASNumber = null, relaysByFlag = null, bridgesByFlag = null, - relaysByContact = null; - private static SortedMap<Integer, Set<String>> - relaysByFirstSeenDays = null, bridgesByFirstSeenDays = null, - relaysByLastSeenDays = null, bridgesByLastSeenDays = null; - private static final long SUMMARY_MAX_AGE = DateTimeHelper.SIX_HOURS; - public static void initialize(DocumentStore documentStoreParam, - Time timeParam) { + public static void initialize(DocumentStore documentStoreParam) { documentStore = documentStoreParam; - time = timeParam; - readSummaryFile(); } - public static boolean update() { - readSummaryFile(); - return successfullyReadSummaryFile; + private String resourceType; + public void setResourceType(String resourceType) { + this.resourceType = resourceType; } - private static void readSummaryFile() { - long newSummaryFileLastModified = -1L; - UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, - false); - if (updateStatus != null && - updateStatus.getDocumentString() != null) { - String updateString = updateStatus.getDocumentString(); - try { - newSummaryFileLastModified = Long.parseLong(updateString.trim()); - } catch (NumberFormatException e) { - /* Handle below. */ - } - } - if (newSummaryFileLastModified < 0L) { - // TODO Does this actually solve anything? Should we instead - // switch to a variant of the maintenance mode and re-check when - // the next requests comes in that happens x seconds after this one? - successfullyReadSummaryFile = false; - return; - } - if (newSummaryFileLastModified + SUMMARY_MAX_AGE - < time.currentTimeMillis()) { - // TODO Does this actually solve anything? Should we instead - // switch to a variant of the maintenance mode and re-check when - // the next requests comes in that happens x seconds after this one? - successfullyReadSummaryFile = false; - return; - } - if (newSummaryFileLastModified > summaryFileLastModified) { - List<String> newRelaysByConsensusWeight = new ArrayList<String>(); - Map<String, String> - newRelayFingerprintSummaryLines = new HashMap<String, String>(), - newBridgeFingerprintSummaryLines = - new HashMap<String, String>(); - Map<String, Set<String>> - newRelaysByCountryCode = new HashMap<String, Set<String>>(), - newRelaysByASNumber = new HashMap<String, Set<String>>(), - newRelaysByFlag = new HashMap<String, Set<String>>(), - newBridgesByFlag = new HashMap<String, Set<String>>(), - newRelaysByContact = new HashMap<String, Set<String>>(); - SortedMap<Integer, Set<String>> - newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), - newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), - newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), - newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - long relaysLastValidAfterMillis = -1L, - bridgesLastPublishedMillis = -1L; - String newRelaysPublishedString, newBridgesPublishedString; - Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), - currentBridges = new HashSet<NodeStatus>(); - SortedSet<String> fingerprints = documentStore.list( - NodeStatus.class, false); - // TODO We should be able to learn if something goes wrong when - // reading the summary file, rather than silently having an empty - // list of fingerprints. - for (String fingerprint : fingerprints) { - NodeStatus node = documentStore.retrieve(NodeStatus.class, true, - fingerprint); - if (node.isRelay()) { - relaysLastValidAfterMillis = Math.max( - relaysLastValidAfterMillis, node.getLastSeenMillis()); - currentRelays.add(node); - } else { - bridgesLastPublishedMillis = Math.max( - bridgesLastPublishedMillis, node.getLastSeenMillis()); - currentBridges.add(node); - } - } - newRelaysPublishedString = DateTimeHelper.format( - relaysLastValidAfterMillis); - newBridgesPublishedString = DateTimeHelper.format( - bridgesLastPublishedMillis); - List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - for (NodeStatus entry : currentRelays) { - String fingerprint = entry.getFingerprint().toUpperCase(); - String hashedFingerprint = entry.getHashedFingerprint(). - toUpperCase(); - entry.setRunning(entry.getLastSeenMillis() == - relaysLastValidAfterMillis); - String line = formatRelaySummaryLine(entry); - newRelayFingerprintSummaryLines.put(fingerprint, line); - newRelayFingerprintSummaryLines.put(hashedFingerprint, line); - long consensusWeight = entry.getConsensusWeight(); - orderRelaysByConsensusWeight.add(String.format("%020d %s", - consensusWeight, fingerprint)); - orderRelaysByConsensusWeight.add(String.format("%020d %s", - consensusWeight, hashedFingerprint)); - if (entry.getCountryCode() != null) { - String countryCode = entry.getCountryCode(); - if (!newRelaysByCountryCode.containsKey(countryCode)) { - newRelaysByCountryCode.put(countryCode, - new HashSet<String>()); - } - newRelaysByCountryCode.get(countryCode).add(fingerprint); - newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); - } - if (entry.getASNumber() != null) { - String aSNumber = entry.getASNumber(); - if (!newRelaysByASNumber.containsKey(aSNumber)) { - newRelaysByASNumber.put(aSNumber, new HashSet<String>()); - } - newRelaysByASNumber.get(aSNumber).add(fingerprint); - newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); - } - for (String flag : entry.getRelayFlags()) { - String flagLowerCase = flag.toLowerCase(); - if (!newRelaysByFlag.containsKey(flagLowerCase)) { - newRelaysByFlag.put(flagLowerCase, new HashSet<String>()); - } - newRelaysByFlag.get(flagLowerCase).add(fingerprint); - newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); - } - int daysSinceFirstSeen = (int) ((newSummaryFileLastModified - - entry.getFirstSeenMillis()) / 86400000L); - if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { - newRelaysByFirstSeenDays.put(daysSinceFirstSeen, - new HashSet<String>()); - } - newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint); - newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedFingerprint); - int daysSinceLastSeen = (int) ((newSummaryFileLastModified - - entry.getLastSeenMillis()) / 86400000L); - if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) { - newRelaysByLastSeenDays.put(daysSinceLastSeen, - new HashSet<String>()); - } - newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); - newRelaysByLastSeenDays.get(daysSinceLastSeen).add( - hashedFingerprint); - String contact = entry.getContact(); - if (!newRelaysByContact.containsKey(contact)) { - newRelaysByContact.put(contact, new HashSet<String>()); - } - newRelaysByContact.get(contact).add(fingerprint); - newRelaysByContact.get(contact).add(hashedFingerprint); - } - Collections.sort(orderRelaysByConsensusWeight); - newRelaysByConsensusWeight = new ArrayList<String>(); - for (String relay : orderRelaysByConsensusWeight) { - newRelaysByConsensusWeight.add(relay.split(" ")[1]); - } - for (NodeStatus entry : currentBridges) { - String hashedFingerprint = entry.getFingerprint().toUpperCase(); - String hashedHashedFingerprint = entry.getHashedFingerprint(). - toUpperCase(); - entry.setRunning(entry.getRelayFlags().contains("Running") && - entry.getLastSeenMillis() == bridgesLastPublishedMillis); - String line = formatBridgeSummaryLine(entry); - newBridgeFingerprintSummaryLines.put(hashedFingerprint, line); - newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, - line); - for (String flag : entry.getRelayFlags()) { - String flagLowerCase = flag.toLowerCase(); - if (!newBridgesByFlag.containsKey(flagLowerCase)) { - newBridgesByFlag.put(flagLowerCase, new HashSet<String>()); - } - newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint); - newBridgesByFlag.get(flagLowerCase).add( - hashedHashedFingerprint); - } - int daysSinceFirstSeen = (int) ((newSummaryFileLastModified - - entry.getFirstSeenMillis()) / 86400000L); - if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) { - newBridgesByFirstSeenDays.put(daysSinceFirstSeen, - new HashSet<String>()); - } - newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedFingerprint); - newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedHashedFingerprint); - int daysSinceLastSeen = (int) ((newSummaryFileLastModified - - entry.getLastSeenMillis()) / 86400000L); - if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) { - newBridgesByLastSeenDays.put(daysSinceLastSeen, - new HashSet<String>()); - } - newBridgesByLastSeenDays.get(daysSinceLastSeen).add( - hashedFingerprint); - newBridgesByLastSeenDays.get(daysSinceLastSeen).add( - hashedHashedFingerprint); - } - relaysByConsensusWeight = newRelaysByConsensusWeight; - relayFingerprintSummaryLines = newRelayFingerprintSummaryLines; - bridgeFingerprintSummaryLines = newBridgeFingerprintSummaryLines; - relaysByCountryCode = newRelaysByCountryCode; - relaysByASNumber = newRelaysByASNumber; - relaysByFlag = newRelaysByFlag; - bridgesByFlag = newBridgesByFlag; - relaysByContact = newRelaysByContact; - relaysByFirstSeenDays = newRelaysByFirstSeenDays; - relaysByLastSeenDays = newRelaysByLastSeenDays; - bridgesByFirstSeenDays = newBridgesByFirstSeenDays; - bridgesByLastSeenDays = newBridgesByLastSeenDays; - relaysPublishedString = newRelaysPublishedString; - bridgesPublishedString = newBridgesPublishedString; - } - summaryFileLastModified = newSummaryFileLastModified; - successfullyReadSummaryFile = true; + private String relaysPublishedString; + public void setRelaysPublishedString(String relaysPublishedString) { + this.relaysPublishedString = relaysPublishedString; } - private static String formatRelaySummaryLine(NodeStatus entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String fingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress); - } - for (String exitAddress : entry.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } - StringBuilder addressesBuilder = new StringBuilder(); - int written = 0; - for (String address : addresses) { - addressesBuilder.append((written++ > 0 ? "," : "") + "\"" - + address.toLowerCase() + "\""); - } - return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}", - (nickname == null ? "" : "\"n\":\"" + nickname + "\","), - fingerprint, addressesBuilder.toString(), running); + private String bridgesPublishedString; + public void setBridgesPublishedString(String bridgesPublishedString) { + this.bridgesPublishedString = bridgesPublishedString; } - private static String formatBridgeSummaryLine(NodeStatus entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String hashedFingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - return String.format("{%s\"h\":\"%s\",\"r\":%s}", - (nickname == null ? "" : "\"n\":\"" + nickname + "\","), - hashedFingerprint, running); + private List<String> orderedRelays = new ArrayList<String>(); + public void setOrderedRelays(List<String> orderedRelays) { + this.orderedRelays = orderedRelays; } - public static long getLastModified() { - readSummaryFile(); - return summaryFileLastModified; + private List<String> orderedBridges = new ArrayList<String>(); + public void setOrderedBridges(List<String> orderedBridges) { + this.orderedBridges = orderedBridges; } - private String resourceType, type, running, search[], lookup, country, - as, flag, contact[], fields[], order[], offset, limit; - private int[] firstSeenDays, lastSeenDays; - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - public void setType(String type) { - this.type = type; - } - public void setRunning(String running) { - this.running = running; - } - public void setSearch(String[] search) { - this.search = new String[search.length]; - System.arraycopy(search, 0, this.search, 0, search.length); - } - public void setLookup(String lookup) { - this.lookup = lookup; - } - public void setCountry(String country) { - this.country = country; - } - public void setAs(String as) { - this.as = as; - } - public void setFlag(String flag) { - this.flag = flag; - } - public void setFirstSeenDays(int[] firstSeenDays) { - this.firstSeenDays = new int[firstSeenDays.length]; - System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, - firstSeenDays.length); - } - public void setLastSeenDays(int[] lastSeenDays) { - this.lastSeenDays = new int[lastSeenDays.length]; - System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, - lastSeenDays.length); - } - public void setContact(String[] contact) { - this.contact = new String[contact.length]; - System.arraycopy(contact, 0, this.contact, 0, contact.length); - } + private String[] fields; public void setFields(String[] fields) { this.fields = new String[fields.length]; System.arraycopy(fields, 0, this.fields, 0, fields.length); } - public void setOrder(String[] order) { - this.order = new String[order.length]; - System.arraycopy(order, 0, this.order, 0, order.length); - } - public void setOffset(String offset) { - this.offset = offset; - } - public void setLimit(String limit) { - this.limit = limit; - } public void buildResponse(PrintWriter pw) { - - /* Filter relays and bridges matching the request. */ - Map<String, String> filteredRelays = new HashMap<String, String>( - relayFingerprintSummaryLines); - Map<String, String> filteredBridges = new HashMap<String, String>( - bridgeFingerprintSummaryLines); - filterByResourceType(filteredRelays, filteredBridges); - filterByType(filteredRelays, filteredBridges); - filterByRunning(filteredRelays, filteredBridges); - filterBySearchTerms(filteredRelays, filteredBridges); - filterByFingerprint(filteredRelays, filteredBridges); - filterByCountryCode(filteredRelays, filteredBridges); - filterByASNumber(filteredRelays, filteredBridges); - filterByFlag(filteredRelays, filteredBridges); - filterNodesByFirstSeenDays(filteredRelays, filteredBridges); - filterNodesByLastSeenDays(filteredRelays, filteredBridges); - filterByContact(filteredRelays, filteredBridges); - - /* Re-order and limit results. */ - List<String> orderedRelays = new ArrayList<String>(); - List<String> orderedBridges = new ArrayList<String>(); - order(filteredRelays, filteredBridges, orderedRelays, orderedBridges); - offset(orderedRelays, orderedBridges); - limit(orderedRelays, orderedBridges); - - /* Write the response. */ writeRelays(orderedRelays, pw); writeBridges(orderedBridges, pw); } - private void filterByResourceType(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.resourceType.equals("clients")) { - filteredRelays.clear(); - } - if (this.resourceType.equals("weights")) { - filteredBridges.clear(); - } - } - - private void filterByType(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.type == null) { - return; - } else if (this.type.equals("relay")) { - filteredBridges.clear(); - } else { - filteredRelays.clear(); - } - } - - private void filterByRunning(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.running == null) { - return; - } - boolean runningRequested = this.running.equals("true"); - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { - if (e.getValue().contains("\"r\":true") != runningRequested) { - removeRelays.add(e.getKey()); - } - } - for (String fingerprint : removeRelays) { - filteredRelays.remove(fingerprint); - } - Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredBridges.entrySet()) { - if (e.getValue().contains("\"r\":true") != runningRequested) { - removeBridges.add(e.getKey()); - } - } - for (String fingerprint : removeBridges) { - filteredBridges.remove(fingerprint); - } - } - - private void filterBySearchTerms(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.search == null) { - return; - } - for (String searchTerm : this.search) { - filterBySearchTerm(filteredRelays, filteredBridges, searchTerm); - } - } - - private void filterBySearchTerm(Map<String, String> filteredRelays, - Map<String, String> filteredBridges, String searchTerm) { - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { - String fingerprint = e.getKey(); - String line = e.getValue(); - boolean lineMatches = false; - String nickname = "unnamed"; - if (line.contains("\"n\":\"")) { - nickname = line.substring(line.indexOf("\"n\":\"") + 5). - split("\"")[0].toLowerCase(); - } - if (searchTerm.startsWith("$")) { - /* Search is for $-prefixed fingerprint. */ - if (fingerprint.startsWith( - searchTerm.substring(1).toUpperCase())) { - /* $-prefixed fingerprint matches. */ - lineMatches = true; - } - } else if (nickname.contains(searchTerm.toLowerCase())) { - /* Nickname matches. */ - lineMatches = true; - } else if (fingerprint.startsWith(searchTerm.toUpperCase())) { - /* Non-$-prefixed fingerprint matches. */ - lineMatches = true; - } else if (line.substring(line.indexOf("\"a\":[")).contains("\"" - + searchTerm.toLowerCase())) { - /* Address matches. */ - lineMatches = true; - } - if (!lineMatches) { - removeRelays.add(e.getKey()); - } - } - for (String fingerprint : removeRelays) { - filteredRelays.remove(fingerprint); - } - Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredBridges.entrySet()) { - String hashedFingerprint = e.getKey(); - String line = e.getValue(); - boolean lineMatches = false; - String nickname = "unnamed"; - if (line.contains("\"n\":\"")) { - nickname = line.substring(line.indexOf("\"n\":\"") + 5). - split("\"")[0].toLowerCase(); - } - if (searchTerm.startsWith("$")) { - /* Search is for $-prefixed hashed fingerprint. */ - if (hashedFingerprint.startsWith( - searchTerm.substring(1).toUpperCase())) { - /* $-prefixed hashed fingerprint matches. */ - lineMatches = true; - } - } else if (nickname.contains(searchTerm.toLowerCase())) { - /* Nickname matches. */ - lineMatches = true; - } else if (hashedFingerprint.startsWith(searchTerm.toUpperCase())) { - /* Non-$-prefixed hashed fingerprint matches. */ - lineMatches = true; - } - if (!lineMatches) { - removeBridges.add(e.getKey()); - } - } - for (String fingerprint : removeBridges) { - filteredBridges.remove(fingerprint); - } - } - - private void filterByFingerprint(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.lookup == null) { - return; - } - String fingerprint = this.lookup; - String relayLine = filteredRelays.get(fingerprint); - filteredRelays.clear(); - if (relayLine != null) { - filteredRelays.put(fingerprint, relayLine); - } - String bridgeLine = filteredBridges.get(fingerprint); - filteredBridges.clear(); - if (bridgeLine != null) { - filteredBridges.put(fingerprint, bridgeLine); - } - } - - private void filterByCountryCode(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.country == null) { - return; - } - String countryCode = this.country.toLowerCase(); - if (!relaysByCountryCode.containsKey(countryCode)) { - filteredRelays.clear(); - } else { - Set<String> relaysWithCountryCode = - relaysByCountryCode.get(countryCode); - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { - String fingerprint = e.getKey(); - if (!relaysWithCountryCode.contains(fingerprint)) { - removeRelays.add(fingerprint); - } - } - for (String fingerprint : removeRelays) { - filteredRelays.remove(fingerprint); - } - } - filteredBridges.clear(); - } - - private void filterByASNumber(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.as == null) { - return; - } - String aSNumber = this.as.toUpperCase(); - if (!aSNumber.startsWith("AS")) { - aSNumber = "AS" + aSNumber; - } - if (!relaysByASNumber.containsKey(aSNumber)) { - filteredRelays.clear(); - } else { - Set<String> relaysWithASNumber = - relaysByASNumber.get(aSNumber); - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { - String fingerprint = e.getKey(); - if (!relaysWithASNumber.contains(fingerprint)) { - removeRelays.add(fingerprint); - } - } - for (String fingerprint : removeRelays) { - filteredRelays.remove(fingerprint); - } - } - filteredBridges.clear(); - } - - private void filterByFlag(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.flag == null) { - return; - } - String flag = this.flag.toLowerCase(); - if (!relaysByFlag.containsKey(flag)) { - filteredRelays.clear(); - } else { - Set<String> relaysWithFlag = relaysByFlag.get(flag); - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredRelays.entrySet()) { - String fingerprint = e.getKey(); - if (!relaysWithFlag.contains(fingerprint)) { - removeRelays.add(fingerprint); - } - } - for (String fingerprint : removeRelays) { - filteredRelays.remove(fingerprint); - } - } - if (!bridgesByFlag.containsKey(flag)) { - filteredBridges.clear(); - } else { - Set<String> bridgesWithFlag = bridgesByFlag.get(flag); - Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, String> e : filteredBridges.entrySet()) { - String fingerprint = e.getKey(); - if (!bridgesWithFlag.contains(fingerprint)) { - removeBridges.add(fingerprint); - } - } - for (String fingerprint : removeBridges) { - filteredBridges.remove(fingerprint); - } - } - } - - private void filterNodesByFirstSeenDays( - Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.firstSeenDays == null) { - return; - } - filterNodesByDays(filteredRelays, relaysByFirstSeenDays, - this.firstSeenDays); - filterNodesByDays(filteredBridges, bridgesByFirstSeenDays, - this.firstSeenDays); - } - - private void filterNodesByLastSeenDays( - Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.lastSeenDays == null) { - return; - } - filterNodesByDays(filteredRelays, relaysByLastSeenDays, - this.lastSeenDays); - filterNodesByDays(filteredBridges, bridgesByLastSeenDays, - this.lastSeenDays); - } - - private void filterNodesByDays(Map<String, String> filteredNodes, - SortedMap<Integer, Set<String>> nodesByDays, int[] days) { - Set<String> removeNodes = new HashSet<String>(); - for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { - removeNodes.addAll(nodes); - } - if (days[1] < Integer.MAX_VALUE) { - for (Set<String> nodes : - nodesByDays.tailMap(days[1] + 1).values()) { - removeNodes.addAll(nodes); - } - } - for (String fingerprint : removeNodes) { - filteredNodes.remove(fingerprint); - } - } - - private void filterByContact(Map<String, String> filteredRelays, - Map<String, String> filteredBridges) { - if (this.contact == null) { - return; - } - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, Set<String>> e : relaysByContact.entrySet()) { - String contact = e.getKey(); - for (String contactPart : this.contact) { - if (contact == null || - !contact.contains(contactPart.toLowerCase())) { - removeRelays.addAll(e.getValue()); - break; - } - } - } - for (String fingerprint : removeRelays) { - filteredRelays.remove(fingerprint); - } - filteredBridges.clear(); - } - - private void order(Map<String, String> filteredRelays, - Map<String, String> filteredBridges, List<String> orderedRelays, - List<String> orderedBridges) { - if (this.order != null && this.order.length == 1) { - List<String> orderBy = new ArrayList<String>( - relaysByConsensusWeight); - if (this.order[0].startsWith("-")) { - Collections.reverse(orderBy); - } - for (String relay : orderBy) { - if (filteredRelays.containsKey(relay) && - !orderedRelays.contains(filteredRelays.get(relay))) { - orderedRelays.add(filteredRelays.remove(relay)); - } - } - for (String relay : filteredRelays.keySet()) { - if (!orderedRelays.contains(filteredRelays.get(relay))) { - orderedRelays.add(filteredRelays.remove(relay)); - } - } - Set<String> uniqueBridges = new HashSet<String>( - filteredBridges.values()); - orderedBridges.addAll(uniqueBridges); - } else { - Set<String> uniqueRelays = new HashSet<String>( - filteredRelays.values()); - orderedRelays.addAll(uniqueRelays); - Set<String> uniqueBridges = new HashSet<String>( - filteredBridges.values()); - orderedBridges.addAll(uniqueBridges); - } - } - - private void offset(List<String> orderedRelays, - List<String> orderedBridges) { - if (offset == null) { - return; - } - int offsetValue = Integer.parseInt(offset); - while (offsetValue-- > 0 && - (!orderedRelays.isEmpty() || !orderedBridges.isEmpty())) { - if (!orderedRelays.isEmpty()) { - orderedRelays.remove(0); - } else { - orderedBridges.remove(0); - } - } - } - - private void limit(List<String> orderedRelays, - List<String> orderedBridges) { - if (limit == null) { - return; - } - int limitValue = Integer.parseInt(limit); - while (!orderedRelays.isEmpty() && - limitValue < orderedRelays.size()) { - orderedRelays.remove(orderedRelays.size() - 1); - } - limitValue -= orderedRelays.size(); - while (!orderedBridges.isEmpty() && - limitValue < orderedBridges.size()) { - orderedBridges.remove(orderedBridges.size() - 1); - } - } - private void writeRelays(List<String> relays, PrintWriter pw) { pw.write("{\"relays_published\":\"" + relaysPublishedString + "\",\n\"relays\":[");