commit bb1d0569bb8dd618b44281f7e90e3f7d9e783e00 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Apr 19 10:21:20 2014 +0200

Create node index in background thread. --- etc/web.xml.template | 14 +- src/org/torproject/onionoo/ApplicationFactory.java | 11 + src/org/torproject/onionoo/NodeIndexer.java | 438 ++++++++++++++++++++ src/org/torproject/onionoo/RequestHandler.java | 321 ++------------ src/org/torproject/onionoo/ResourceServlet.java | 38 +- src/org/torproject/onionoo/ResponseBuilder.java | 16 +- .../torproject/onionoo/ResourceServletTest.java | 23 +- 7 files changed, 526 insertions(+), 335 deletions(-)

diff --git a/etc/web.xml.template b/etc/web.xml.template index f69314f..988d47a 100644 --- a/etc/web.xml.template +++ b/etc/web.xml.template @@ -12,10 +12,6 @@ org.torproject.onionoo.ResourceServlet </servlet-class> <init-param> - <param-name>outDir</param-name> - <param-value>/srv/onionoo/out/</param-value> - </init-param> - <init-param> <param-name>maintenance</param-name> <param-value>0</param-value> </init-param> @@ -45,5 +41,15 @@ <url-pattern>/uptime</url-pattern> </servlet-mapping>

+ <context-param> + <param-name>outDir</param-name> + <param-value>/srv/onionoo.torproject.org/onionoo/out/</param-value> + </context-param> + + <listener> + <listener-class> + org.torproject.onionoo.NodeIndexer + </listener-class> + </listener> </web-app>

diff --git a/src/org/torproject/onionoo/ApplicationFactory.java b/src/org/torproject/onionoo/ApplicationFactory.java index 98952df..44f2c17 100644 --- a/src/org/torproject/onionoo/ApplicationFactory.java +++ b/src/org/torproject/onionoo/ApplicationFactory.java @@ -37,4 +37,15 @@ public class ApplicationFactory { } return documentStoreInstance; } + + private static NodeIndexer nodeIndexerInstance; + public static void setNodeIndexer(NodeIndexer nodeIndexer) { + nodeIndexerInstance = nodeIndexer; + } + public static NodeIndexer getNodeIndexer() { + if (nodeIndexerInstance == null) { + nodeIndexerInstance = new NodeIndexer(); + } + return nodeIndexerInstance; + } } diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java new file mode 100644 index 0000000..87ecf9f --- /dev/null +++ b/src/org/torproject/onionoo/NodeIndexer.java @@ -0,0 +1,438 @@ +package org.torproject.onionoo; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; + +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +class NodeIndex { + + private String relaysPublishedString; + public void setRelaysPublishedString(String relaysPublishedString) { + this.relaysPublishedString = relaysPublishedString; + } + public String getRelaysPublishedString() { + return relaysPublishedString; + } + + private String bridgesPublishedString; + public void setBridgesPublishedString(String bridgesPublishedString) { + this.bridgesPublishedString = bridgesPublishedString; + } + public String getBridgesPublishedString() { + return bridgesPublishedString; + } + + private List<String> relaysByConsensusWeight; + public void setRelaysByConsensusWeight( + List<String> relaysByConsensusWeight) { + this.relaysByConsensusWeight = relaysByConsensusWeight; + } + public List<String> getRelaysByConsensusWeight() { + return relaysByConsensusWeight; + } + + private Map<String, String> relayFingerprintSummaryLines; + public void setRelayFingerprintSummaryLines( + Map<String, String> relayFingerprintSummaryLines) { + this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; + } + public Map<String, String> getRelayFingerprintSummaryLines() { + return this.relayFingerprintSummaryLines; + } + + private Map<String, String> bridgeFingerprintSummaryLines; + public void setBridgeFingerprintSummaryLines( + Map<String, String> bridgeFingerprintSummaryLines) { + this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; + } + public Map<String, String> getBridgeFingerprintSummaryLines() { + return this.bridgeFingerprintSummaryLines; + } + + private Map<String, Set<String>> relaysByCountryCode = null; + public void setRelaysByCountryCode( + Map<String, Set<String>> relaysByCountryCode) { + this.relaysByCountryCode = relaysByCountryCode; + } + public Map<String, Set<String>> getRelaysByCountryCode() { + return relaysByCountryCode; + } + + private Map<String, Set<String>> relaysByASNumber = null; + public void setRelaysByASNumber( + Map<String, Set<String>> relaysByASNumber) { + this.relaysByASNumber = relaysByASNumber; + } + public Map<String, Set<String>> getRelaysByASNumber() { + return relaysByASNumber; + } + + private Map<String, Set<String>> relaysByFlag = null; + public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) { + this.relaysByFlag = relaysByFlag; + } + public Map<String, Set<String>> getRelaysByFlag() { + return relaysByFlag; + } + + private Map<String, Set<String>> bridgesByFlag = null; + public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) { + this.bridgesByFlag = bridgesByFlag; + } + public Map<String, Set<String>> getBridgesByFlag() { + return bridgesByFlag; + } + + private Map<String, Set<String>> relaysByContact = null; + public void setRelaysByContact( + Map<String, Set<String>> relaysByContact) { + this.relaysByContact = relaysByContact; + } + public Map<String, Set<String>> getRelaysByContact() { + return relaysByContact; + } + + private SortedMap<Integer, Set<String>> relaysByFirstSeenDays; + public void setRelaysByFirstSeenDays( + SortedMap<Integer, Set<String>> relaysByFirstSeenDays) { + this.relaysByFirstSeenDays = relaysByFirstSeenDays; + } + public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() { + return relaysByFirstSeenDays; + } + + private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays; + public void setBridgesByFirstSeenDays( + SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) { + this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; + } + public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() { + return bridgesByFirstSeenDays; + } + + private SortedMap<Integer, Set<String>> relaysByLastSeenDays; + public void setRelaysByLastSeenDays( + SortedMap<Integer, Set<String>> relaysByLastSeenDays) { + this.relaysByLastSeenDays = relaysByLastSeenDays; + } + public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() { + return relaysByLastSeenDays; + } + + private SortedMap<Integer, Set<String>> bridgesByLastSeenDays; + public void setBridgesByLastSeenDays( + SortedMap<Integer, Set<String>> bridgesByLastSeenDays) { + this.bridgesByLastSeenDays = bridgesByLastSeenDays; + } + public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() { + return bridgesByLastSeenDays; + } +} + +public class NodeIndexer implements ServletContextListener, Runnable { + + public void contextInitialized(ServletContextEvent contextEvent) { + ServletContext servletContext = contextEvent.getServletContext(); + File outDir = new File(servletContext.getInitParameter("outDir")); + DocumentStore documentStore = ApplicationFactory.getDocumentStore(); + documentStore.setOutDir(outDir); + /* The servlet container created us, and we need to avoid that + * ApplicationFactory creates another instance of us. */ + ApplicationFactory.setNodeIndexer(this); + this.startIndexing(); + } + + public void contextDestroyed(ServletContextEvent contextEvent) { + this.stopIndexing(); + } + + private long lastIndexed = -1L; + + private NodeIndex latestNodeIndex = null; + + private Thread nodeIndexerThread = null; + + public synchronized long getLastIndexed(long timeoutMillis) { + if (this.lastIndexed == 0L && this.nodeIndexerThread != null && + timeoutMillis > 0L) { + try { + this.wait(timeoutMillis); + } catch (InterruptedException e) { + } + } + return this.lastIndexed; + } + + public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) { + if (this.latestNodeIndex == null && this.nodeIndexerThread != null && + timeoutMillis > 0L) { + try { + this.wait(timeoutMillis); + } catch (InterruptedException e) { + } + } + return this.latestNodeIndex; + } + + public synchronized void startIndexing() { + if (this.nodeIndexerThread == null) { + this.nodeIndexerThread = new Thread(this); + this.nodeIndexerThread.setDaemon(true); + this.nodeIndexerThread.start(); + } + } + + public void run() { + while (this.nodeIndexerThread != null) { + this.indexNodeStatuses(); + try { + Thread.sleep(DateTimeHelper.ONE_MINUTE); + } catch (InterruptedException e) { + } + } + } + + public synchronized void stopIndexing() { + Thread indexerThread = this.nodeIndexerThread; + this.nodeIndexerThread = null; + indexerThread.interrupt(); + } + + private void indexNodeStatuses() { + long updateStatusMillis = -1L; + DocumentStore documentStore = ApplicationFactory.getDocumentStore(); + UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, + false); + if (updateStatus != null && + updateStatus.getDocumentString() != null) { + String updateString = updateStatus.getDocumentString(); + try { + updateStatusMillis = Long.parseLong(updateString.trim()); + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + synchronized (this) { + if (updateStatusMillis <= this.lastIndexed) { + return; + } + } + List<String> newRelaysByConsensusWeight = new ArrayList<String>(); + Map<String, String> + newRelayFingerprintSummaryLines = new HashMap<String, String>(), + newBridgeFingerprintSummaryLines = new HashMap<String, String>(); + Map<String, Set<String>> + newRelaysByCountryCode = new HashMap<String, Set<String>>(), + newRelaysByASNumber = new HashMap<String, Set<String>>(), + newRelaysByFlag = new HashMap<String, Set<String>>(), + newBridgesByFlag = new HashMap<String, Set<String>>(), + newRelaysByContact = new HashMap<String, Set<String>>(); + SortedMap<Integer, Set<String>> + newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), + newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), + newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), + newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); + Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), + currentBridges = new HashSet<NodeStatus>(); + SortedSet<String> fingerprints = documentStore.list(NodeStatus.class, + false); + long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L; + for (String fingerprint : fingerprints) { + NodeStatus node = documentStore.retrieve(NodeStatus.class, true, + fingerprint); + if (node.isRelay()) { + relaysLastValidAfterMillis = Math.max( + relaysLastValidAfterMillis, node.getLastSeenMillis()); + currentRelays.add(node); + } else { + bridgesLastPublishedMillis = Math.max( + bridgesLastPublishedMillis, node.getLastSeenMillis()); + currentBridges.add(node); + } + } + Time time = ApplicationFactory.getTime(); + List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); + for (NodeStatus entry : currentRelays) { + String fingerprint = entry.getFingerprint().toUpperCase(); + String hashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + entry.setRunning(entry.getLastSeenMillis() == + relaysLastValidAfterMillis); + String line = formatRelaySummaryLine(entry); + newRelayFingerprintSummaryLines.put(fingerprint, line); + newRelayFingerprintSummaryLines.put(hashedFingerprint, line); + long consensusWeight = entry.getConsensusWeight(); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, fingerprint)); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, hashedFingerprint)); + if (entry.getCountryCode() != null) { + String countryCode = entry.getCountryCode(); + if (!newRelaysByCountryCode.containsKey(countryCode)) { + newRelaysByCountryCode.put(countryCode, + new HashSet<String>()); + } + newRelaysByCountryCode.get(countryCode).add(fingerprint); + newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); + } + if (entry.getASNumber() != null) { + String aSNumber = entry.getASNumber(); + if (!newRelaysByASNumber.containsKey(aSNumber)) { + newRelaysByASNumber.put(aSNumber, new HashSet<String>()); + } + newRelaysByASNumber.get(aSNumber).add(fingerprint); + newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); + } + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!newRelaysByFlag.containsKey(flagLowerCase)) { + newRelaysByFlag.put(flagLowerCase, new HashSet<String>()); + } + newRelaysByFlag.get(flagLowerCase).add(fingerprint); + newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); + } + int daysSinceFirstSeen = (int) ((time.currentTimeMillis() + - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { + newRelaysByFirstSeenDays.put(daysSinceFirstSeen, + new HashSet<String>()); + } + newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint); + newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedFingerprint); + int daysSinceLastSeen = (int) ((time.currentTimeMillis() + - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) { + newRelaysByLastSeenDays.put(daysSinceLastSeen, + new HashSet<String>()); + } + newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); + newRelaysByLastSeenDays.get(daysSinceLastSeen).add( + hashedFingerprint); + String contact = entry.getContact(); + if (!newRelaysByContact.containsKey(contact)) { + newRelaysByContact.put(contact, new HashSet<String>()); + } + newRelaysByContact.get(contact).add(fingerprint); + newRelaysByContact.get(contact).add(hashedFingerprint); + } + Collections.sort(orderRelaysByConsensusWeight); + newRelaysByConsensusWeight = new ArrayList<String>(); + for (String relay : orderRelaysByConsensusWeight) { + newRelaysByConsensusWeight.add(relay.split(" ")[1]); + } + for (NodeStatus entry : currentBridges) { + String hashedFingerprint = entry.getFingerprint().toUpperCase(); + String hashedHashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + entry.setRunning(entry.getRelayFlags().contains("Running") && + entry.getLastSeenMillis() == bridgesLastPublishedMillis); + String line = formatBridgeSummaryLine(entry); + newBridgeFingerprintSummaryLines.put(hashedFingerprint, line); + newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, + line); + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!newBridgesByFlag.containsKey(flagLowerCase)) { + newBridgesByFlag.put(flagLowerCase, new HashSet<String>()); + } + newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint); + newBridgesByFlag.get(flagLowerCase).add( + hashedHashedFingerprint); + } + int daysSinceFirstSeen = (int) ((time.currentTimeMillis() + - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) { + newBridgesByFirstSeenDays.put(daysSinceFirstSeen, + new HashSet<String>()); + } + newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedFingerprint); + newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedHashedFingerprint); + int daysSinceLastSeen = (int) ((time.currentTimeMillis() + - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) { + newBridgesByLastSeenDays.put(daysSinceLastSeen, + new HashSet<String>()); + } + newBridgesByLastSeenDays.get(daysSinceLastSeen).add( + hashedFingerprint); + newBridgesByLastSeenDays.get(daysSinceLastSeen).add( + hashedHashedFingerprint); + } + NodeIndex newNodeIndex = new NodeIndex(); + newNodeIndex.setRelaysByConsensusWeight(newRelaysByConsensusWeight); + newNodeIndex.setRelayFingerprintSummaryLines( + newRelayFingerprintSummaryLines); + newNodeIndex.setBridgeFingerprintSummaryLines( + newBridgeFingerprintSummaryLines); + newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode); + newNodeIndex.setRelaysByASNumber(newRelaysByASNumber); + newNodeIndex.setRelaysByFlag(newRelaysByFlag); + newNodeIndex.setBridgesByFlag(newBridgesByFlag); + newNodeIndex.setRelaysByContact(newRelaysByContact); + newNodeIndex.setRelaysByFirstSeenDays(newRelaysByFirstSeenDays); + newNodeIndex.setRelaysByLastSeenDays(newRelaysByLastSeenDays); + newNodeIndex.setBridgesByFirstSeenDays(newBridgesByFirstSeenDays); + newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays); + newNodeIndex.setRelaysPublishedString(DateTimeHelper.format( + relaysLastValidAfterMillis)); + newNodeIndex.setBridgesPublishedString(DateTimeHelper.format( + bridgesLastPublishedMillis)); + synchronized (this) { + this.lastIndexed = updateStatusMillis; + this.latestNodeIndex = newNodeIndex; + this.notifyAll(); + } + } + + private String formatRelaySummaryLine(NodeStatus entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String fingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + List<String> addresses = new ArrayList<String>(); + addresses.add(entry.getAddress()); + for (String orAddress : entry.getOrAddresses()) { + addresses.add(orAddress); + } + for (String exitAddress : entry.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + StringBuilder addressesBuilder = new StringBuilder(); + int written = 0; + for (String address : addresses) { + addressesBuilder.append((written++ > 0 ? "," : "") + """ + + address.toLowerCase() + """); + } + return String.format("{%s"f":"%s","a":[%s],"r":%s}", + (nickname == null ? "" : ""n":"" + nickname + "","), + fingerprint, addressesBuilder.toString(), running); + } + + private String formatBridgeSummaryLine(NodeStatus entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String hashedFingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + return String.format("{%s"h":"%s","r":%s}", + (nickname == null ? "" : ""n":"" + nickname + "","), + hashedFingerprint, running); + } +} + diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java index 89871ae..f0e58da 100644 --- a/src/org/torproject/onionoo/RequestHandler.java +++ b/src/org/torproject/onionoo/RequestHandler.java @@ -10,278 +10,13 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap;

public class RequestHandler {

- private static long summaryFileLastModified = -1L; - private static DocumentStore documentStore; - private static Time time; - private static boolean successfullyReadSummaryFile = false; - private static String relaysPublishedString, bridgesPublishedString; - private static List<String> relaysByConsensusWeight = null; - private static Map<String, String> relayFingerprintSummaryLines = null, - bridgeFingerprintSummaryLines = null; - private static Map<String, Set<String>> relaysByCountryCode = null, - relaysByASNumber = null, relaysByFlag = null, bridgesByFlag = null, - relaysByContact = null; - private static SortedMap<Integer, Set<String>> - relaysByFirstSeenDays = null, bridgesByFirstSeenDays = null, - relaysByLastSeenDays = null, bridgesByLastSeenDays = null; - private static final long SUMMARY_MAX_AGE = DateTimeHelper.SIX_HOURS; - - public static void initialize() { - documentStore = ApplicationFactory.getDocumentStore(); - time = ApplicationFactory.getTime(); - readSummaryFile(); - } - - public static boolean update() { - readSummaryFile(); - return successfullyReadSummaryFile; - } - - private static void readSummaryFile() { - long newSummaryFileLastModified = -1L; - UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, - false); - if (updateStatus != null && - updateStatus.getDocumentString() != null) { - String updateString = updateStatus.getDocumentString(); - try { - newSummaryFileLastModified = Long.parseLong(updateString.trim()); - } catch (NumberFormatException e) { - /* Handle below. */ - } - } - if (newSummaryFileLastModified < 0L) { - // TODO Does this actually solve anything? Should we instead - // switch to a variant of the maintenance mode and re-check when - // the next requests comes in that happens x seconds after this one? - successfullyReadSummaryFile = false; - return; - } - if (newSummaryFileLastModified + SUMMARY_MAX_AGE - < time.currentTimeMillis()) { - // TODO Does this actually solve anything? Should we instead - // switch to a variant of the maintenance mode and re-check when - // the next requests comes in that happens x seconds after this one? - successfullyReadSummaryFile = false; - return; - } - if (newSummaryFileLastModified > summaryFileLastModified) { - List<String> newRelaysByConsensusWeight = new ArrayList<String>(); - Map<String, String> - newRelayFingerprintSummaryLines = new HashMap<String, String>(), - newBridgeFingerprintSummaryLines = - new HashMap<String, String>(); - Map<String, Set<String>> - newRelaysByCountryCode = new HashMap<String, Set<String>>(), - newRelaysByASNumber = new HashMap<String, Set<String>>(), - newRelaysByFlag = new HashMap<String, Set<String>>(), - newBridgesByFlag = new HashMap<String, Set<String>>(), - newRelaysByContact = new HashMap<String, Set<String>>(); - SortedMap<Integer, Set<String>> - newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), - newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), - newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), - newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - long relaysLastValidAfterMillis = -1L, - bridgesLastPublishedMillis = -1L; - String newRelaysPublishedString, newBridgesPublishedString; - Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), - currentBridges = new HashSet<NodeStatus>(); - SortedSet<String> fingerprints = documentStore.list( - NodeStatus.class, false); - // TODO We should be able to learn if something goes wrong when - // reading the summary file, rather than silently having an empty - // list of fingerprints. - for (String fingerprint : fingerprints) { - NodeStatus node = documentStore.retrieve(NodeStatus.class, true, - fingerprint); - if (node.isRelay()) { - relaysLastValidAfterMillis = Math.max( - relaysLastValidAfterMillis, node.getLastSeenMillis()); - currentRelays.add(node); - } else { - bridgesLastPublishedMillis = Math.max( - bridgesLastPublishedMillis, node.getLastSeenMillis()); - currentBridges.add(node); - } - } - newRelaysPublishedString = DateTimeHelper.format( - relaysLastValidAfterMillis); - newBridgesPublishedString = DateTimeHelper.format( - bridgesLastPublishedMillis); - List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - for (NodeStatus entry : currentRelays) { - String fingerprint = entry.getFingerprint().toUpperCase(); - String hashedFingerprint = entry.getHashedFingerprint(). - toUpperCase(); - entry.setRunning(entry.getLastSeenMillis() == - relaysLastValidAfterMillis); - String line = formatRelaySummaryLine(entry); - newRelayFingerprintSummaryLines.put(fingerprint, line); - newRelayFingerprintSummaryLines.put(hashedFingerprint, line); - long consensusWeight = entry.getConsensusWeight(); - orderRelaysByConsensusWeight.add(String.format("%020d %s", - consensusWeight, fingerprint)); - orderRelaysByConsensusWeight.add(String.format("%020d %s", - consensusWeight, hashedFingerprint)); - if (entry.getCountryCode() != null) { - String countryCode = entry.getCountryCode(); - if (!newRelaysByCountryCode.containsKey(countryCode)) { - newRelaysByCountryCode.put(countryCode, - new HashSet<String>()); - } - newRelaysByCountryCode.get(countryCode).add(fingerprint); - newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); - } - if (entry.getASNumber() != null) { - String aSNumber = entry.getASNumber(); - if (!newRelaysByASNumber.containsKey(aSNumber)) { - newRelaysByASNumber.put(aSNumber, new HashSet<String>()); - } - newRelaysByASNumber.get(aSNumber).add(fingerprint); - newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); - } - for (String flag : entry.getRelayFlags()) { - String flagLowerCase = flag.toLowerCase(); - if (!newRelaysByFlag.containsKey(flagLowerCase)) { - newRelaysByFlag.put(flagLowerCase, new HashSet<String>()); - } - newRelaysByFlag.get(flagLowerCase).add(fingerprint); - newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); - } - int daysSinceFirstSeen = (int) ((newSummaryFileLastModified - - entry.getFirstSeenMillis()) / 86400000L); - if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { - newRelaysByFirstSeenDays.put(daysSinceFirstSeen, - new HashSet<String>()); - } - newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint); - newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedFingerprint); - int daysSinceLastSeen = (int) ((newSummaryFileLastModified - - entry.getLastSeenMillis()) / 86400000L); - if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) { - newRelaysByLastSeenDays.put(daysSinceLastSeen, - new HashSet<String>()); - } - newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); - newRelaysByLastSeenDays.get(daysSinceLastSeen).add( - hashedFingerprint); - String contact = entry.getContact(); - if (!newRelaysByContact.containsKey(contact)) { - newRelaysByContact.put(contact, new HashSet<String>()); - } - newRelaysByContact.get(contact).add(fingerprint); - newRelaysByContact.get(contact).add(hashedFingerprint); - } - Collections.sort(orderRelaysByConsensusWeight); - newRelaysByConsensusWeight = new ArrayList<String>(); - for (String relay : orderRelaysByConsensusWeight) { - newRelaysByConsensusWeight.add(relay.split(" ")[1]); - } - for (NodeStatus entry : currentBridges) { - String hashedFingerprint = entry.getFingerprint().toUpperCase(); - String hashedHashedFingerprint = entry.getHashedFingerprint(). - toUpperCase(); - entry.setRunning(entry.getRelayFlags().contains("Running") && - entry.getLastSeenMillis() == bridgesLastPublishedMillis); - String line = formatBridgeSummaryLine(entry); - newBridgeFingerprintSummaryLines.put(hashedFingerprint, line); - newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, - line); - for (String flag : entry.getRelayFlags()) { - String flagLowerCase = flag.toLowerCase(); - if (!newBridgesByFlag.containsKey(flagLowerCase)) { - newBridgesByFlag.put(flagLowerCase, new HashSet<String>()); - } - newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint); - newBridgesByFlag.get(flagLowerCase).add( - hashedHashedFingerprint); - } - int daysSinceFirstSeen = (int) ((newSummaryFileLastModified - - entry.getFirstSeenMillis()) / 86400000L); - if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) { - newBridgesByFirstSeenDays.put(daysSinceFirstSeen, - new HashSet<String>()); - } - newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedFingerprint); - newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedHashedFingerprint); - int daysSinceLastSeen = (int) ((newSummaryFileLastModified - - entry.getLastSeenMillis()) / 86400000L); - if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) { - newBridgesByLastSeenDays.put(daysSinceLastSeen, - new HashSet<String>()); - } - newBridgesByLastSeenDays.get(daysSinceLastSeen).add( - hashedFingerprint); - newBridgesByLastSeenDays.get(daysSinceLastSeen).add( - hashedHashedFingerprint); - } - relaysByConsensusWeight = newRelaysByConsensusWeight; - relayFingerprintSummaryLines = newRelayFingerprintSummaryLines; - bridgeFingerprintSummaryLines = newBridgeFingerprintSummaryLines; - relaysByCountryCode = newRelaysByCountryCode; - relaysByASNumber = newRelaysByASNumber; - relaysByFlag = newRelaysByFlag; - bridgesByFlag = newBridgesByFlag; - relaysByContact = newRelaysByContact; - relaysByFirstSeenDays = newRelaysByFirstSeenDays; - relaysByLastSeenDays = newRelaysByLastSeenDays; - bridgesByFirstSeenDays = newBridgesByFirstSeenDays; - bridgesByLastSeenDays = newBridgesByLastSeenDays; - relaysPublishedString = newRelaysPublishedString; - bridgesPublishedString = newBridgesPublishedString; - } - summaryFileLastModified = newSummaryFileLastModified; - successfullyReadSummaryFile = true; - } - - private static String formatRelaySummaryLine(NodeStatus entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String fingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress); - } - for (String exitAddress : entry.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } - StringBuilder addressesBuilder = new StringBuilder(); - int written = 0; - for (String address : addresses) { - addressesBuilder.append((written++ > 0 ? "," : "") + """ - + address.toLowerCase() + """); - } - return String.format("{%s"f":"%s","a":[%s],"r":%s}", - (nickname == null ? "" : ""n":"" + nickname + "","), - fingerprint, addressesBuilder.toString(), running); - } - - private static String formatBridgeSummaryLine(NodeStatus entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String hashedFingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - return String.format("{%s"h":"%s","r":%s}", - (nickname == null ? "" : ""n":"" + nickname + "","), - hashedFingerprint, running); - } + private NodeIndex nodeIndex;

- public static long getLastModified() { - readSummaryFile(); - return summaryFileLastModified; + public RequestHandler(NodeIndex nodeIndex) { + this.nodeIndex = nodeIndex; }

private String resourceType; @@ -368,8 +103,10 @@ public class RequestHandler { new HashMap<String, String>();

public void handleRequest() { - this.filteredRelays.putAll(relayFingerprintSummaryLines); - this.filteredBridges.putAll(bridgeFingerprintSummaryLines); + this.filteredRelays.putAll( + this.nodeIndex.getRelayFingerprintSummaryLines()); + this.filteredBridges.putAll( + this.nodeIndex.getBridgeFingerprintSummaryLines()); this.filterByResourceType(); this.filterByType(); this.filterByRunning(); @@ -531,11 +268,12 @@ public class RequestHandler { return; } String countryCode = this.country.toLowerCase(); - if (!relaysByCountryCode.containsKey(countryCode)) { + if (!this.nodeIndex.getRelaysByCountryCode().containsKey( + countryCode)) { this.filteredRelays.clear(); } else { Set<String> relaysWithCountryCode = - relaysByCountryCode.get(countryCode); + this.nodeIndex.getRelaysByCountryCode().get(countryCode); Set<String> removeRelays = new HashSet<String>(); for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { String fingerprint = e.getKey(); @@ -558,11 +296,11 @@ public class RequestHandler { if (!aSNumber.startsWith("AS")) { aSNumber = "AS" + aSNumber; } - if (!relaysByASNumber.containsKey(aSNumber)) { + if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) { this.filteredRelays.clear(); } else { Set<String> relaysWithASNumber = - relaysByASNumber.get(aSNumber); + this.nodeIndex.getRelaysByASNumber().get(aSNumber); Set<String> removeRelays = new HashSet<String>(); for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { String fingerprint = e.getKey(); @@ -582,10 +320,11 @@ public class RequestHandler { return; } String flag = this.flag.toLowerCase(); - if (!relaysByFlag.containsKey(flag)) { + if (!this.nodeIndex.getRelaysByFlag().containsKey(flag)) { this.filteredRelays.clear(); } else { - Set<String> relaysWithFlag = relaysByFlag.get(flag); + Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get( + flag); Set<String> removeRelays = new HashSet<String>(); for (Map.Entry<String, String> e : this.filteredRelays.entrySet()) { String fingerprint = e.getKey(); @@ -597,10 +336,11 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } } - if (!bridgesByFlag.containsKey(flag)) { + if (!this.nodeIndex.getBridgesByFlag().containsKey(flag)) { this.filteredBridges.clear(); } else { - Set<String> bridgesWithFlag = bridgesByFlag.get(flag); + Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get( + flag); Set<String> removeBridges = new HashSet<String>(); for (Map.Entry<String, String> e : this.filteredBridges.entrySet()) { @@ -619,20 +359,20 @@ public class RequestHandler { if (this.firstSeenDays == null) { return; } - filterNodesByDays(this.filteredRelays, relaysByFirstSeenDays, - this.firstSeenDays); - filterNodesByDays(this.filteredBridges, bridgesByFirstSeenDays, - this.firstSeenDays); + filterNodesByDays(this.filteredRelays, + this.nodeIndex.getRelaysByFirstSeenDays(), this.firstSeenDays); + filterNodesByDays(this.filteredBridges, + this.nodeIndex.getBridgesByFirstSeenDays(), this.firstSeenDays); }

private void filterNodesByLastSeenDays() { if (this.lastSeenDays == null) { return; } - filterNodesByDays(this.filteredRelays, relaysByLastSeenDays, - this.lastSeenDays); - filterNodesByDays(this.filteredBridges, bridgesByLastSeenDays, - this.lastSeenDays); + filterNodesByDays(this.filteredRelays, + this.nodeIndex.getRelaysByLastSeenDays(), this.lastSeenDays); + filterNodesByDays(this.filteredBridges, + this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays); }

private void filterNodesByDays(Map<String, String> filteredNodes, @@ -657,7 +397,8 @@ public class RequestHandler { return; } Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, Set<String>> e : relaysByContact.entrySet()) { + for (Map.Entry<String, Set<String>> e : + this.nodeIndex.getRelaysByContact().entrySet()) { String contact = e.getKey(); for (String contactPart : this.contact) { if (contact == null || @@ -676,7 +417,7 @@ public class RequestHandler { private void order() { if (this.order != null && this.order.length == 1) { List<String> orderBy = new ArrayList<String>( - relaysByConsensusWeight); + this.nodeIndex.getRelaysByConsensusWeight()); if (this.order[0].startsWith("-")) { Collections.reverse(orderBy); } @@ -747,10 +488,10 @@ public class RequestHandler { }

public String getRelaysPublishedString() { - return relaysPublishedString; + return this.nodeIndex.getRelaysPublishedString(); }

public String getBridgesPublishedString() { - return bridgesPublishedString; + return this.nodeIndex.getBridgesPublishedString(); } } diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index dcfefc5..f2f3005 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

-import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.util.Arrays; @@ -27,30 +26,17 @@ public class ResourceServlet extends HttpServlet { /* Called by servlet container, not by test class. */ public void init(ServletConfig config) throws ServletException { super.init(config); - boolean maintenanceMode = - config.getInitParameter("maintenance") != null - && config.getInitParameter("maintenance").equals("1"); - File outDir = new File(config.getInitParameter("outDir")); - DocumentStore documentStore = ApplicationFactory.getDocumentStore(); - documentStore.setOutDir(outDir); - this.init(maintenanceMode); - } - - /* Called (indirectly) by servlet container and (directly) by test - * class. */ - protected void init(boolean maintenanceMode) { - this.maintenanceMode = maintenanceMode; - if (!maintenanceMode) { - RequestHandler.initialize(); - ResponseBuilder.initialize(); - } + this.maintenanceMode = + config.getInitParameter("maintenance") != null && + config.getInitParameter("maintenance").equals("1"); }

public long getLastModified(HttpServletRequest request) { if (this.maintenanceMode) { return super.getLastModified(request); } else { - return RequestHandler.getLastModified(); + return ApplicationFactory.getNodeIndexer().getLastIndexed( + DateTimeHelper.TEN_SECONDS); } }

@@ -109,7 +95,16 @@ public class ResourceServlet extends HttpServlet { return; }

- if (!RequestHandler.update()) { + if (ApplicationFactory.getNodeIndexer().getLastIndexed( + DateTimeHelper.TEN_SECONDS) + DateTimeHelper.SIX_HOURS + < ApplicationFactory.getTime().currentTimeMillis()) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + NodeIndex nodeIndex = ApplicationFactory.getNodeIndexer(). + getLatestNodeIndex(DateTimeHelper.TEN_SECONDS); + if (nodeIndex == null) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; } @@ -135,7 +130,8 @@ public class ResourceServlet extends HttpServlet { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - RequestHandler rh = new RequestHandler(); + + RequestHandler rh = new RequestHandler(nodeIndex); rh.setResourceType(resourceType);

/* Extract parameters either from the old-style URI or from request diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index a841d29..d14ee5b 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -9,10 +9,10 @@ import java.util.Scanner;

public class ResponseBuilder {

- private static DocumentStore documentStore; + private DocumentStore documentStore;

- public static void initialize() { - documentStore = ApplicationFactory.getDocumentStore(); + public ResponseBuilder() { + this.documentStore = ApplicationFactory.getDocumentStore(); }

private String resourceType; @@ -114,7 +114,7 @@ public class ResponseBuilder { return ""; } fingerprint = fingerprint.substring(0, 40); - DetailsDocument detailsDocument = documentStore.retrieve( + DetailsDocument detailsDocument = this.documentStore.retrieve( DetailsDocument.class, false, fingerprint); if (detailsDocument != null && detailsDocument.getDocumentString() != null) { @@ -184,7 +184,7 @@ public class ResponseBuilder { return ""; } fingerprint = fingerprint.substring(0, 40); - BandwidthDocument bandwidthDocument = documentStore.retrieve( + BandwidthDocument bandwidthDocument = this.documentStore.retrieve( BandwidthDocument.class, false, fingerprint); if (bandwidthDocument != null && bandwidthDocument.getDocumentString() != null) { @@ -208,7 +208,7 @@ public class ResponseBuilder { return ""; } fingerprint = fingerprint.substring(0, 40); - WeightsDocument weightsDocument = documentStore.retrieve( + WeightsDocument weightsDocument = this.documentStore.retrieve( WeightsDocument.class, false, fingerprint); if (weightsDocument != null && weightsDocument.getDocumentString() != null) { @@ -231,7 +231,7 @@ public class ResponseBuilder { return ""; } fingerprint = fingerprint.substring(0, 40); - ClientsDocument clientsDocument = documentStore.retrieve( + ClientsDocument clientsDocument = this.documentStore.retrieve( ClientsDocument.class, false, fingerprint); if (clientsDocument != null && clientsDocument.getDocumentString() != null) { @@ -257,7 +257,7 @@ public class ResponseBuilder { return ""; } fingerprint = fingerprint.substring(0, 40); - UptimeDocument uptimeDocument = documentStore.retrieve( + UptimeDocument uptimeDocument = this.documentStore.retrieve( UptimeDocument.class, false, fingerprint); if (uptimeDocument != null && uptimeDocument.getDocumentString() != null) { diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index cbe787c..f5395ed 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -32,12 +32,8 @@ public class ResourceServletTest {

private SortedMap<String, String> relays, bridges;

- // 2013-04-24 12:22:22 - private static long lastModified = 1366806142000L; - - private long currentTimeMillis = 1366806142000L; - - private boolean maintenanceMode = false; + private long currentTimeMillis = DateTimeHelper.parse( + "2013-04-24 12:22:22");

private class TestingHttpServletRequestWrapper extends HttpServletRequestWrapper { @@ -149,6 +145,7 @@ public class ResourceServletTest { try { this.createDummyTime(); this.createDummyDocumentStore(); + this.createNodeIndexer(); this.makeRequest(requestURI, parameterMap); this.parseResponse(); } catch (IOException e) { @@ -162,13 +159,10 @@ public class ResourceServletTest { }

private void createDummyDocumentStore() { - /* TODO Incrementing static lastModified is necessary for - * ResponseBuilder to read state from the newly created DocumentStore. - * Otherwise, ResponseBuilder would use data from the previous test - * run. This is bad design and should be fixed. */ DummyDocumentStore documentStore = new DummyDocumentStore(); UpdateStatus updateStatus = new UpdateStatus(); - updateStatus.setDocumentString(String.valueOf(lastModified++)); + updateStatus.setDocumentString(String.valueOf( + this.currentTimeMillis)); documentStore.addDocument(updateStatus, null); for (Map.Entry<String, String> e : relays.entrySet()) { documentStore.addDocument(NodeStatus.fromString(e.getValue()), @@ -181,10 +175,15 @@ public class ResourceServletTest { ApplicationFactory.setDocumentStore(documentStore); }

+ private void createNodeIndexer() { + NodeIndexer newNodeIndexer = new NodeIndexer(); + newNodeIndexer.startIndexing(); + ApplicationFactory.setNodeIndexer(newNodeIndexer); + } + private void makeRequest(String requestURI, Map<String, String[]> parameterMap) throws IOException { ResourceServlet rs = new ResourceServlet(); - rs.init(this.maintenanceMode); this.request = new TestingHttpServletRequestWrapper(requestURI, parameterMap); this.response = new TestingHttpServletResponseWrapper();