commit ab7f4ee58825a7980579e6cd4e334ee373b29047 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Jul 22 13:53:42 2014 +0200 Move front-end parts of NodeStatus to SummaryDocument. The NodeStatus class was seriously overloaded: 1. NodeDetailsStatusUpdater uses the list of NodeStatus instances as input to its GeoIP and rDNS look-up operations and to calculate path-selection probabilities. 2. DetailsDocumentWriter uses fields from NodeStatus instances (some of them being stored persistently, some only kept in memory) to write part of DetailsDocument instances. 3. NodeIndexer obtains all relevant data to populate the search index from NodeStatus instances. 4. ResponseBuilder uses NodeStatus instances to produce summary documents on-the-fly. As first step to reduce this complexity, let's copy the parts used for 3 and 4 to a currently unused class, SummaryDocument, and use that class in NodeIndexer and ResponseBuilder. It makes sense to combine those two purposes for performance reasons. Parts 1 and 2 will be simplified in later commits. --- src/org/torproject/onionoo/DocumentStore.java | 235 +++++++++++++------- src/org/torproject/onionoo/Main.java | 4 +- .../onionoo/NodeDetailsStatusUpdater.java | 2 +- src/org/torproject/onionoo/NodeIndexer.java | 38 ++-- src/org/torproject/onionoo/RequestHandler.java | 65 +++--- src/org/torproject/onionoo/ResponseBuilder.java | 52 ++--- src/org/torproject/onionoo/SummaryDocument.java | 195 +++++++++++++++- .../torproject/onionoo/SummaryDocumentWriter.java | 87 ++++++++ .../org/torproject/onionoo/DummyDocumentStore.java | 2 +- .../torproject/onionoo/ResourceServletTest.java | 107 +++++---- 10 files changed, 569 insertions(+), 218 deletions(-) diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index e3bf618..434ecb6 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -11,7 +11,9 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -43,46 +45,39 @@ public class DocumentStore { this.time = ApplicationFactory.getTime(); } - private boolean listedArchivedNodeStatuses = false, - listedCurrentNodeStatuses = false; - private long listOperations = 0L, listedFiles = 0L, storedFiles = 0L, storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L, removedFiles = 0L; - /* Node statuses are cached in memory, as opposed to all other document - * types. This cache is initialized when listing NodeStatus documents, - * either including or excluding archived node statuses. Later retrieve - * operations depend on which NodeStatus documents were listed. */ + /* Node statuses and summary documents are cached in memory, as opposed + * to all other document types. These caches are initialized when first + * accessing or modifying a NodeStatus or SummaryDocument document, + * respectively. */ private SortedMap<String, NodeStatus> cachedNodeStatuses; + private SortedMap<String, SummaryDocument> cachedSummaryDocuments; public <T extends Document> SortedSet<String> list( - Class<T> documentType, boolean includeArchive) { + Class<T> documentType) { if (documentType.equals(NodeStatus.class)) { - return this.listNodeStatuses(includeArchive); + return this.listNodeStatuses(); + } else if (documentType.equals(SummaryDocument.class)) { + return this.listSummaryDocuments(); } else { return this.listDocumentFiles(documentType); } } - private SortedSet<String> listNodeStatuses(boolean includeArchive) { - if ((includeArchive && listedCurrentNodeStatuses) || - (!includeArchive && listedArchivedNodeStatuses)) { - System.err.println("Listing node statuses is only permitted " - + "including the archive or excluding the archive, but not " - + "both. Returning empty list."); - return new TreeSet<String>(); - } + private SortedSet<String> listNodeStatuses() { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(includeArchive); + this.cacheNodeStatuses(); } return new TreeSet<String>(this.cachedNodeStatuses.keySet()); } - private void cacheNodeStatuses(boolean includeArchive) { + private void cacheNodeStatuses() { SortedMap<String, NodeStatus> parsedNodeStatuses = new TreeMap<String, NodeStatus>(); - File directory = includeArchive ? this.statusDir : this.outDir; + File directory = this.statusDir; if (directory != null) { File summaryFile = new File(directory, "summary"); if (summaryFile.exists()) { @@ -109,14 +104,56 @@ public class DocumentStore { } } } - if (includeArchive) { - this.listedArchivedNodeStatuses = true; - } else { - this.listedCurrentNodeStatuses = true; - } this.cachedNodeStatuses = parsedNodeStatuses; } + private SortedSet<String> listSummaryDocuments() { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + } + + private void cacheSummaryDocuments() { + SortedMap<String, SummaryDocument> parsedSummaryDocuments = + new TreeMap<String, SummaryDocument>(); + File directory = this.outDir; + if (directory != null) { + File summaryFile = new File(directory, "summary"); + if (summaryFile.exists()) { + String line = null; + try { + Gson gson = new Gson(); + BufferedReader br = new BufferedReader(new FileReader( + summaryFile)); + while ((line = br.readLine()) != null) { + if (line.length() == 0) { + continue; + } + SummaryDocument summaryDocument = gson.fromJson(line, + SummaryDocument.class); + if (summaryDocument != null) { + parsedSummaryDocuments.put(summaryDocument.getFingerprint(), + summaryDocument); + } + } + br.close(); + this.listedFiles += parsedSummaryDocuments.size(); + this.listOperations++; + } catch (IOException e) { + System.err.println("Could not read file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } catch (JsonParseException e) { + System.err.println("Could not parse summary document '" + line + + "' in file '" + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + } + this.cachedSummaryDocuments = parsedSummaryDocuments; + } + private <T extends Document> SortedSet<String> listDocumentFiles( Class<T> documentType) { SortedSet<String> fingerprints = new TreeSet<String>(); @@ -178,6 +215,9 @@ public class DocumentStore { String fingerprint) { if (document instanceof NodeStatus) { return this.storeNodeStatus((NodeStatus) document, fingerprint); + } else if (document instanceof SummaryDocument) { + return this.storeSummaryDocument((SummaryDocument) document, + fingerprint); } else { return this.storeDocumentFile(document, fingerprint); } @@ -186,12 +226,21 @@ public class DocumentStore { private <T extends Document> boolean storeNodeStatus( NodeStatus nodeStatus, String fingerprint) { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(true); + this.cacheNodeStatuses(); } this.cachedNodeStatuses.put(fingerprint, nodeStatus); return true; } + private <T extends Document> boolean storeSummaryDocument( + SummaryDocument summaryDocument, String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + this.cachedSummaryDocuments.put(fingerprint, summaryDocument); + return true; + } + private <T extends Document> boolean storeDocumentFile(T document, String fingerprint) { File documentFile = this.getDocumentFile(document.getClass(), @@ -270,6 +319,8 @@ public class DocumentStore { boolean parse, String fingerprint) { if (documentType.equals(NodeStatus.class)) { return documentType.cast(this.retrieveNodeStatus(fingerprint)); + } else if (documentType.equals(SummaryDocument.class)) { + return documentType.cast(this.retrieveSummaryDocument(fingerprint)); } else { return this.retrieveDocumentFile(documentType, parse, fingerprint); } @@ -277,12 +328,17 @@ public class DocumentStore { private NodeStatus retrieveNodeStatus(String fingerprint) { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(true); + this.cacheNodeStatuses(); } - if (this.cachedNodeStatuses.containsKey(fingerprint)) { - return this.cachedNodeStatuses.get(fingerprint); - } else if (this.listedArchivedNodeStatuses) { - return null; + return this.cachedNodeStatuses.get(fingerprint); + } + + private SummaryDocument retrieveSummaryDocument(String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + if (this.cachedSummaryDocuments.containsKey(fingerprint)) { + return this.cachedSummaryDocuments.get(fingerprint); } /* TODO This is an evil hack to support looking up relays or bridges * that haven't been running for a week without having to load @@ -294,39 +350,35 @@ public class DocumentStore { return null; } boolean isRelay = detailsDocument.getHashedFingerprint() == null; + boolean running = false; String nickname = detailsDocument.getNickname(); - String address = null, countryCode = null, hostName = null, - defaultPolicy = null, portList = null, aSNumber = null, - contact = null; - SortedSet<String> orAddressesAndPorts = new TreeSet<String>(); + List<String> addresses = new ArrayList<String>(); + String countryCode = null, aSNumber = null, contact = null; for (String orAddressAndPort : detailsDocument.getOrAddresses()) { - if (address == null) { - if (!orAddressAndPort.contains(":")) { - return null; - } - address = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(":")); - } else { - orAddressesAndPorts.add(orAddressAndPort); + if (!orAddressAndPort.contains(":")) { + return null; + } + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(":")); + if (!addresses.contains(orAddress)) { + addresses.add(orAddress); } } - SortedSet<String> exitAddresses = new TreeSet<String>(); if (detailsDocument.getExitAddresses() != null) { - exitAddresses.addAll(detailsDocument.getExitAddresses()); + for (String exitAddress : detailsDocument.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } } SortedSet<String> relayFlags = new TreeSet<String>(), family = null; long lastSeenMillis = -1L, consensusWeight = -1L, - lastRdnsLookup = -1L, firstSeenMillis = -1L, - lastChangedAddresses = -1L; - int orPort = 0, dirPort = 0; - Boolean recommendedVersion = null; - NodeStatus nodeStatus = new NodeStatus(isRelay, nickname, fingerprint, - address, orAddressesAndPorts, exitAddresses, lastSeenMillis, - orPort, dirPort, relayFlags, consensusWeight, countryCode, - hostName, lastRdnsLookup, defaultPolicy, portList, - firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion, family); - return nodeStatus; + firstSeenMillis = -1L; + SummaryDocument summaryDocument = new SummaryDocument(isRelay, + nickname, fingerprint, addresses, lastSeenMillis, running, + relayFlags, consensusWeight, countryCode, firstSeenMillis, + aSNumber, contact, family); + return summaryDocument; } private <T extends Document> T retrieveDocumentFile( @@ -455,6 +507,8 @@ public class DocumentStore { String fingerprint) { if (documentType.equals(NodeStatus.class)) { return this.removeNodeStatus(fingerprint); + } else if (documentType.equals(SummaryDocument.class)) { + return this.removeSummaryDocument(fingerprint); } else { return this.removeDocumentFile(documentType, fingerprint); } @@ -462,11 +516,18 @@ public class DocumentStore { private boolean removeNodeStatus(String fingerprint) { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(true); + this.cacheNodeStatuses(); } return this.cachedNodeStatuses.remove(fingerprint) != null; } + private boolean removeSummaryDocument(String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + return this.cachedSummaryDocuments.remove(fingerprint) != null; + } + private <T extends Document> boolean removeDocumentFile( Class<T> documentType, String fingerprint) { File documentFile = this.getDocumentFile(documentType, fingerprint); @@ -550,18 +611,20 @@ public class DocumentStore { * containing current time. It's important to write the update file * now, not earlier, because the front-end should not read new node * statuses until all details, bandwidths, and weights are ready. */ - if (this.listedArchivedNodeStatuses) { - this.writeNodeStatuses(false); - this.writeNodeStatuses(true); - this.writeUpdateStatus(); - } else if (this.listedCurrentNodeStatuses) { - this.writeNodeStatuses(false); + if (this.cachedNodeStatuses != null || + this.cachedSummaryDocuments != null) { + if (this.cachedNodeStatuses != null) { + this.writeNodeStatuses(); + } + if (this.cachedSummaryDocuments != null) { + this.writeSummaryDocuments(); + } this.writeUpdateStatus(); } } - private void writeNodeStatuses(boolean includeArchive) { - File directory = includeArchive ? this.statusDir : this.outDir; + private void writeNodeStatuses() { + File directory = this.statusDir; if (directory == null) { return; } @@ -569,21 +632,8 @@ public class DocumentStore { SortedMap<String, NodeStatus> cachedRelays = new TreeMap<String, NodeStatus>(), cachedBridges = new TreeMap<String, NodeStatus>(); - long cutoff = 0L; - if (!includeArchive) { - long maxLastSeenMillis = 0L; - for (NodeStatus node : this.cachedNodeStatuses.values()) { - if (node.getLastSeenMillis() > maxLastSeenMillis) { - maxLastSeenMillis = node.getLastSeenMillis(); - } - } - cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; - } for (Map.Entry<String, NodeStatus> e : this.cachedNodeStatuses.entrySet()) { - if (e.getValue().getLastSeenMillis() < cutoff) { - continue; - } if (e.getValue().isRelay()) { cachedRelays.put(e.getKey(), e.getValue()); } else { @@ -624,6 +674,35 @@ public class DocumentStore { } } + private void writeSummaryDocuments() { + StringBuilder sb = new StringBuilder(); + Gson gson = new Gson(); + for (SummaryDocument summaryDocument : + this.cachedSummaryDocuments.values()) { + String line = gson.toJson(summaryDocument); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize relay summary document '" + + summaryDocument.getFingerprint() + "'"); + } + } + String documentString = sb.toString(); + File summaryFile = new File(this.outDir, "summary"); + try { + summaryFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + private void writeUpdateStatus() { if (this.outDir == null) { return; diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 0ae12e4..e9afe2f 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -45,6 +45,8 @@ public class Main { StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu, usu }; + SummaryDocumentWriter sdw = new SummaryDocumentWriter(); + Logger.printStatusTime("Initialized summary document writer"); DetailsDocumentWriter ddw = new DetailsDocumentWriter(); Logger.printStatusTime("Initialized details document writer"); BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(); @@ -55,7 +57,7 @@ public class Main { Logger.printStatusTime("Initialized clients document writer"); UptimeDocumentWriter udw = new UptimeDocumentWriter(); Logger.printStatusTime("Initialized uptime document writer"); - DocumentWriter[] dws = new DocumentWriter[] { ddw, bdw, wdw, cdw, + DocumentWriter[] dws = new DocumentWriter[] { sdw, ddw, bdw, wdw, cdw, udw }; Logger.printStatus("Downloading descriptors."); diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index e898638..a1149a5 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -310,7 +310,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private void readStatusSummary() { SortedSet<String> fingerprints = this.documentStore.list( - NodeStatus.class, true); + NodeStatus.class); for (String fingerprint : fingerprints) { NodeStatus node = this.documentStore.retrieve(NodeStatus.class, true, fingerprint); diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java index 8a04afb..d24284e 100644 --- a/src/org/torproject/onionoo/NodeIndexer.java +++ b/src/org/torproject/onionoo/NodeIndexer.java @@ -44,21 +44,21 @@ class NodeIndex { } - private Map<String, NodeStatus> relayFingerprintSummaryLines; + private Map<String, SummaryDocument> relayFingerprintSummaryLines; public void setRelayFingerprintSummaryLines( - Map<String, NodeStatus> relayFingerprintSummaryLines) { + Map<String, SummaryDocument> relayFingerprintSummaryLines) { this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; } - public Map<String, NodeStatus> getRelayFingerprintSummaryLines() { + public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() { return this.relayFingerprintSummaryLines; } - private Map<String, NodeStatus> bridgeFingerprintSummaryLines; + private Map<String, SummaryDocument> bridgeFingerprintSummaryLines; public void setBridgeFingerprintSummaryLines( - Map<String, NodeStatus> bridgeFingerprintSummaryLines) { + Map<String, SummaryDocument> bridgeFingerprintSummaryLines) { this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; } - public Map<String, NodeStatus> getBridgeFingerprintSummaryLines() { + public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() { return this.bridgeFingerprintSummaryLines; } @@ -239,11 +239,11 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } List<String> newRelaysByConsensusWeight = new ArrayList<String>(); - Map<String, NodeStatus> + Map<String, SummaryDocument> newRelayFingerprintSummaryLines = - new HashMap<String, NodeStatus>(), + new HashMap<String, SummaryDocument>(), newBridgeFingerprintSummaryLines = - new HashMap<String, NodeStatus>(); + new HashMap<String, SummaryDocument>(); Map<String, Set<String>> newRelaysByCountryCode = new HashMap<String, Set<String>>(), newRelaysByASNumber = new HashMap<String, Set<String>>(), @@ -256,14 +256,14 @@ public class NodeIndexer implements ServletContextListener, Runnable { newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), - currentBridges = new HashSet<NodeStatus>(); - SortedSet<String> fingerprints = documentStore.list(NodeStatus.class, - false); + Set<SummaryDocument> currentRelays = new HashSet<SummaryDocument>(), + currentBridges = new HashSet<SummaryDocument>(); + SortedSet<String> fingerprints = documentStore.list( + SummaryDocument.class); long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L; for (String fingerprint : fingerprints) { - NodeStatus node = documentStore.retrieve(NodeStatus.class, true, - fingerprint); + SummaryDocument node = documentStore.retrieve(SummaryDocument.class, + true, fingerprint); if (node.isRelay()) { relaysLastValidAfterMillis = Math.max( relaysLastValidAfterMillis, node.getLastSeenMillis()); @@ -276,12 +276,10 @@ public class NodeIndexer implements ServletContextListener, Runnable { } Time time = ApplicationFactory.getTime(); List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - for (NodeStatus entry : currentRelays) { + for (SummaryDocument entry : currentRelays) { String fingerprint = entry.getFingerprint().toUpperCase(); String hashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); - entry.setRunning(entry.getLastSeenMillis() == - relaysLastValidAfterMillis); newRelayFingerprintSummaryLines.put(fingerprint, entry); newRelayFingerprintSummaryLines.put(hashedFingerprint, entry); long consensusWeight = entry.getConsensusWeight(); @@ -360,12 +358,10 @@ public class NodeIndexer implements ServletContextListener, Runnable { } e.getValue().retainAll(inMutualFamilyRelation); } - for (NodeStatus entry : currentBridges) { + for (SummaryDocument entry : currentBridges) { String hashedFingerprint = entry.getFingerprint().toUpperCase(); String hashedHashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); - entry.setRunning(entry.getRelayFlags().contains("Running") && - entry.getLastSeenMillis() == bridgesLastPublishedMillis); newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry); newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, entry); diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java index 66559e6..39a66e1 100644 --- a/src/org/torproject/onionoo/RequestHandler.java +++ b/src/org/torproject/onionoo/RequestHandler.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; +import java.util.SortedSet; public class RequestHandler { @@ -109,11 +110,11 @@ public class RequestHandler { this.family = family; } - private Map<String, NodeStatus> filteredRelays = - new HashMap<String, NodeStatus>(); + private Map<String, SummaryDocument> filteredRelays = + new HashMap<String, SummaryDocument>(); - private Map<String, NodeStatus> filteredBridges = - new HashMap<String, NodeStatus>(); + private Map<String, SummaryDocument> filteredBridges = + new HashMap<String, SummaryDocument>(); public void handleRequest() { this.filteredRelays.putAll( @@ -164,8 +165,9 @@ public class RequestHandler { } boolean runningRequested = this.running.equals("true"); Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) { - if (e.getValue().getRunning() != runningRequested) { + for (Map.Entry<String, SummaryDocument> e : + filteredRelays.entrySet()) { + if (e.getValue().isRunning() != runningRequested) { removeRelays.add(e.getKey()); } } @@ -173,8 +175,9 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) { - if (e.getValue().getRunning() != runningRequested) { + for (Map.Entry<String, SummaryDocument> e : + filteredBridges.entrySet()) { + if (e.getValue().isRunning() != runningRequested) { removeBridges.add(e.getKey()); } } @@ -194,9 +197,10 @@ public class RequestHandler { private void filterBySearchTerm(String searchTerm) { Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) { + for (Map.Entry<String, SummaryDocument> e : + filteredRelays.entrySet()) { String fingerprint = e.getKey(); - NodeStatus entry = e.getValue(); + SummaryDocument entry = e.getValue(); boolean lineMatches = false; String nickname = entry.getNickname() != null ? entry.getNickname().toLowerCase() : "unnamed"; @@ -214,14 +218,7 @@ public class RequestHandler { /* Non-$-prefixed fingerprint matches. */ lineMatches = true; } else { - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress().toLowerCase()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress.toLowerCase()); - } - for (String exitAddress : entry.getExitAddresses()) { - addresses.add(exitAddress.toLowerCase()); - } + List<String> addresses = entry.getAddresses(); for (String address : addresses) { if (address.startsWith(searchTerm.toLowerCase())) { /* Address matches. */ @@ -238,9 +235,10 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) { + for (Map.Entry<String, SummaryDocument> e : + filteredBridges.entrySet()) { String hashedFingerprint = e.getKey(); - NodeStatus entry = e.getValue(); + SummaryDocument entry = e.getValue(); boolean lineMatches = false; String nickname = entry.getNickname() != null ? entry.getNickname().toLowerCase() : "unnamed"; @@ -272,12 +270,12 @@ public class RequestHandler { return; } String fingerprint = this.lookup; - NodeStatus relayLine = this.filteredRelays.get(fingerprint); + SummaryDocument relayLine = this.filteredRelays.get(fingerprint); this.filteredRelays.clear(); if (relayLine != null) { this.filteredRelays.put(fingerprint, relayLine); } - NodeStatus bridgeLine = this.filteredBridges.get(fingerprint); + SummaryDocument bridgeLine = this.filteredBridges.get(fingerprint); this.filteredBridges.clear(); if (bridgeLine != null) { this.filteredBridges.put(fingerprint, bridgeLine); @@ -291,8 +289,8 @@ public class RequestHandler { this.filteredRelays.clear(); this.filteredBridges.clear(); String fingerprint = this.fingerprint; - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, true, - fingerprint); + SummaryDocument entry = this.documentStore.retrieve( + SummaryDocument.class, true, fingerprint); if (entry != null) { if (entry.isRelay()) { this.filteredRelays.put(fingerprint, entry); @@ -409,7 +407,8 @@ public class RequestHandler { this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays); } - private void filterNodesByDays(Map<String, NodeStatus> filteredNodes, + private void filterNodesByDays( + Map<String, SummaryDocument> filteredNodes, SortedMap<Integer, Set<String>> nodesByDays, int[] days) { Set<String> removeNodes = new HashSet<String>(); for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { @@ -483,14 +482,14 @@ public class RequestHandler { this.orderedRelays.add(this.filteredRelays.remove(relay)); } } - Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>( + Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( this.filteredBridges.values()); this.orderedBridges.addAll(uniqueBridges); } else { - Set<NodeStatus> uniqueRelays = new HashSet<NodeStatus>( + Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>( this.filteredRelays.values()); this.orderedRelays.addAll(uniqueRelays); - Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>( + Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( this.filteredBridges.values()); this.orderedBridges.addAll(uniqueBridges); } @@ -528,13 +527,15 @@ public class RequestHandler { } } - private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>(); - public List<NodeStatus> getOrderedRelays() { + private List<SummaryDocument> orderedRelays = + new ArrayList<SummaryDocument>(); + public List<SummaryDocument> getOrderedRelays() { return this.orderedRelays; } - private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>(); - public List<NodeStatus> getOrderedBridges() { + private List<SummaryDocument> orderedBridges = + new ArrayList<SummaryDocument>(); + public List<SummaryDocument> getOrderedBridges() { return this.orderedBridges; } diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index 77c4b6c..2086ea8 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -32,13 +32,15 @@ public class ResponseBuilder { this.bridgesPublishedString = bridgesPublishedString; } - private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>(); - public void setOrderedRelays(List<NodeStatus> orderedRelays) { + private List<SummaryDocument> orderedRelays = + new ArrayList<SummaryDocument>(); + public void setOrderedRelays(List<SummaryDocument> orderedRelays) { this.orderedRelays = orderedRelays; } - private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>(); - public void setOrderedBridges(List<NodeStatus> orderedBridges) { + private List<SummaryDocument> orderedBridges = + new ArrayList<SummaryDocument>(); + public void setOrderedBridges(List<SummaryDocument> orderedBridges) { this.orderedBridges = orderedBridges; } @@ -53,11 +55,11 @@ public class ResponseBuilder { writeBridges(orderedBridges, pw); } - private void writeRelays(List<NodeStatus> relays, PrintWriter pw) { + private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) { pw.write("{\"relays_published\":\"" + relaysPublishedString + "\",\n\"relays\":["); int written = 0; - for (NodeStatus entry : relays) { + for (SummaryDocument entry : relays) { String lines = this.formatNodeStatus(entry); if (lines.length() > 0) { pw.print((written++ > 0 ? ",\n" : "\n") + lines); @@ -66,11 +68,12 @@ public class ResponseBuilder { pw.print("\n],\n"); } - private void writeBridges(List<NodeStatus> bridges, PrintWriter pw) { + private void writeBridges(List<SummaryDocument> bridges, + PrintWriter pw) { pw.write("\"bridges_published\":\"" + bridgesPublishedString + "\",\n\"bridges\":["); int written = 0; - for (NodeStatus entry : bridges) { + for (SummaryDocument entry : bridges) { String lines = this.formatNodeStatus(entry); if (lines.length() > 0) { pw.print((written++ > 0 ? ",\n" : "\n") + lines); @@ -79,7 +82,7 @@ public class ResponseBuilder { pw.print("\n]}\n"); } - private String formatNodeStatus(NodeStatus entry) { + private String formatNodeStatus(SummaryDocument entry) { if (this.resourceType == null) { return ""; } else if (this.resourceType.equals("summary")) { @@ -99,26 +102,17 @@ public class ResponseBuilder { } } - private String writeSummaryLine(NodeStatus entry) { + private String writeSummaryLine(SummaryDocument entry) { return entry.isRelay() ? writeRelaySummaryLine(entry) : writeBridgeSummaryLine(entry); } - private String writeRelaySummaryLine(NodeStatus entry) { + private String writeRelaySummaryLine(SummaryDocument entry) { String nickname = !entry.getNickname().equals("Unnamed") ? entry.getNickname() : null; String fingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress); - } - for (String exitAddress : entry.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } + String running = entry.isRunning() ? "true" : "false"; + List<String> addresses = entry.getAddresses(); StringBuilder addressesBuilder = new StringBuilder(); int written = 0; for (String address : addresses) { @@ -130,17 +124,17 @@ public class ResponseBuilder { fingerprint, addressesBuilder.toString(), running); } - private String writeBridgeSummaryLine(NodeStatus entry) { + private String writeBridgeSummaryLine(SummaryDocument entry) { String nickname = !entry.getNickname().equals("Unnamed") ? entry.getNickname() : null; String hashedFingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; + String running = entry.isRunning() ? "true" : "false"; return String.format("{%s\"h\":\"%s\",\"r\":%s}", (nickname == null ? "" : "\"n\":\"" + nickname + "\","), hashedFingerprint, running); } - private String writeDetailsLines(NodeStatus entry) { + private String writeDetailsLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); if (this.fields != null) { /* TODO Maybe there's a more elegant way (more maintainable, more @@ -267,7 +261,7 @@ public class ResponseBuilder { } } - private String writeBandwidthLines(NodeStatus entry) { + private String writeBandwidthLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); BandwidthDocument bandwidthDocument = this.documentStore.retrieve( BandwidthDocument.class, false, fingerprint); @@ -279,7 +273,7 @@ public class ResponseBuilder { } } - private String writeWeightsLines(NodeStatus entry) { + private String writeWeightsLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); WeightsDocument weightsDocument = this.documentStore.retrieve( WeightsDocument.class, false, fingerprint); @@ -291,7 +285,7 @@ public class ResponseBuilder { } } - private String writeClientsLines(NodeStatus entry) { + private String writeClientsLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); ClientsDocument clientsDocument = this.documentStore.retrieve( ClientsDocument.class, false, fingerprint); @@ -303,7 +297,7 @@ public class ResponseBuilder { } } - private String writeUptimeLines(NodeStatus entry) { + private String writeUptimeLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); UptimeDocument uptimeDocument = this.documentStore.retrieve( UptimeDocument.class, false, fingerprint); diff --git a/src/org/torproject/onionoo/SummaryDocument.java b/src/org/torproject/onionoo/SummaryDocument.java index 63862a2..3c0f700 100644 --- a/src/org/torproject/onionoo/SummaryDocument.java +++ b/src/org/torproject/onionoo/SummaryDocument.java @@ -1,8 +1,201 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + class SummaryDocument extends Document { + private boolean t; + public void setRelay(boolean isRelay) { + this.t = isRelay; + } + public boolean isRelay() { + return this.t; + } + + private String f; + public void setFingerprint(String fingerprint) { + if (fingerprint != null) { + Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); + if (!fingerprintPattern.matcher(fingerprint).matches()) { + throw new IllegalArgumentException("Fingerprint '" + fingerprint + + "' is not a valid fingerprint."); + } + } + this.f = fingerprint; + } + public String getFingerprint() { + return this.f; + } + + public String getHashedFingerprint() { + String hashedFingerprint = null; + if (this.f != null) { + try { + hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + this.f.toCharArray())).toUpperCase(); + } catch (DecoderException e) { + /* Format tested in setFingerprint(). */ + } + } + return hashedFingerprint; + } + + private String n; + public void setNickname(String nickname) { + if (nickname == null || nickname.equals("Unnamed")) { + this.n = null; + } else { + this.n = nickname; + } + } + public String getNickname() { + return this.n == null ? "Unnamed" : this.n; + } + + private String[] ad; + public void setAddresses(List<String> addresses) { + this.ad = this.collectionToStringArray(addresses); + } + public List<String> getAddresses() { + return this.stringArrayToList(this.ad); + } + + private String[] collectionToStringArray( + Collection<String> collection) { + String[] stringArray = null; + if (collection != null && !collection.isEmpty()) { + stringArray = new String[collection.size()]; + int i = 0; + for (String string : collection) { + stringArray[i++] = string; + } + } + return stringArray; + } + private List<String> stringArrayToList(String[] stringArray) { + List<String> list; + if (stringArray == null) { + list = new ArrayList<String>(); + } else { + list = Arrays.asList(stringArray); + } + return list; + } + private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { + SortedSet<String> sortedSet = new TreeSet<String>(); + if (stringArray != null) { + sortedSet.addAll(Arrays.asList(stringArray)); + } + return sortedSet; + } + + private String cc; + public void setCountryCode(String countryCode) { + this.cc = countryCode; + } + public String getCountryCode() { + return this.cc; + } + + private String as; + public void setASNumber(String aSNumber) { + this.as = aSNumber; + } + public String getASNumber() { + return this.as; + } + + private String fs; + public void setFirstSeenMillis(long firstSeenMillis) { + this.fs = DateTimeHelper.format(firstSeenMillis); + } + public long getFirstSeenMillis() { + return DateTimeHelper.parse(this.fs); + } + + private String ls; + public void setLastSeenMillis(long lastSeenMillis) { + this.ls = DateTimeHelper.format(lastSeenMillis); + } + public long getLastSeenMillis() { + return DateTimeHelper.parse(this.ls); + } + + private String[] rf; + public void setRelayFlags(SortedSet<String> relayFlags) { + this.rf = this.collectionToStringArray(relayFlags); + } + public SortedSet<String> getRelayFlags() { + return this.stringArrayToSortedSet(this.rf); + } + + private long cw; + public void setConsensusWeight(long consensusWeight) { + this.cw = consensusWeight; + } + public long getConsensusWeight() { + return this.cw; + } + + private boolean r; + public void setRunning(boolean isRunning) { + this.r = isRunning; + } + public boolean isRunning() { + return this.r; + } + + private String c; + public void setContact(String contact) { + if (contact != null && contact.length() == 0) { + this.c = null; + } else { + this.c = contact; + } + } + public String getContact() { + return this.c; + } + + private String[] ff; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.ff = this.collectionToStringArray(familyFingerprints); + } + public SortedSet<String> getFamilyFingerprints() { + return this.stringArrayToSortedSet(this.ff); + } + + public SummaryDocument(boolean isRelay, String nickname, + String fingerprint, List<String> addresses, long lastSeenMillis, + boolean running, SortedSet<String> relayFlags, long consensusWeight, + String countryCode, long firstSeenMillis, String aSNumber, + String contact, SortedSet<String> familyFingerprints) { + this.setRelay(isRelay); + this.setNickname(nickname); + this.setFingerprint(fingerprint); + this.setAddresses(addresses); + this.setLastSeenMillis(lastSeenMillis); + this.setRunning(running); + this.setRelayFlags(relayFlags); + this.setConsensusWeight(consensusWeight); + this.setCountryCode(countryCode); + this.setFirstSeenMillis(firstSeenMillis); + this.setASNumber(aSNumber); + this.setContact(contact); + this.setFamilyFingerprints(familyFingerprints); + } } diff --git a/src/org/torproject/onionoo/SummaryDocumentWriter.java b/src/org/torproject/onionoo/SummaryDocumentWriter.java new file mode 100644 index 0000000..7b257f5 --- /dev/null +++ b/src/org/torproject/onionoo/SummaryDocumentWriter.java @@ -0,0 +1,87 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.ArrayList; +import java.util.List; +import java.util.SortedSet; + +public class SummaryDocumentWriter implements DocumentWriter { + + private DocumentStore documentStore; + + public SummaryDocumentWriter() { + this.documentStore = ApplicationFactory.getDocumentStore(); + } + + private int writtenDocuments = 0, deletedDocuments = 0; + + public void writeDocuments() { + long maxLastSeenMillis = 0L; + for (String fingerprint : this.documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus != null && + nodeStatus.getLastSeenMillis() > maxLastSeenMillis) { + maxLastSeenMillis = nodeStatus.getLastSeenMillis(); + } + } + long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; + for (String fingerprint : this.documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, + true, fingerprint); + if (nodeStatus == null) { + continue; + } + if (nodeStatus.getLastSeenMillis() < cutoff) { + if (this.documentStore.remove(SummaryDocument.class, + fingerprint)) { + this.deletedDocuments++; + } + continue; + } + boolean isRelay = nodeStatus.isRelay(); + String nickname = nodeStatus.getNickname(); + List<String> addresses = new ArrayList<String>(); + addresses.add(nodeStatus.getAddress()); + for (String orAddress : nodeStatus.getOrAddresses()) { + if (!addresses.contains(orAddress)) { + addresses.add(orAddress); + } + } + for (String exitAddress : nodeStatus.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + long lastSeenMillis = nodeStatus.getLastSeenMillis(); + boolean running = nodeStatus.getRunning(); + SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + long consensusWeight = nodeStatus.getConsensusWeight(); + String countryCode = nodeStatus.getCountryCode(); + long firstSeenMillis = nodeStatus.getFirstSeenMillis(); + String aSNumber = nodeStatus.getASNumber(); + String contact = nodeStatus.getContact(); + SortedSet<String> familyFingerprints = + nodeStatus.getFamilyFingerprints(); + SummaryDocument summaryDocument = new SummaryDocument(isRelay, + nickname, fingerprint, addresses, lastSeenMillis, running, + relayFlags, consensusWeight, countryCode, firstSeenMillis, + aSNumber, contact, familyFingerprints); + if (this.documentStore.store(summaryDocument, fingerprint)) { + this.writtenDocuments++; + }; + } + Logger.printStatusTime("Wrote summary document files"); + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " summary document files written\n"); + sb.append(" " + Logger.formatDecimalNumber(this.deletedDocuments) + + " summary document files deleted\n"); + return sb.toString(); + } +} diff --git a/test/org/torproject/onionoo/DummyDocumentStore.java b/test/org/torproject/onionoo/DummyDocumentStore.java index 6969b8f..5a5905b 100644 --- a/test/org/torproject/onionoo/DummyDocumentStore.java +++ b/test/org/torproject/onionoo/DummyDocumentStore.java @@ -51,7 +51,7 @@ public class DummyDocumentStore extends DocumentStore { } public <T extends Document> SortedSet<String> list( - Class<T> documentType, boolean includeArchive) { + Class<T> documentType) { this.performedListOperations++; return new TreeSet<String>(this.getStoredDocumentsByClass( documentType).keySet()); diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index f8d3281..e482f82 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -31,7 +31,8 @@ import com.google.gson.Gson; * which tests servlet specifics. */ public class ResourceServletTest { - private SortedMap<String, NodeStatus> relays, bridges; + private SortedMap<String, org.torproject.onionoo.SummaryDocument> + relays, bridges; private long currentTimeMillis = DateTimeHelper.parse( "2013-04-24 12:22:22"); @@ -100,77 +101,73 @@ public class ResourceServletTest { @Before public void createSampleRelaysAndBridges() { - NodeStatus relayTorkaZ = new NodeStatus(true, "TorkaZ", - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", "62.216.201.221", - new TreeSet<String>(), new TreeSet<String>(Arrays.asList( - new String[] { "62.216.201.222", "62.216.201.223" })), - DateTimeHelper.parse("2013-04-19 05:00:00"), 9001, 0, - new TreeSet<String>(Arrays.asList(new String[] { "Running", - "Valid" })), 20L, "de", null, -1L, "reject", "1-65535", - DateTimeHelper.parse("2013-04-18 05:00:00"), - DateTimeHelper.parse("2013-04-19 05:00:00"), "AS8767", + org.torproject.onionoo.SummaryDocument relayTorkaZ = + new org.torproject.onionoo.SummaryDocument(true, "TorkaZ", + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList( + new String[] { "62.216.201.221", "62.216.201.222", + "62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"), + false, new TreeSet<String>(Arrays.asList(new String[] { "Running", + "Valid" })), 20L, "de", + DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767", "torkaz <klaus dot zufall at gmx dot de> " - + "<fb-token:np5_g_83jmf=>", null, new TreeSet<String>( - Arrays.asList(new String[] { - "001C13B3A55A71B977CA65EC85539D79C653A3FC", + + "<fb-token:np5_g_83jmf=>", new TreeSet<String>(Arrays.asList( + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" }))); - NodeStatus relayFerrari458 = new NodeStatus(true, "Ferrari458", - "001C13B3A55A71B977CA65EC85539D79C653A3FC", "68.38.171.200", - new TreeSet<String>(Arrays.asList(new String[] { - "[2001:4f8:3:2e::51]:9001" })), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-24 12:00:00"), 9001, 9030, + org.torproject.onionoo.SummaryDocument relayFerrari458 = + new org.torproject.onionoo.SummaryDocument(true, "Ferrari458", + "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( + new String[] { "68.38.171.200", "[2001:4f8:3:2e::51]" }), + DateTimeHelper.parse("2013-04-24 12:00:00"), true, new TreeSet<String>(Arrays.asList(new String[] { "Fast", "Named", "Running", "V2Dir", "Valid" })), 1140L, "us", - "c-68-38-171-200.hsd1.pa.comcast.net", 1366805763009L, "reject", - "1-65535", DateTimeHelper.parse("2013-02-12 16:00:00"), - DateTimeHelper.parse("2013-02-26 18:00:00"), "AS7922", null, null, + DateTimeHelper.parse("2013-02-12 16:00:00"), "AS7922", null, new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" }))); - relayFerrari458.setRunning(true); - NodeStatus relayTimMayTribute = new NodeStatus(true, "TimMayTribute", - "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", "89.69.68.246", - new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-22 20:00:00"), 9001, 9030, + org.torproject.onionoo.SummaryDocument relayTimMayTribute = + new org.torproject.onionoo.SummaryDocument(true, "TimMayTribute", + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", Arrays.asList( + new String[] { "89.69.68.246" }), + DateTimeHelper.parse("2013-04-22 20:00:00"), false, new TreeSet<String>(Arrays.asList(new String[] { "Fast", - "Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1", null, -1L, - "reject", "1-65535", DateTimeHelper.parse("2013-04-16 18:00:00"), + "Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1", DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024D/51E2A1C7 steven j. murdoch " + "<tor+steven.murdoch@cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", - null, new TreeSet<String>()); - NodeStatus bridgeec2bridgercc7f31fe = new NodeStatus(false, + new TreeSet<String>()); + org.torproject.onionoo.SummaryDocument bridgeec2bridgercc7f31fe = + new org.torproject.onionoo.SummaryDocument(false, "ec2bridgercc7f31fe", "0000831B236DFF73D409AD17B40E2A728A53994F", - "10.199.7.176", new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-21 18:07:03"), 443, 0, + Arrays.asList(new String[] { "10.199.7.176" }), + DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L, - "??", null, -1L, null, null, - DateTimeHelper.parse("2013-04-20 15:37:04"), -1L, null, null, - null, null); - NodeStatus bridgeUnnamed = new NodeStatus(false, "Unnamed", - "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", "10.0.52.84", - new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-20 17:37:04"), 443, 0, + null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, + null); + org.torproject.onionoo.SummaryDocument bridgeUnnamed = + new org.torproject.onionoo.SummaryDocument(false, "Unnamed", + "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", Arrays.asList( + new String[] { "10.0.52.84" }), + DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L, - "??", null, -1L, null, null, - DateTimeHelper.parse("2013-04-14 07:07:05"), -1L, null, null, - null, null); - NodeStatus bridgegummy = new NodeStatus(false, "gummy", - "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", "10.63.169.98", - new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-24 01:07:04"), 9001, 0, + null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, + null); + org.torproject.onionoo.SummaryDocument bridgegummy = + new org.torproject.onionoo.SummaryDocument(false, "gummy", + "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", Arrays.asList( + new String[] { "10.63.169.98" }), + DateTimeHelper.parse("2013-04-24 01:07:04"), true, new TreeSet<String>(Arrays.asList(new String[] { "Running", - "Valid" })), -1L, "??", null, -1L, null, null, - DateTimeHelper.parse("2013-01-16 21:07:04"), -1L, null, null, - null, null); - bridgegummy.setRunning(true); - this.relays = new TreeMap<String, NodeStatus>(); + "Valid" })), -1L, null, + DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null); + this.relays = + new TreeMap<String, org.torproject.onionoo.SummaryDocument>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", relayFerrari458); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); - this.bridges = new TreeMap<String, NodeStatus>(); + this.bridges = + new TreeMap<String, org.torproject.onionoo.SummaryDocument>(); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", @@ -203,10 +200,12 @@ public class ResourceServletTest { updateStatus.setDocumentString(String.valueOf( this.currentTimeMillis)); documentStore.addDocument(updateStatus, null); - for (Map.Entry<String, NodeStatus> e : relays.entrySet()) { + for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e : + this.relays.entrySet()) { documentStore.addDocument(e.getValue(), e.getKey()); } - for (Map.Entry<String, NodeStatus> e : bridges.entrySet()) { + for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e : + this.bridges.entrySet()) { documentStore.addDocument(e.getValue(), e.getKey()); } ApplicationFactory.setDocumentStore(documentStore);