commit 99d9f094722b1f0f1b0ba1b51ff53e942a06a263 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Oct 28 14:32:28 2014 +0100 Add new method to list recently stored documents. This is a rather big refactoring of how document writers obtain a list of documents they should write. The old way was to register a "fingerprint listener" at the descriptor source and write documents of all relays and bridges whose fingerprints it they learned about. One drawback (or even bug; cf. #12651) was that document writers didn't learn about relays or bridges that went offline, because there's no descriptor for that. The new way is for document writers to ask for a list of status files that have changed since the last "update status" file was written. This can be implemented for documents cached in memory and for documents stored on disk. Document writers can then use this list to only write the documents that need updating. --- .../org/torproject/onionoo/docs/DocumentStore.java | 85 +++++++++++++------- .../org/torproject/onionoo/docs/UpdateStatus.java | 29 ++++++- .../org/torproject/onionoo/server/NodeIndexer.java | 12 +-- .../onionoo/updater/DescriptorSource.java | 77 +----------------- .../onionoo/updater/FingerprintListener.java | 10 --- .../onionoo/writer/BandwidthDocumentWriter.java | 36 +++------ .../onionoo/writer/ClientsDocumentWriter.java | 35 +++----- .../onionoo/writer/DetailsDocumentWriter.java | 64 +++++++-------- .../onionoo/writer/UptimeDocumentWriter.java | 57 ++++--------- .../onionoo/writer/WeightsDocumentWriter.java | 43 +++------- .../torproject/onionoo/DummyDescriptorSource.java | 47 ----------- .../org/torproject/onionoo/DummyDocumentStore.java | 11 ++- .../torproject/onionoo/ResourceServletTest.java | 3 +- .../onionoo/UptimeDocumentWriterTest.java | 3 - 14 files changed, 174 insertions(+), 338 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 23ac03f..5137fb4 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -66,22 +66,41 @@ public class DocumentStore { private SortedMap<String, NodeStatus> cachedNodeStatuses; private SortedMap<String, SummaryDocument> cachedSummaryDocuments; + /* Last-modified timestamp of cached network statuses and summary + * documents when reading them from disk. */ + private long lastModifiedNodeStatuses = 0L; + private long lastModifiedSummaryDocuments = 0L; + + /* Fingerprints of updated node statuses and summary documents that are + * not yet written to disk. */ + private SortedSet<String> updatedNodeStatuses; + private SortedSet<String> updatedSummaryDocuments; + public <T extends Document> SortedSet<String> list( Class<T> documentType) { + return this.list(documentType, 0L); + } + + public <T extends Document> SortedSet<String> list( + Class<T> documentType, long updatedAfter) { if (documentType.equals(NodeStatus.class)) { - return this.listNodeStatuses(); + return this.listNodeStatuses(updatedAfter); } else if (documentType.equals(SummaryDocument.class)) { - return this.listSummaryDocuments(); + return this.listSummaryDocuments(updatedAfter); } else { - return this.listDocumentFiles(documentType); + return this.listDocumentFiles(documentType, updatedAfter); } } - private SortedSet<String> listNodeStatuses() { + private SortedSet<String> listNodeStatuses(long updatedAfter) { if (this.cachedNodeStatuses == null) { this.cacheNodeStatuses(); } - return new TreeSet<String>(this.cachedNodeStatuses.keySet()); + if (updatedAfter >= this.lastModifiedNodeStatuses) { + return new TreeSet<String>(this.updatedNodeStatuses); + } else { + return new TreeSet<String>(this.cachedNodeStatuses.keySet()); + } } private void cacheNodeStatuses() { @@ -105,6 +124,7 @@ public class DocumentStore { } } br.close(); + this.lastModifiedNodeStatuses = summaryFile.lastModified(); this.listedFiles += parsedNodeStatuses.size(); this.listOperations++; } catch (IOException e) { @@ -114,13 +134,18 @@ public class DocumentStore { } } this.cachedNodeStatuses = parsedNodeStatuses; + this.updatedNodeStatuses = new TreeSet<String>(); } - private SortedSet<String> listSummaryDocuments() { + private SortedSet<String> listSummaryDocuments(long updatedAfter) { if (this.cachedSummaryDocuments == null) { this.cacheSummaryDocuments(); } - return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + if (updatedAfter >= this.lastModifiedSummaryDocuments) { + return new TreeSet<String>(this.updatedSummaryDocuments); + } else { + return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + } } private void cacheSummaryDocuments() { @@ -147,6 +172,7 @@ public class DocumentStore { } } br.close(); + this.lastModifiedSummaryDocuments = summaryFile.lastModified(); this.listedFiles += parsedSummaryDocuments.size(); this.listOperations++; } catch (IOException e) { @@ -159,10 +185,11 @@ public class DocumentStore { } } this.cachedSummaryDocuments = parsedSummaryDocuments; + this.updatedSummaryDocuments = new TreeSet<String>(); } private <T extends Document> SortedSet<String> listDocumentFiles( - Class<T> documentType) { + Class<T> documentType, long updatedAfter) { SortedSet<String> fingerprints = new TreeSet<String>(); File directory = null; String subdirectory = null; @@ -204,8 +231,9 @@ public class DocumentStore { File file = files.pop(); if (file.isDirectory()) { files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().length() == 40) { - fingerprints.add(file.getName()); + } else if (file.getName().length() == 40 && + (updatedAfter == 0L || file.lastModified() > updatedAfter)) { + fingerprints.add(file.getName()); } } } @@ -235,6 +263,7 @@ public class DocumentStore { if (this.cachedNodeStatuses == null) { this.cacheNodeStatuses(); } + this.updatedNodeStatuses.add(fingerprint); this.cachedNodeStatuses.put(fingerprint, nodeStatus); return true; } @@ -244,6 +273,7 @@ public class DocumentStore { if (this.cachedSummaryDocuments == null) { this.cacheSummaryDocuments(); } + this.updatedSummaryDocuments.add(fingerprint); this.cachedSummaryDocuments.put(fingerprint, summaryDocument); return true; } @@ -289,7 +319,8 @@ public class DocumentStore { } else if (document instanceof BandwidthStatus || document instanceof WeightsStatus || document instanceof ClientsStatus || - document instanceof UptimeStatus) { + document instanceof UptimeStatus || + document instanceof UpdateStatus) { documentString = document.toDocumentString(); } else { log.error("Serializing is not supported for type " @@ -435,7 +466,8 @@ public class DocumentStore { } else if (documentType.equals(BandwidthStatus.class) || documentType.equals(WeightsStatus.class) || documentType.equals(ClientsStatus.class) || - documentType.equals(UptimeStatus.class)) { + documentType.equals(UptimeStatus.class) || + documentType.equals(UpdateStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); } else if (documentType.equals(DetailsStatus.class)) { return this.retrieveParsedDocumentFile(documentType, "{" @@ -523,6 +555,7 @@ public class DocumentStore { if (this.cachedNodeStatuses == null) { this.cacheNodeStatuses(); } + this.updatedNodeStatuses.remove(fingerprint); return this.cachedNodeStatuses.remove(fingerprint) != null; } @@ -530,6 +563,7 @@ public class DocumentStore { if (this.cachedSummaryDocuments == null) { this.cacheSummaryDocuments(); } + this.updatedSummaryDocuments.remove(fingerprint); return this.cachedSummaryDocuments.remove(fingerprint) != null; } @@ -550,9 +584,6 @@ public class DocumentStore { File documentFile = null; if (fingerprint == null && !documentType.equals(UpdateStatus.class) && !documentType.equals(UptimeStatus.class)) { - // TODO Instead of using the update file workaround, add new method - // lastModified(Class<T> documentType) that serves a similar - // purpose. return null; } File directory = null; @@ -631,6 +662,10 @@ public class DocumentStore { public void invalidateDocumentCache() { this.cachedNodeStatuses = null; this.cachedSummaryDocuments = null; + this.lastModifiedNodeStatuses = 0L; + this.lastModifiedSummaryDocuments = 0L; + this.updatedNodeStatuses = null; + this.updatedSummaryDocuments = null; } private void writeNodeStatuses() { @@ -675,6 +710,8 @@ public class DocumentStore { BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); bw.write(documentString); bw.close(); + this.lastModifiedNodeStatuses = summaryFile.lastModified(); + this.updatedNodeStatuses.clear(); this.storedFiles++; this.storedBytes += documentString.length(); } catch (IOException e) { @@ -703,6 +740,8 @@ public class DocumentStore { BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); bw.write(documentString); bw.close(); + this.lastModifiedSummaryDocuments = summaryFile.lastModified(); + this.updatedSummaryDocuments.clear(); this.storedFiles++; this.storedBytes += documentString.length(); } catch (IOException e) { @@ -715,19 +754,9 @@ public class DocumentStore { if (this.outDir == null) { return; } - File updateFile = new File(this.outDir, "update"); - String documentString = String.valueOf(this.time.currentTimeMillis()); - try { - updateFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile)); - bw.write(documentString); - bw.close(); - this.storedFiles++; - this.storedBytes += documentString.length(); - } catch (IOException e) { - log.error("Could not write file '" - + updateFile.getAbsolutePath() + "'.", e); - } + UpdateStatus updateStatus = new UpdateStatus(); + updateStatus.setUpdatedMillis(this.time.currentTimeMillis()); + this.store(updateStatus); } public String getStatsString() { diff --git a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java index 7bd710b..efdcf24 100644 --- a/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UpdateStatus.java @@ -1,7 +1,34 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013, 2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class UpdateStatus extends Document { + + private static Logger log = LoggerFactory.getLogger(UpdateStatus.class); + + private long updatedMillis; + public void setUpdatedMillis(long updatedMillis) { + this.updatedMillis = updatedMillis; + } + public long getUpdatedMillis() { + return this.updatedMillis; + } + + public void setFromDocumentString(String documentString) { + try { + this.updatedMillis = Long.parseLong(documentString.trim()); + } catch (NumberFormatException e) { + log.error("Could not parse timestamp '" + documentString + "'. " + + "Setting to 1970-01-01 00:00:00."); + this.updatedMillis = 0L; + } + } + + public String toDocumentString() { + return String.valueOf(this.updatedMillis); + } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index bafd442..f8870db 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -99,15 +99,9 @@ public class NodeIndexer implements ServletContextListener, Runnable { long updateStatusMillis = -1L; DocumentStore documentStore = DocumentStoreFactory.getDocumentStore(); UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, - false); - if (updateStatus != null && - updateStatus.getDocumentString() != null) { - String updateString = updateStatus.getDocumentString(); - try { - updateStatusMillis = Long.parseLong(updateString.trim()); - } catch (NumberFormatException e) { - /* Handle below. */ - } + true); + if (updateStatus != null) { + updateStatusMillis = updateStatus.getUpdatedMillis(); } synchronized (this) { if (updateStatusMillis <= this.lastIndexed) { diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 30001f8..26b7b10 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -9,19 +9,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.ExtraInfoDescriptor; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; import org.torproject.onionoo.util.FormattingUtils; public class DescriptorSource { @@ -39,8 +30,6 @@ public class DescriptorSource { this.descriptorQueues = new ArrayList<DescriptorQueue>(); this.descriptorListeners = new HashMap<DescriptorType, Set<DescriptorListener>>(); - this.fingerprintListeners = - new HashMap<DescriptorType, Set<FingerprintListener>>(); } private DescriptorQueue getDescriptorQueue( @@ -59,9 +48,6 @@ public class DescriptorSource { private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners; - private Map<DescriptorType, Set<FingerprintListener>> - fingerprintListeners; - public void registerDescriptorListener(DescriptorListener listener, DescriptorType descriptorType) { if (!this.descriptorListeners.containsKey(descriptorType)) { @@ -71,15 +57,6 @@ public class DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } - public void registerFingerprintListener(FingerprintListener listener, - DescriptorType descriptorType) { - if (!this.fingerprintListeners.containsKey(descriptorType)) { - this.fingerprintListeners.put(descriptorType, - new HashSet<FingerprintListener>()); - } - this.fingerprintListeners.get(descriptorType).add(listener); - } - public void downloadDescriptors() { for (DescriptorType descriptorType : DescriptorType.values()) { log.info("Loading: " + descriptorType); @@ -91,8 +68,7 @@ public class DescriptorSource { downloadedFiles = 0, deletedLocalFiles = 0; private void downloadDescriptors(DescriptorType descriptorType) { - if (!this.descriptorListeners.containsKey(descriptorType) && - !this.fingerprintListeners.containsKey(descriptorType)) { + if (!this.descriptorListeners.containsKey(descriptorType)) { return; } DescriptorDownloader descriptorDownloader = @@ -140,14 +116,11 @@ public class DescriptorSource { private void readDescriptors(DescriptorType descriptorType, DescriptorHistory descriptorHistory, boolean relay) { - if (!this.descriptorListeners.containsKey(descriptorType) && - !this.fingerprintListeners.containsKey(descriptorType)) { + if (!this.descriptorListeners.containsKey(descriptorType)) { return; } Set<DescriptorListener> descriptorListeners = this.descriptorListeners.get(descriptorType); - Set<FingerprintListener> fingerprintListeners = - this.fingerprintListeners.get(descriptorType); DescriptorQueue descriptorQueue = this.getDescriptorQueue( descriptorType, descriptorHistory); Descriptor descriptor; @@ -155,52 +128,6 @@ public class DescriptorSource { for (DescriptorListener descriptorListener : descriptorListeners) { descriptorListener.processDescriptor(descriptor, relay); } - if (fingerprintListeners == null) { - continue; - } - SortedSet<String> fingerprints = new TreeSet<String>(); - if (descriptorType == DescriptorType.RELAY_CONSENSUSES && - descriptor instanceof RelayNetworkStatusConsensus) { - fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor). - getStatusEntries().keySet()); - } else if (descriptorType - == DescriptorType.RELAY_SERVER_DESCRIPTORS && - descriptor instanceof ServerDescriptor) { - fingerprints.add(((ServerDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS && - descriptor instanceof ExtraInfoDescriptor) { - fingerprints.add(((ExtraInfoDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.EXIT_LISTS && - descriptor instanceof ExitList) { - for (ExitListEntry entry : - ((ExitList) descriptor).getExitListEntries()) { - fingerprints.add(entry.getFingerprint()); - } - } else if (descriptorType == DescriptorType.BRIDGE_STATUSES && - descriptor instanceof BridgeNetworkStatus) { - fingerprints.addAll(((BridgeNetworkStatus) descriptor). - getStatusEntries().keySet()); - } else if (descriptorType == - DescriptorType.BRIDGE_SERVER_DESCRIPTORS && - descriptor instanceof ServerDescriptor) { - fingerprints.add(((ServerDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS && - descriptor instanceof ExtraInfoDescriptor) { - fingerprints.add(((ExtraInfoDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == - DescriptorType.BRIDGE_POOL_ASSIGNMENTS && - descriptor instanceof BridgePoolAssignment) { - fingerprints.addAll(((BridgePoolAssignment) descriptor). - getEntries().keySet()); - } - for (FingerprintListener fingerprintListener : - fingerprintListeners) { - fingerprintListener.processFingerprints(fingerprints, relay); - } } switch (descriptorType) { case RELAY_CONSENSUSES: diff --git a/src/main/java/org/torproject/onionoo/updater/FingerprintListener.java b/src/main/java/org/torproject/onionoo/updater/FingerprintListener.java deleted file mode 100644 index 5e16eae..0000000 --- a/src/main/java/org/torproject/onionoo/updater/FingerprintListener.java +++ /dev/null @@ -1,10 +0,0 @@ -/* Copyright 2013, 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo.updater; - -import java.util.SortedSet; - -public interface FingerprintListener { - abstract void processFingerprints(SortedSet<String> fingerprints, - boolean relay); -} \ No newline at end of file diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index c5167d0..9bad965 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -3,11 +3,9 @@ package org.torproject.onionoo.writer; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -19,47 +17,31 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.TimeFactory; -public class BandwidthDocumentWriter implements FingerprintListener, - DocumentWriter{ +public class BandwidthDocumentWriter implements DocumentWriter { private static final Logger log = LoggerFactory.getLogger( BandwidthDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public BandwidthDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private Set<String> updateBandwidthDocuments = new HashSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - this.updateBandwidthDocuments.addAll(fingerprints); } public void writeDocuments() { - for (String fingerprint : this.updateBandwidthDocuments) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updateBandwidthDocuments = this.documentStore.list( + BandwidthStatus.class, updatedMillis); + for (String fingerprint : updateBandwidthDocuments) { BandwidthStatus bandwidthStatus = this.documentStore.retrieve( BandwidthStatus.class, true, fingerprint); if (bandwidthStatus == null) { diff --git a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java index 4086bc7..885494c 100644 --- a/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -9,7 +9,6 @@ import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; -import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,10 +19,7 @@ import org.torproject.onionoo.docs.ClientsStatus; import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.FormattingUtils; import org.torproject.onionoo.util.TimeFactory; @@ -50,43 +46,30 @@ import org.torproject.onionoo.util.TimeFactory; * "transports":{"obfs2":0.4581}, * "versions":{"v4":1.0000}} */ -public class ClientsDocumentWriter implements FingerprintListener, - DocumentWriter { +public class ClientsDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( ClientsDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public ClientsDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private SortedSet<String> updateDocuments = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (!relay) { - this.updateDocuments.addAll(fingerprints); - } } private int writtenDocuments = 0; public void writeDocuments() { - for (String hashedFingerprint : this.updateDocuments) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updateDocuments = this.documentStore.list( + ClientsStatus.class, updatedMillis); + for (String hashedFingerprint : updateDocuments) { ClientsStatus clientsStatus = this.documentStore.retrieve( ClientsStatus.class, true, hashedFingerprint); if (clientsStatus == null) { diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 47b7c79..0692070 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -16,59 +16,55 @@ import org.torproject.onionoo.docs.DetailsStatus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.NodeStatus; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.util.TimeFactory; -public class DetailsDocumentWriter implements FingerprintListener, - DocumentWriter { +public class DetailsDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( DetailsDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public DetailsDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.EXIT_LISTS); } private SortedSet<String> newRelays = new TreeSet<String>(), newBridges = new TreeSet<String>(); - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelays.addAll(fingerprints); - } else { - this.newBridges.addAll(fingerprints); - } - } - public void writeDocuments() { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updatedNodeStatuses = this.documentStore.list( + NodeStatus.class, updatedMillis); + for (String fingerprint : updatedNodeStatuses) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus.isRelay()) { + newRelays.add(fingerprint); + } else { + newBridges.add(fingerprint); + } + } + SortedSet<String> updatedDetailsStatuses = this.documentStore.list( + DetailsStatus.class, updatedMillis); + for (String fingerprint : updatedDetailsStatuses) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus == null) { + continue; + } else if (nodeStatus.isRelay()) { + newRelays.add(fingerprint); + } else { + newBridges.add(fingerprint); + } + } this.updateRelayDetailsFiles(); this.updateBridgeDetailsFiles(); log.info("Wrote details document files"); diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index 973fa67..093bc98 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -7,7 +7,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.SortedSet; -import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,52 +14,25 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeHistory; import org.torproject.onionoo.docs.UptimeStatus; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; import org.torproject.onionoo.util.FormattingUtils; import org.torproject.onionoo.util.TimeFactory; -public class UptimeDocumentWriter implements FingerprintListener, - DocumentWriter { +public class UptimeDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( UptimeDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public UptimeDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - } - - private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), - newBridgeFingerprints = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelayFingerprints.addAll(fingerprints); - } else { - this.newBridgeFingerprints.addAll(fingerprints); - } } public void writeDocuments() { @@ -69,29 +41,34 @@ public class UptimeDocumentWriter implements FingerprintListener, if (uptimeStatus == null) { return; } - for (String fingerprint : this.newRelayFingerprints) { - this.updateDocument(true, fingerprint, - uptimeStatus.getRelayHistory()); - } - for (String fingerprint : this.newBridgeFingerprints) { - this.updateDocument(false, fingerprint, - uptimeStatus.getBridgeHistory()); + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updatedUptimeStatuses = this.documentStore.list( + UptimeStatus.class, updatedMillis); + for (String fingerprint : updatedUptimeStatuses) { + this.updateDocument(fingerprint, uptimeStatus); } log.info("Wrote uptime document files"); } private int writtenDocuments = 0; - private void updateDocument(boolean relay, String fingerprint, - SortedSet<UptimeHistory> knownStatuses) { + private void updateDocument(String fingerprint, + UptimeStatus knownStatuses) { UptimeStatus uptimeStatus = this.documentStore.retrieve( UptimeStatus.class, true, fingerprint); if (uptimeStatus != null) { + boolean relay = uptimeStatus.getBridgeHistory().isEmpty(); SortedSet<UptimeHistory> history = relay ? uptimeStatus.getRelayHistory() : uptimeStatus.getBridgeHistory(); + SortedSet<UptimeHistory> knownStatusesHistory = relay + ? knownStatuses.getRelayHistory() + : knownStatuses.getBridgeHistory(); UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, - fingerprint, history, knownStatuses); + fingerprint, history, knownStatusesHistory); this.documentStore.store(uptimeDocument, fingerprint); this.writtenDocuments++; } diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index 00c529e..7d52a47 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -3,11 +3,9 @@ package org.torproject.onionoo.writer; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -17,56 +15,33 @@ import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.UpdateStatus; import org.torproject.onionoo.docs.WeightsDocument; import org.torproject.onionoo.docs.WeightsStatus; -import org.torproject.onionoo.updater.DescriptorSource; -import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; import org.torproject.onionoo.util.TimeFactory; -public class WeightsDocumentWriter implements FingerprintListener, - DocumentWriter { +public class WeightsDocumentWriter implements DocumentWriter { private final static Logger log = LoggerFactory.getLogger( WeightsDocumentWriter.class); - private DescriptorSource descriptorSource; - private DocumentStore documentStore; private long now; public WeightsDocumentWriter() { - this.descriptorSource = DescriptorSourceFactory.getDescriptorSource(); this.documentStore = DocumentStoreFactory.getDocumentStore(); this.now = TimeFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - } - - private Set<String> updateWeightsDocuments = new HashSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.updateWeightsDocuments.addAll(fingerprints); - } } public void writeDocuments() { - this.writeWeightsDataFiles(); - log.info("Wrote weights document files"); - } - - private void writeWeightsDataFiles() { - for (String fingerprint : this.updateWeightsDocuments) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, true); + long updatedMillis = updateStatus != null ? + updateStatus.getUpdatedMillis() : 0L; + SortedSet<String> updateWeightsDocuments = this.documentStore.list( + WeightsStatus.class, updatedMillis); + for (String fingerprint : updateWeightsDocuments) { WeightsStatus weightsStatus = this.documentStore.retrieve( WeightsStatus.class, true, fingerprint); if (weightsStatus == null) { diff --git a/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java b/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java index e93b063..4dbf066 100644 --- a/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java +++ b/src/test/java/org/torproject/onionoo/DummyDescriptorSource.java @@ -5,14 +5,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import org.torproject.descriptor.Descriptor; import org.torproject.onionoo.updater.DescriptorListener; import org.torproject.onionoo.updater.DescriptorSource; import org.torproject.onionoo.updater.DescriptorType; -import org.torproject.onionoo.updater.FingerprintListener; public class DummyDescriptorSource extends DescriptorSource { @@ -39,27 +36,6 @@ public class DummyDescriptorSource extends DescriptorSource { return this.descriptors.get(descriptorType); } - private Map<DescriptorType, SortedSet<String>> fingerprints = - new HashMap<DescriptorType, SortedSet<String>>(); - - public void addFingerprints(DescriptorType descriptorType, - Collection<String> fingerprints) { - this.getFingerprintsByType(descriptorType).addAll(fingerprints); - } - - public void addFingerprint(DescriptorType descriptorType, - String fingerprint) { - this.getFingerprintsByType(descriptorType).add(fingerprint); - } - - private SortedSet<String> getFingerprintsByType( - DescriptorType descriptorType) { - if (!this.fingerprints.containsKey(descriptorType)) { - this.fingerprints.put(descriptorType, new TreeSet<String>()); - } - return this.fingerprints.get(descriptorType); - } - private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners = new HashMap<DescriptorType, Set<DescriptorListener>>(); @@ -73,23 +49,9 @@ public class DummyDescriptorSource extends DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } - private Map<DescriptorType, Set<FingerprintListener>> - fingerprintListeners = new HashMap<DescriptorType, - Set<FingerprintListener>>(); - - public void registerFingerprintListener(FingerprintListener listener, - DescriptorType descriptorType) { - if (!this.fingerprintListeners.containsKey(descriptorType)) { - this.fingerprintListeners.put(descriptorType, - new HashSet<FingerprintListener>()); - } - this.fingerprintListeners.get(descriptorType).add(listener); - } - public void readDescriptors() { Set<DescriptorType> descriptorTypes = new HashSet<DescriptorType>(); descriptorTypes.addAll(this.descriptorListeners.keySet()); - descriptorTypes.addAll(this.fingerprintListeners.keySet()); for (DescriptorType descriptorType : descriptorTypes) { boolean relay; switch (descriptorType) { @@ -118,15 +80,6 @@ public class DummyDescriptorSource extends DescriptorSource { } } } - if (this.fingerprints.containsKey(descriptorType) && - this.fingerprintListeners.containsKey(descriptorType)) { - Set<FingerprintListener> listeners = - this.fingerprintListeners.get(descriptorType); - for (FingerprintListener listener : listeners) { - listener.processFingerprints(this.getFingerprintsByType( - descriptorType), relay); - } - } } } diff --git a/src/test/java/org/torproject/onionoo/DummyDocumentStore.java b/src/test/java/org/torproject/onionoo/DummyDocumentStore.java index 54311aa..f5601b0 100644 --- a/src/test/java/org/torproject/onionoo/DummyDocumentStore.java +++ b/src/test/java/org/torproject/onionoo/DummyDocumentStore.java @@ -54,10 +54,17 @@ public class DummyDocumentStore extends DocumentStore { } public <T extends Document> SortedSet<String> list( + Class<T> documentType, long modifiedAfter) { + return this.list(documentType); + } + + public <T extends Document> SortedSet<String> list( Class<T> documentType) { this.performedListOperations++; - return new TreeSet<String>(this.getStoredDocumentsByClass( - documentType).keySet()); + SortedSet<String> fingerprints = new TreeSet<String>( + this.getStoredDocumentsByClass(documentType).keySet()); + fingerprints.remove(FINGERPRINT_NULL); + return fingerprints; } private int performedRemoveOperations = 0; diff --git a/src/test/java/org/torproject/onionoo/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/ResourceServletTest.java index 1631661..cc48295 100644 --- a/src/test/java/org/torproject/onionoo/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/ResourceServletTest.java @@ -210,8 +210,7 @@ public class ResourceServletTest { private void createDummyDocumentStore() { DummyDocumentStore documentStore = new DummyDocumentStore(); UpdateStatus updateStatus = new UpdateStatus(); - updateStatus.setDocumentString(String.valueOf( - this.currentTimeMillis)); + updateStatus.setUpdatedMillis(this.currentTimeMillis); documentStore.addDocument(updateStatus, null); for (Map.Entry<String, org.torproject.onionoo.docs.SummaryDocument> e : this.relays.entrySet()) { diff --git a/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java index 9b19e79..8f7fff0 100644 --- a/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java +++ b/src/test/java/org/torproject/onionoo/UptimeDocumentWriterTest.java @@ -16,7 +16,6 @@ import org.torproject.onionoo.docs.GraphHistory; import org.torproject.onionoo.docs.UptimeDocument; import org.torproject.onionoo.docs.UptimeStatus; import org.torproject.onionoo.updater.DescriptorSourceFactory; -import org.torproject.onionoo.updater.DescriptorType; import org.torproject.onionoo.util.TimeFactory; import org.torproject.onionoo.writer.UptimeDocumentWriter; @@ -71,8 +70,6 @@ public class UptimeDocumentWriterTest { status = new UptimeStatus(); status.setFromDocumentString(gabelmooUptime); this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); - this.descriptorSource.addFingerprint(DescriptorType.RELAY_CONSENSUSES, - GABELMOO_FINGERPRINT); } private static final long ONE_SECOND = 1000L,