commit cc8c00720cdf2a624139c59d8297b40c866a30cd Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Sep 25 19:10:52 2013 +0200 Stop deleting details, bandwidth, and weights documents. So far, we deleted details, bandwidth, and weights documents when a relay or bridge went offline for at least 7 days. The reason was that we wouldn't return documents for these relays or bridges anyway, and once the relay or bridge comes back online, we'd write a new document from our status files. Very soon we'll want to support searching for relays and bridges that have been offline for more than 7 days, and we'll also want to serve their details, bandwidth, and weights documents. We need to stop deleting them. Related to this, we now write new details, bandwidth, and weights documents for all relays and bridges, regardless of whether they were running in the last 7 days. This step also removes some dependencies in the code which may allow us to parallelize more stuff in the future. --- .../torproject/onionoo/BandwidthDataWriter.java | 29 -------------- src/org/torproject/onionoo/DetailsDataWriter.java | 27 ++----------- src/org/torproject/onionoo/Main.java | 21 ++-------- src/org/torproject/onionoo/WeightsDataWriter.java | 41 +++++--------------- 4 files changed, 16 insertions(+), 102 deletions(-) diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java index 2e0b8a7..8a0ccdf 100644 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ b/src/org/torproject/onionoo/BandwidthDataWriter.java @@ -9,10 +9,8 @@ import java.util.List; import java.util.Locale; import java.util.Scanner; import java.util.SortedMap; -import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; -import java.util.TreeSet; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -42,8 +40,6 @@ public class BandwidthDataWriter implements DescriptorListener { private DocumentStore documentStore; - private SortedSet<String> currentFingerprints = new TreeSet<String>(); - public BandwidthDataWriter(DescriptorSource descriptorSource, DocumentStore documentStore) { this.descriptorSource = descriptorSource; @@ -71,11 +67,6 @@ public class BandwidthDataWriter implements DescriptorListener { } } - public void setCurrentNodes( - SortedMap<String, NodeStatus> currentNodes) { - this.currentFingerprints.addAll(currentNodes.keySet()); - } - private void parseDescriptor(ExtraInfoDescriptor descriptor) { String fingerprint = descriptor.getFingerprint(); boolean updateHistory = false; @@ -238,13 +229,6 @@ public class BandwidthDataWriter implements DescriptorListener { private void writeBandwidthDataFileToDisk(String fingerprint, SortedMap<Long, long[]> writeHistory, SortedMap<Long, long[]> readHistory) { - if ((writeHistory.isEmpty() || writeHistory.lastKey() < this.now - - 7L * 24L * 60L * 60L * 1000L) && - (readHistory.isEmpty() || readHistory.lastKey() < this.now - - 7L * 24L * 60L * 60L * 1000L)) { - /* Don't write bandwidth data file to disk. */ - return; - } String writeHistoryString = formatHistoryString(writeHistory); String readHistoryString = formatHistoryString(readHistory); StringBuilder sb = new StringBuilder(); @@ -372,18 +356,5 @@ public class BandwidthDataWriter implements DescriptorListener { } return result; } - - public void deleteObsoleteBandwidthFiles() { - SortedSet<String> obsoleteBandwidthFiles = this.documentStore.list( - BandwidthDocument.class, false); - for (String fingerprint : this.currentFingerprints) { - if (obsoleteBandwidthFiles.contains(fingerprint)) { - obsoleteBandwidthFiles.remove(fingerprint); - } - } - for (String fingerprint : obsoleteBandwidthFiles) { - this.documentStore.remove(BandwidthDocument.class, fingerprint); - } - } } diff --git a/src/org/torproject/onionoo/DetailsDataWriter.java b/src/org/torproject/onionoo/DetailsDataWriter.java index 51ca2e9..4954dfc 100644 --- a/src/org/torproject/onionoo/DetailsDataWriter.java +++ b/src/org/torproject/onionoo/DetailsDataWriter.java @@ -414,12 +414,8 @@ public class DetailsDataWriter implements DescriptorListener { } public void writeOutDetails() { - SortedSet<String> remainingDetailsFiles = new TreeSet<String>(); - remainingDetailsFiles.addAll(this.documentStore.list( - DetailsDocument.class, false)); - this.updateRelayDetailsFiles(remainingDetailsFiles); - this.updateBridgeDetailsFiles(remainingDetailsFiles); - this.deleteDetailsFiles(remainingDetailsFiles); + this.updateRelayDetailsFiles(); + this.updateBridgeDetailsFiles(); } private static String escapeJSON(String s) { @@ -430,17 +426,13 @@ public class DetailsDataWriter implements DescriptorListener { return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); } - private void updateRelayDetailsFiles( - SortedSet<String> remainingDetailsFiles) { + private void updateRelayDetailsFiles() { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (Map.Entry<String, NodeStatus> relay : this.relays.entrySet()) { String fingerprint = relay.getKey(); - /* Don't delete this details file later on. */ - remainingDetailsFiles.remove(fingerprint); - /* Generate network-status-specific part. */ NodeStatus entry = relay.getValue(); String nickname = entry.getNickname(); @@ -627,17 +619,13 @@ public class DetailsDataWriter implements DescriptorListener { } } - private void updateBridgeDetailsFiles( - SortedSet<String> remainingDetailsFiles) { + private void updateBridgeDetailsFiles() { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (Map.Entry<String, NodeStatus> bridge : this.bridges.entrySet()) { String fingerprint = bridge.getKey(); - /* Don't delete this details file later on. */ - remainingDetailsFiles.remove(fingerprint); - /* Generate network-status-specific part. */ NodeStatus entry = bridge.getValue(); String nickname = entry.getNickname(); @@ -691,12 +679,5 @@ public class DetailsDataWriter implements DescriptorListener { this.documentStore.store(detailsDocument, fingerprint); } } - - private void deleteDetailsFiles( - SortedSet<String> remainingDetailsFiles) { - for (String fingerprint : remainingDetailsFiles) { - this.documentStore.remove(DetailsDocument.class, fingerprint); - } - } } diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 9d9852b..e81a26a 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -90,29 +90,14 @@ public class Main { printStatusTime("Wrote detail data files"); printStatus("Updating bandwidth data."); - bdw.setCurrentNodes(currentNodes); - printStatusTime("Set current node fingerprints"); - // TODO Evaluate overhead of not deleting obsolete bandwidth files. - // An advantage would be that we don't need ndw's currentNodes - // anymore, which allows us to run ndw and bdw in parallel in the - // future. - bdw.deleteObsoleteBandwidthFiles(); - printStatusTime("Deleted obsolete bandwidth files"); printStatus("Updating weights data."); - wdw.setCurrentNodes(currentNodes); - printStatusTime("Set current node fingerprints"); wdw.updateWeightsHistories(); printStatusTime("Updated weights histories"); + wdw.updateWeightsStatuses(); + printStatusTime("Updated weights status files"); wdw.writeWeightsDataFiles(); - printStatusTime("Wrote weights data files"); - // TODO Evaluate overhead of not deleting obsolete weights files. An - // advantage would be that we don't need ndw's currentNodes anymore, - // which allows us to run ndw and wdw in parallel in the future. - wdw.deleteObsoleteWeightsDataFiles(); - printStatusTime("Deleted obsolete weights files"); - wdw.updateAdvertisedBandwidths(); - printStatusTime("Updated advertised bandwidths"); + printStatusTime("Wrote weights document files"); printStatus("Shutting down."); dso.writeHistoryFiles(); diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index ba4482b..b8e24ca 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -31,8 +31,6 @@ public class WeightsDataWriter implements DescriptorListener { private DocumentStore documentStore; - private SortedSet<String> currentFingerprints = new TreeSet<String>(); - public WeightsDataWriter(DescriptorSource descriptorSource, DocumentStore documentStore) { this.descriptorSource = descriptorSource; @@ -64,8 +62,9 @@ public class WeightsDataWriter implements DescriptorListener { this.consensuses.add(consensus); } - private Set<String> updateAdvertisedBandwidths = - new HashSet<String>(); + private Set<String> updateWeightsStatuses = new HashSet<String>(); + + private Set<String> updateWeightsDocuments = new HashSet<String>(); private Map<String, Set<String>> descriptorDigestsByFingerprint = new HashMap<String, Set<String>>(); @@ -83,7 +82,7 @@ public class WeightsDataWriter implements DescriptorListener { serverDescriptor.getBandwidthRate()); this.advertisedBandwidths.put(digest, advertisedBandwidth); String fingerprint = serverDescriptor.getFingerprint(); - this.updateAdvertisedBandwidths.add(fingerprint); + this.updateWeightsStatuses.add(fingerprint); if (!this.descriptorDigestsByFingerprint.containsKey( fingerprint)) { this.descriptorDigestsByFingerprint.put(fingerprint, @@ -92,11 +91,6 @@ public class WeightsDataWriter implements DescriptorListener { this.descriptorDigestsByFingerprint.get(fingerprint).add(digest); } - public void setCurrentNodes( - SortedMap<String, NodeStatus> currentNodes) { - this.currentFingerprints.addAll(currentNodes.keySet()); - } - public void updateWeightsHistories() { for (RelayNetworkStatusConsensus consensus : this.consensuses) { long validAfterMillis = consensus.getValidAfterMillis(), @@ -280,6 +274,8 @@ public class WeightsDataWriter implements DescriptorListener { history.put(interval, weights); history = this.compressHistory(history); this.writeHistoryToDisk(fingerprint, history); + this.updateWeightsDocuments.add(fingerprint); + this.updateWeightsStatuses.remove(fingerprint); } } @@ -441,14 +437,9 @@ public class WeightsDataWriter implements DescriptorListener { } public void writeWeightsDataFiles() { - for (String fingerprint : this.currentFingerprints) { + for (String fingerprint : this.updateWeightsDocuments) { SortedMap<long[], double[]> history = this.readHistoryFromDisk(fingerprint); - if (history.isEmpty() || history.lastKey()[1] < this.now - - 7L * 24L * 60L * 60L * 1000L) { - /* Don't write weights data file to disk. */ - continue; - } WeightsDocument weightsDocument = new WeightsDocument(); weightsDocument.documentString = this.formatHistoryString( fingerprint, history); @@ -605,22 +596,8 @@ public class WeightsDataWriter implements DescriptorListener { } } - public void deleteObsoleteWeightsDataFiles() { - SortedSet<String> obsoleteWeightsFiles; - obsoleteWeightsFiles = this.documentStore.list(WeightsDocument.class, - false); - for (String fingerprint : this.currentFingerprints) { - if (obsoleteWeightsFiles.contains(fingerprint)) { - obsoleteWeightsFiles.remove(fingerprint); - } - } - for (String fingerprint : obsoleteWeightsFiles) { - this.documentStore.remove(WeightsDocument.class, fingerprint); - } - } - - public void updateAdvertisedBandwidths() { - for (String fingerprint : this.updateAdvertisedBandwidths) { + public void updateWeightsStatuses() { + for (String fingerprint : this.updateWeightsStatuses) { SortedMap<long[], double[]> history = this.readHistoryFromDisk(fingerprint); this.writeHistoryToDisk(fingerprint, history);