commit 1cc54e92b52686f6054f90a5fcad0c53bc99c742 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Fri Jul 18 22:26:31 2014 +0200 Consider only newly stored descriptors in stale check. A few days ago there was an unusual situation with parsing gabelmoo's cached descriptors and fetching descriptors from the other directory authorities. gabelmoo failed to produce a consensus at :00 of the hour and downloaded the consensus from the other directory authorities after :05 of the hour. As a result, gabelmoo's cached consensus was 1 hour older than the consensus that we downloaded from the other directory authorities. Our stale check considers all stored descriptors and determines if there were any consensuses for which we didn't also store votes or microdesc consensuses. In this case the stale check triggered, because we didn't store any votes or microdesc consensuses with the same valid-after time as gabelmoo's outdated cached consensus. This warning was confusing. The fix is simple: if we're storing a descriptor that we stored before, don't include it in the stale check. --- .../ernie/db/relaydescs/ArchiveWriter.java | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index e0e52f2..ffe3dbc 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -365,13 +365,15 @@ public class ArchiveWriter extends Thread { printFormat.setTimeZone(TimeZone.getTimeZone("UTC")); File tarballFile = new File(this.outputDirectory + "/consensus/" + printFormat.format(new Date(validAfter)) + "-consensus"); + boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = new File("rsync/relay-descriptors/consensuses/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile }; if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) { this.storedConsensusesCounter++; } - if (this.now - validAfter < 3L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore && + this.now - validAfter < 3L * 60L * 60L * 1000L) { this.storedConsensuses.put(validAfter, serverDescriptorDigests); this.expectedVotes.put(validAfter, dirSources.size()); } @@ -392,6 +394,7 @@ public class ArchiveWriter extends Thread { + "/consensus-microdesc/" + dayDirectoryFileFormat.format(validAfter) + "-consensus-microdesc"); + boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = new File("rsync/relay-descriptors/microdescs/" + "consensus-microdesc/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile }; @@ -399,7 +402,8 @@ public class ArchiveWriter extends Thread { null)) { this.storedMicrodescConsensusesCounter++; } - if (this.now - validAfter < 3L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore && + this.now - validAfter < 3L * 60L * 60L * 1000L) { this.storedMicrodescConsensuses.put(validAfter, microdescriptorDigests); } @@ -416,13 +420,15 @@ public class ArchiveWriter extends Thread { File tarballFile = new File(this.outputDirectory + "/vote/" + printFormat.format(new Date(validAfter)) + "-vote-" + fingerprint + "-" + digest); + boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = new File("rsync/relay-descriptors/votes/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile }; if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) { this.storedVotesCounter++; } - if (this.now - validAfter < 3L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore && + this.now - validAfter < 3L * 60L * 60L * 1000L) { if (!this.storedVotes.containsKey(validAfter)) { this.storedVotes.put(validAfter, new TreeMap<String, SortedSet<String>>()); @@ -457,6 +463,7 @@ public class ArchiveWriter extends Thread { + "/server-descriptor/" + printFormat.format(new Date(published)) + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/" + digest); + boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = new File( "rsync/relay-descriptors/server-descriptors/" + digest); File rsyncCatFile = new File("rsync/relay-descriptors/" @@ -469,7 +476,8 @@ public class ArchiveWriter extends Thread { append)) { this.storedServerDescriptorsCounter++; } - if (this.now - published < 48L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore && + this.now - published < 48L * 60L * 60L * 1000L) { if (!this.storedServerDescriptors.containsKey(published)) { this.storedServerDescriptors.put(published, new HashMap<String, String>()); @@ -490,6 +498,7 @@ public class ArchiveWriter extends Thread { + extraInfoDigest.substring(0, 1) + "/" + extraInfoDigest.substring(1, 2) + "/" + extraInfoDigest); + boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = new File("rsync/relay-descriptors/extra-infos/" + extraInfoDigest); File rsyncCatFile = new File("rsync/relay-descriptors/" @@ -500,7 +509,8 @@ public class ArchiveWriter extends Thread { if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) { this.storedExtraInfoDescriptorsCounter++; } - if (this.now - published < 48L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore && + this.now - published < 48L * 60L * 60L * 1000L) { if (!this.storedExtraInfoDescriptors.containsKey(published)) { this.storedExtraInfoDescriptors.put(published, new HashSet<String>()); @@ -527,6 +537,7 @@ public class ArchiveWriter extends Thread { + microdescriptorDigest.substring(0, 1) + "/" + microdescriptorDigest.substring(1, 2) + "/" + microdescriptorDigest); + boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncFile = new File("rsync/relay-descriptors/microdescs/micro/" + microdescriptorDigest); File rsyncCatFile = new File("rsync/relay-descriptors/" @@ -539,7 +550,8 @@ public class ArchiveWriter extends Thread { append)) { this.storedMicrodescriptorsCounter++; } - if (this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore && + this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) { if (!this.storedMicrodescriptors.containsKey(validAfter)) { this.storedMicrodescriptors.put(validAfter, new HashSet<String>());