commit b75f6809502952760b0c7426d867f3817a54d1c1 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sat Oct 27 18:05:34 2012 -0400 Download votes of all known authorities. We currently look at the downloaded consensus to decide which votes to download. This approach fails when we're missing a consensus. Instead, download votes published by all known authorities, not just the ones contained in the consensus. Implements #5812. --- config.template | 4 +++ .../torproject/ernie/db/main/Configuration.java | 18 ++++++++++++++++ .../ernie/db/relaydescs/ArchiveWriter.java | 1 + .../db/relaydescs/RelayDescriptorDownloader.java | 22 +++++++++++++++++-- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/config.template b/config.template index 8f9888b..a31dc9e 100644 --- a/config.template +++ b/config.template @@ -26,6 +26,10 @@ ## download missing relay descriptors from #DownloadFromDirectoryAuthorities 86.59.21.38,76.73.17.194:9030,171.25.193.9:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170,154.35.32.5 # +## Comma separated list of directory authority fingerprints to download +## votes +#DownloadVotesByFingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,27B6B5996C426270A5C95488AA5BCEB6BCC86956,49015F787433103580E3B66A1707A00E60F2D15B,585769C78764D58426B8B52B6651A5A71137189A,80550987E1D626E3EBA5E5E75A458DE0626D088C,D586D18309DED4CD6D57C18FDB97EFA96D330566,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,ED03BB616EB2F60BEC80151114BB25CEF515B226,EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 +# ## Download the current consensus (only if DownloadRelayDescriptors is 1) #DownloadCurrentConsensus 1 # diff --git a/src/org/torproject/ernie/db/main/Configuration.java b/src/org/torproject/ernie/db/main/Configuration.java index 65ab948..bfb7307 100644 --- a/src/org/torproject/ernie/db/main/Configuration.java +++ b/src/org/torproject/ernie/db/main/Configuration.java @@ -41,6 +41,16 @@ public class Configuration { "86.59.21.38,76.73.17.194:9030,213.115.239.118:443," + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131," + "194.109.206.212,212.112.245.170").split(",")); + private List<String> downloadVotesByFingerprint = Arrays.asList(( + "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4," + + "27B6B5996C426270A5C95488AA5BCEB6BCC86956," + + "49015F787433103580E3B66A1707A00E60F2D15B," + + "585769C78764D58426B8B52B6651A5A71137189A," + + "80550987E1D626E3EBA5E5E75A458DE0626D088C," + + "D586D18309DED4CD6D57C18FDB97EFA96D330566," + + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58," + + "ED03BB616EB2F60BEC80151114BB25CEF515B226," + + "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97").split(",")); private boolean downloadCurrentConsensus = true; private boolean downloadCurrentVotes = true; private boolean downloadMissingServerDescriptors = true; @@ -133,6 +143,11 @@ public class Configuration { new URL("http://" + dir + "/"); this.downloadFromDirectoryAuthorities.add(dir); } + } else if (line.startsWith("DownloadVotesByFingerprint")) { + this.downloadVotesByFingerprint = new ArrayList<String>(); + for (String fingerprint : line.split(" ")[1].split(",")) { + this.downloadVotesByFingerprint.add(fingerprint); + } } else if (line.startsWith("DownloadCurrentConsensus")) { this.downloadCurrentConsensus = Integer.parseInt( line.split(" ")[1]) != 0; @@ -297,6 +312,9 @@ public class Configuration { public List<String> getDownloadFromDirectoryAuthorities() { return this.downloadFromDirectoryAuthorities; } + public List<String> getDownloadVotesByFingerprint() { + return this.downloadVotesByFingerprint; + } public boolean getDownloadCurrentConsensus() { return this.downloadCurrentConsensus; } diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index 9a07ada..8562bd5 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -60,6 +60,7 @@ public class ArchiveWriter extends Thread { List<String> dirSources = config.getDownloadFromDirectoryAuthorities(); rdd = new RelayDescriptorDownloader(rdp, dirSources, + config.getDownloadVotesByFingerprint(), config.getDownloadCurrentConsensus(), config.getDownloadCurrentVotes(), config.getDownloadMissingServerDescriptors(), diff --git a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java index 0bea50a..ce2f16a 100644 --- a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java +++ b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java @@ -88,6 +88,12 @@ public class RelayDescriptorDownloader { private List<String> authorities; /** + * Fingerprints of directory authorities that we will use to download + * votes without requiring a successfully downloaded consensus. + */ + private List<String> authorityFingerprints; + + /** * Should we try to download the current consensus if we don't have it? */ private boolean downloadCurrentConsensus; @@ -201,7 +207,8 @@ public class RelayDescriptorDownloader { * <code>stats/last-downloaded-all-descriptors</code>. */ public RelayDescriptorDownloader(RelayDescriptorParser rdp, - List<String> authorities, boolean downloadCurrentConsensus, + List<String> authorities, List<String> authorityFingerprints, + boolean downloadCurrentConsensus, boolean downloadCurrentVotes, boolean downloadMissingServerDescriptors, boolean downloadMissingExtraInfos, @@ -211,6 +218,8 @@ public class RelayDescriptorDownloader { /* Memorize argument values. */ this.rdp = rdp; this.authorities = new ArrayList<String>(authorities); + this.authorityFingerprints = new ArrayList<String>( + authorityFingerprints); this.downloadCurrentConsensus = downloadCurrentConsensus; this.downloadCurrentVotes = downloadCurrentVotes; this.downloadMissingServerDescriptors = @@ -467,13 +476,20 @@ public class RelayDescriptorDownloader { */ public void downloadDescriptors() { - /* Put the current consensus on the missing list, unless we already - * have it. */ + /* Put the current consensus and votes on the missing list, unless we + * already have them. */ String consensusKey = "consensus," + this.currentValidAfter; if (!this.missingDescriptors.containsKey(consensusKey)) { this.missingDescriptors.put(consensusKey, "NA"); this.newMissingConsensuses++; } + for (String authority : authorityFingerprints) { + String voteKey = "vote," + this.currentValidAfter + "," + authority; + if (!this.missingDescriptors.containsKey(voteKey)) { + this.missingDescriptors.put(voteKey, "NA"); + this.newMissingVotes++; + } + } /* Download descriptors from authorities which are in random order, so * that we distribute the load somewhat fairly over time. */