[metrics-db/master] Download votes of all known authorities. - tor-commits

27 Oct 2012

commit b75f6809502952760b0c7426d867f3817a54d1c1
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Sat Oct 27 18:05:34 2012 -0400
Download votes of all known authorities.
We currently look at the downloaded consensus to decide which votes to
    download.  This approach fails when we're missing a consensus.  Instead,
    download votes published by all known authorities, not just the ones
    contained in the consensus.  Implements #5812.
---
 config.template                                    |    4 +++
 .../torproject/ernie/db/main/Configuration.java    |   18 ++++++++++++++++
 .../ernie/db/relaydescs/ArchiveWriter.java         |    1 +
 .../db/relaydescs/RelayDescriptorDownloader.java   |   22 +++++++++++++++++--
 4 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/config.template b/config.template
index 8f9888b..a31dc9e 100644
--- a/config.template
+++ b/config.template
@@ -26,6 +26,10 @@
 ## download missing relay descriptors from
 #DownloadFromDirectoryAuthorities 86.59.21.38,76.73.17.194:9030,171.25.193.9:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170,154.35.32.5
 #
+## Comma separated list of directory authority fingerprints to download
+## votes
+#DownloadVotesByFingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,27B6B5996C426270A5C95488AA5BCEB6BCC86956,49015F787433103580E3B66A1707A00E60F2D15B,585769C78764D58426B8B52B6651A5A71137189A,80550987E1D626E3EBA5E5E75A458DE0626D088C,D586D18309DED4CD6D57C18FDB97EFA96D330566,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,ED03BB616EB2F60BEC80151114BB25CEF515B226,EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97
+#
 ## Download the current consensus (only if DownloadRelayDescriptors is 1)
 #DownloadCurrentConsensus 1
 #
diff --git a/src/org/torproject/ernie/db/main/Configuration.java b/src/org/torproject/ernie/db/main/Configuration.java
index 65ab948..bfb7307 100644
--- a/src/org/torproject/ernie/db/main/Configuration.java
+++ b/src/org/torproject/ernie/db/main/Configuration.java
@@ -41,6 +41,16 @@ public class Configuration {
       "86.59.21.38,76.73.17.194:9030,213.115.239.118:443,"
       + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
       + "194.109.206.212,212.112.245.170").split(","));
+  private List<String> downloadVotesByFingerprint = Arrays.asList((
+      "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
+      + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
+      + "49015F787433103580E3B66A1707A00E60F2D15B,"
+      + "585769C78764D58426B8B52B6651A5A71137189A,"
+      + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
+      + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
+      + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
+      + "ED03BB616EB2F60BEC80151114BB25CEF515B226,"
+      + "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97").split(","));
   private boolean downloadCurrentConsensus = true;
   private boolean downloadCurrentVotes = true;
   private boolean downloadMissingServerDescriptors = true;
@@ -133,6 +143,11 @@ public class Configuration {
             new URL("http://" + dir + "/");
             this.downloadFromDirectoryAuthorities.add(dir);
           }
+        } else if (line.startsWith("DownloadVotesByFingerprint")) {
+          this.downloadVotesByFingerprint = new ArrayList<String>();
+          for (String fingerprint : line.split(" ")[1].split(",")) {
+            this.downloadVotesByFingerprint.add(fingerprint);
+          }
         } else if (line.startsWith("DownloadCurrentConsensus")) {
           this.downloadCurrentConsensus = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -297,6 +312,9 @@ public class Configuration {
   public List<String> getDownloadFromDirectoryAuthorities() {
     return this.downloadFromDirectoryAuthorities;
   }
+  public List<String> getDownloadVotesByFingerprint() {
+    return this.downloadVotesByFingerprint;
+  }
   public boolean getDownloadCurrentConsensus() {
     return this.downloadCurrentConsensus;
   }
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index 9a07ada..8562bd5 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -60,6 +60,7 @@ public class ArchiveWriter extends Thread {
       List<String> dirSources =
           config.getDownloadFromDirectoryAuthorities();
       rdd = new RelayDescriptorDownloader(rdp, dirSources,
+          config.getDownloadVotesByFingerprint(),
           config.getDownloadCurrentConsensus(),
           config.getDownloadCurrentVotes(),
           config.getDownloadMissingServerDescriptors(),
diff --git a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java
index 0bea50a..ce2f16a 100644
--- a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java
+++ b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java
@@ -88,6 +88,12 @@ public class RelayDescriptorDownloader {
   private List<String> authorities;
/**
+   * Fingerprints of directory authorities that we will use to download
+   * votes without requiring a successfully downloaded consensus.
+   */
+  private List<String> authorityFingerprints;
+
+  /**
    * Should we try to download the current consensus if we don't have it?
    */
   private boolean downloadCurrentConsensus;
@@ -201,7 +207,8 @@ public class RelayDescriptorDownloader {
    * <code>stats/last-downloaded-all-descriptors</code>.
    */
   public RelayDescriptorDownloader(RelayDescriptorParser rdp,
-      List<String> authorities, boolean downloadCurrentConsensus,
+      List<String> authorities, List<String> authorityFingerprints,
+      boolean downloadCurrentConsensus,
       boolean downloadCurrentVotes,
       boolean downloadMissingServerDescriptors,
       boolean downloadMissingExtraInfos,
@@ -211,6 +218,8 @@ public class RelayDescriptorDownloader {
     /* Memorize argument values. */
     this.rdp = rdp;
     this.authorities = new ArrayList<String>(authorities);
+    this.authorityFingerprints = new ArrayList<String>(
+        authorityFingerprints);
     this.downloadCurrentConsensus = downloadCurrentConsensus;
     this.downloadCurrentVotes = downloadCurrentVotes;
     this.downloadMissingServerDescriptors =
@@ -467,13 +476,20 @@ public class RelayDescriptorDownloader {
    */
   public void downloadDescriptors() {
-    /* Put the current consensus on the missing list, unless we already
-     * have it. */
+    /* Put the current consensus and votes on the missing list, unless we
+     * already have them. */
     String consensusKey = "consensus," + this.currentValidAfter;
     if (!this.missingDescriptors.containsKey(consensusKey)) {
       this.missingDescriptors.put(consensusKey, "NA");
       this.newMissingConsensuses++;
     }
+    for (String authority : authorityFingerprints) {
+      String voteKey = "vote," + this.currentValidAfter + "," + authority;
+      if (!this.missingDescriptors.containsKey(voteKey)) {
+        this.missingDescriptors.put(voteKey, "NA");
+        this.newMissingVotes++;
+      }
+    }
/* Download descriptors from authorities which are in random order, so
      * that we distribute the load somewhat fairly over time. */

    