commit 7ebdf3fc4f4eac3fc9b557e42b5ce75fffe0c571 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu Oct 20 16:22:28 2011 +0200 Check the consensus from each authority. So far, we downloaded the consensus from the authorities in random order and stopped after receiving the first valid response. The problem is that some authorities may reply with a non-fresh consensus whereas others reply with a fresh one. We now download the consensus from each authority and warn if one or more of them isn't fresh anymore. We also warn if an authority doesn't return a consensus within a timeout of 60 seconds at all. Implements #4250. --- src/org/torproject/chc/Downloader.java | 56 ++++++++++-------- src/org/torproject/chc/Main.java | 10 +-- src/org/torproject/chc/MetricsWebsiteReport.java | 17 ++++- src/org/torproject/chc/NagiosReport.java | 69 +++++++++++++++++++-- src/org/torproject/chc/Parser.java | 34 ++++++++--- src/org/torproject/chc/Report.java | 4 +- src/org/torproject/chc/StdOutReport.java | 70 +++++++++++++++++++-- 7 files changed, 203 insertions(+), 57 deletions(-) diff --git a/src/org/torproject/chc/Downloader.java b/src/org/torproject/chc/Downloader.java index cf451ee..17ae71c 100644 --- a/src/org/torproject/chc/Downloader.java +++ b/src/org/torproject/chc/Downloader.java @@ -14,41 +14,46 @@ public class Downloader { /* List of directory authorities to download consensuses and votes * from. */ - private static final List<String> AUTHORITIES = - new ArrayList<String>(Arrays.asList(("212.112.245.170,86.59.21.38," - + "216.224.124.114:9030,213.115.239.118:443,193.23.244.244," - + "208.83.223.34:443,128.31.0.34:9131,194.109.206.212"). - split(","))); + private SortedMap<String, String> authorities = + new TreeMap<String, String>(); + public Downloader() { + this.authorities.put("gabelmoo", "212.112.245.170"); + this.authorities.put("tor26", "86.59.21.38"); + this.authorities.put("ides", "216.224.124.114:9030"); + this.authorities.put("maatuska", "213.115.239.118:443"); + this.authorities.put("dannenberg", "193.23.244.244"); + this.authorities.put("urras", "208.83.223.34:443"); + this.authorities.put("moria1", "128.31.0.34:9131"); + this.authorities.put("dizum", "194.109.206.212"); + } /* Download a new consensus and corresponding votes. */ public void downloadFromAuthorities() { this.downloadConsensus(); - if (this.downloadedConsensus != null) { + if (!this.downloadedConsensuses.isEmpty()) { this.parseConsensusToFindReferencedVotes(); this.downloadReferencedVotes(); } } - /* Download the most recent consensus. */ - private String downloadedConsensus; + /* Download the most recent consensus from all authorities. */ + private SortedMap<String, String> downloadedConsensuses = + new TreeMap<String, String>(); private void downloadConsensus() { - List<String> authorities = new ArrayList<String>(AUTHORITIES); - Collections.shuffle(authorities); - for (String authority : authorities) { - if (this.downloadedConsensus != null) { - break; - } + for (Map.Entry<String, String> e : this.authorities.entrySet()) { + String nickname = e.getKey(); + String address = e.getValue(); String resource = "/tor/status-vote/current/consensus.z"; - String fullUrl = "http://" + authority + resource; + String fullUrl = "http://" + address + resource; String response = this.downloadFromAuthority(fullUrl); if (response != null) { - this.downloadedConsensus = response; + this.downloadedConsensuses.put(nickname, response); } else { System.err.println("Could not download consensus from directory " - + "authority " + authority + ". Ignoring."); + + "authority " + nickname + ". Ignoring."); } } - if (this.downloadedConsensus == null) { + if (this.downloadedConsensuses.isEmpty()) { System.err.println("Could not download consensus from any of the " + "directory authorities. Ignoring."); } @@ -122,10 +127,11 @@ public class Downloader { * authorities publishing the corresponding votes. */ private List<String> fingerprints = new ArrayList<String>(); private void parseConsensusToFindReferencedVotes() { - if (this.downloadedConsensus != null) { + for (String downloadedConsensus : + this.downloadedConsensuses.values()) { try { BufferedReader br = new BufferedReader(new StringReader( - this.downloadedConsensus)); + downloadedConsensus)); String line; while ((line = br.readLine()) != null) { if (line.startsWith("valid-after ")) { @@ -173,7 +179,8 @@ public class Downloader { private void downloadReferencedVotes() { for (String fingerprint : this.fingerprints) { String downloadedVote = null; - List<String> authorities = new ArrayList<String>(AUTHORITIES); + List<String> authorities = new ArrayList<String>( + this.authorities.values()); Collections.shuffle(authorities); for (String authority : authorities) { if (downloadedVote != null) { @@ -190,9 +197,10 @@ public class Downloader { } } - /* Return the previously downloaded (unparsed) consensus string. */ - public String getConsensusString() { - return this.downloadedConsensus; + /* Return the previously downloaded (unparsed) consensus string by + * authority nickname. */ + public SortedMap<String, String> getConsensusStrings() { + return this.downloadedConsensuses; } /* Return the previously downloaded (unparsed) vote strings. */ diff --git a/src/org/torproject/chc/Main.java b/src/org/torproject/chc/Main.java index d42ee44..aa25e57 100644 --- a/src/org/torproject/chc/Main.java +++ b/src/org/torproject/chc/Main.java @@ -23,12 +23,10 @@ public class Main { /* Parse consensus and votes and pass them to the reports. */ Parser parser = new Parser(); - Status parsedDownloadedConsensus = parser.parse( - downloader.getConsensusString(), downloader.getVoteStrings()); - if (parsedDownloadedConsensus != null) { - for (Report report : reports) { - report.processDownloadedConsensus(parsedDownloadedConsensus); - } + SortedMap<String, Status> parsedDownloadedConsensuses = parser.parse( + downloader.getConsensusStrings(), downloader.getVoteStrings()); + for (Report report : reports) { + report.processDownloadedConsensuses(parsedDownloadedConsensuses); } /* Finish writing reports. */ diff --git a/src/org/torproject/chc/MetricsWebsiteReport.java b/src/org/torproject/chc/MetricsWebsiteReport.java index f77b89f..01d126c 100644 --- a/src/org/torproject/chc/MetricsWebsiteReport.java +++ b/src/org/torproject/chc/MetricsWebsiteReport.java @@ -29,9 +29,20 @@ public class MetricsWebsiteReport implements Report { * processing. */ private Status downloadedConsensus; private SortedSet<Status> downloadedVotes; - public void processDownloadedConsensus(Status downloadedConsensus) { - this.downloadedConsensus = downloadedConsensus; - this.downloadedVotes = downloadedConsensus.getVotes(); + public void processDownloadedConsensuses( + SortedMap<String, Status> downloadedConsensuses) { + long mostRecentValidAfterMillis = -1L; + for (Status downloadedConsensus : downloadedConsensuses.values()) { + if (downloadedConsensus.getValidAfterMillis() > + mostRecentValidAfterMillis) { + this.downloadedConsensus = downloadedConsensus; + mostRecentValidAfterMillis = + downloadedConsensus.getValidAfterMillis(); + } + } + if (this.downloadedConsensus != null) { + this.downloadedVotes = this.downloadedConsensus.getVotes(); + } } /* Writer to write all HTML output to. */ diff --git a/src/org/torproject/chc/NagiosReport.java b/src/org/torproject/chc/NagiosReport.java index 2b5a6d3..a9f27af 100644 --- a/src/org/torproject/chc/NagiosReport.java +++ b/src/org/torproject/chc/NagiosReport.java @@ -20,11 +20,12 @@ public class NagiosReport implements Report { /* Store the current consensus and corresponding votes for * processing. */ + private SortedMap<String, Status> downloadedConsensuses; private Status downloadedConsensus; private SortedSet<Status> downloadedVotes; - public void processDownloadedConsensus(Status downloadedConsensus) { - this.downloadedConsensus = downloadedConsensus; - this.downloadedVotes = downloadedConsensus.getVotes(); + public void processDownloadedConsensuses( + SortedMap<String, Status> downloadedConsensuses) { + this.downloadedConsensuses = downloadedConsensuses; } /* Lists of output messages sorted by warnings, criticals, and @@ -43,6 +44,9 @@ public class NagiosReport implements Report { /* Check consensus and votes and write any findings to the output * file. */ public void writeReport() { + this.findMostRecentConsensus(); + this.checkMissingConsensuses(); + this.checkAllConsensusesFresh(); if (this.downloadedConsensus != null) { if (this.isConsensusFresh(this.downloadedConsensus)) { this.checkConsensusMethods(); @@ -58,13 +62,66 @@ public class NagiosReport implements Report { this.writeNagiosStatusFile(); } + /* Find most recent consensus and corresponding votes. */ + private void findMostRecentConsensus() { + long mostRecentValidAfterMillis = -1L; + for (Status downloadedConsensus : downloadedConsensuses.values()) { + if (downloadedConsensus.getValidAfterMillis() > + mostRecentValidAfterMillis) { + this.downloadedConsensus = downloadedConsensus; + mostRecentValidAfterMillis = + downloadedConsensus.getValidAfterMillis(); + } + } + if (this.downloadedConsensus != null) { + this.downloadedVotes = this.downloadedConsensus.getVotes(); + } + } + + /* Check if any directory authority didn't tell us a consensus. */ + private void checkMissingConsensuses() { + SortedSet<String> missingConsensuses = new TreeSet<String>( + Arrays.asList(("gabelmoo,tor26,ides,maatuska,dannenberg,urras," + + "moria1,dizum").split(","))); + missingConsensuses.removeAll(this.downloadedConsensuses.keySet()); + if (!missingConsensuses.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String nickname : missingConsensuses) { + sb.append(", " + nickname); + } + this.nagiosCriticals.add("The following directory authorities did " + + "not return a consensus within a timeout of 60 seconds: " + + sb.toString().substring(2)); + } + } + + /* Check if all consensuses are fresh. */ + private void checkAllConsensusesFresh() { + long fresh = System.currentTimeMillis() - 60L * 60L * 1000L; + SortedSet<String> nonFresh = new TreeSet<String>(); + for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + String nickname = e.getKey(); + Status downloadedConsensus = e.getValue(); + if (downloadedConsensus.getValidAfterMillis() < fresh) { + nonFresh.add(nickname); + } + } + if (!nonFresh.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String nickname : nonFresh) { + sb.append(", " + nickname); + } + this.nagiosCriticals.add("The consensuses published by the " + + "following directory authorities are more than 1 hour old " + + "and therefore not fresh anymore: " + + sb.toString().substring(2)); + } + } + /* Check if the most recent consensus is older than 1 hour. */ private boolean isConsensusFresh(Status consensus) { if (consensus.getValidAfterMillis() < System.currentTimeMillis() - 60L * 60L * 1000L) { - this.nagiosCriticals.add("The last known consensus published at " - + dateTimeFormat.format(consensus.getValidAfterMillis()) - + " is more than 1 hour old and therefore not fresh anymore"); return false; } else { return true; diff --git a/src/org/torproject/chc/Parser.java b/src/org/torproject/chc/Parser.java index 47609a8..053c7b4 100644 --- a/src/org/torproject/chc/Parser.java +++ b/src/org/torproject/chc/Parser.java @@ -12,18 +12,34 @@ public class Parser { /* Parse and return a consensus and corresponding votes, or null if * something goes wrong. */ - public Status parse(String consensusString, List<String> voteStrings) { - Status consensus = this.parseConsensusOrVote(consensusString, true); - if (consensus != null) { - for (String voteString : voteStrings) { - Status vote = this.parseConsensusOrVote(voteString, false); - if (consensus.getValidAfterMillis() == - vote.getValidAfterMillis()) { - consensus.addVote(vote); + public SortedMap<String, Status> parse( + SortedMap<String, String> consensusStrings, + List<String> voteStrings) { + SortedSet<Status> parsedVotes = new TreeSet<Status>(); + for (String voteString : voteStrings) { + Status parsedVote = this.parseConsensusOrVote(voteString, false); + if (parsedVote != null) { + parsedVotes.add(parsedVote); + } + } + SortedMap<String, Status> parsedConsensuses = + new TreeMap<String, Status>(); + for (Map.Entry<String, String> e : consensusStrings.entrySet()) { + String nickname = e.getKey(); + String consensusString = e.getValue(); + Status parsedConsensus = this.parseConsensusOrVote(consensusString, + true); + if (parsedConsensus != null) { + for (Status parsedVote : parsedVotes) { + if (parsedConsensus.getValidAfterMillis() == + parsedVote.getValidAfterMillis()) { + parsedConsensus.addVote(parsedVote); + } } + parsedConsensuses.put(nickname, parsedConsensus); } } - return consensus; + return parsedConsensuses; } /* Date-time formats to parse and format timestamps. */ diff --git a/src/org/torproject/chc/Report.java b/src/org/torproject/chc/Report.java index c3937e0..0f1a306 100644 --- a/src/org/torproject/chc/Report.java +++ b/src/org/torproject/chc/Report.java @@ -10,8 +10,8 @@ public interface Report { /* Process the downloaded current consensus and corresponding votes to * find irregularities between them. */ - public abstract void processDownloadedConsensus( - Status downloadedConsensus); + public abstract void processDownloadedConsensuses( + SortedMap<String, Status> downloadedConsensuses); /* Finish writing report. */ public abstract void writeReport(); diff --git a/src/org/torproject/chc/StdOutReport.java b/src/org/torproject/chc/StdOutReport.java index 15a9148..f683ae6 100644 --- a/src/org/torproject/chc/StdOutReport.java +++ b/src/org/torproject/chc/StdOutReport.java @@ -23,17 +23,21 @@ public class StdOutReport implements Report { /* Downloaded consensus and corresponding votes for later * processing. */ + private SortedMap<String, Status> downloadedConsensuses; private Status downloadedConsensus; private SortedSet<Status> downloadedVotes; - public void processDownloadedConsensus(Status downloadedConsensus) { - this.downloadedConsensus = downloadedConsensus; - this.downloadedVotes = downloadedConsensus.getVotes(); + public void processDownloadedConsensuses( + SortedMap<String, Status> downloadedConsensuses) { + this.downloadedConsensuses = downloadedConsensuses; } /* Check consensuses and votes for irregularities and write output to * stdout. */ public void writeReport() { this.readLastWarned(); + this.findMostRecentConsensus(); + this.checkMissingConsensuses(); + this.checkAllConsensusesFresh(); if (this.downloadedConsensus != null) { if (this.isConsensusFresh(this.downloadedConsensus)) { this.checkConsensusMethods(); @@ -87,14 +91,66 @@ public class StdOutReport implements Report { } } + /* Find most recent consensus and corresponding votes. */ + private void findMostRecentConsensus() { + long mostRecentValidAfterMillis = -1L; + for (Status downloadedConsensus : downloadedConsensuses.values()) { + if (downloadedConsensus.getValidAfterMillis() > + mostRecentValidAfterMillis) { + this.downloadedConsensus = downloadedConsensus; + mostRecentValidAfterMillis = + downloadedConsensus.getValidAfterMillis(); + } + } + if (this.downloadedConsensus != null) { + this.downloadedVotes = this.downloadedConsensus.getVotes(); + } + } + + /* Check if any directory authority didn't tell us a consensus. */ + private void checkMissingConsensuses() { + SortedSet<String> missingConsensuses = new TreeSet<String>( + Arrays.asList(("gabelmoo,tor26,ides,maatuska,dannenberg,urras," + + "moria1,dizum").split(","))); + missingConsensuses.removeAll(this.downloadedConsensuses.keySet()); + if (!missingConsensuses.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String nickname : missingConsensuses) { + sb.append(", " + nickname); + } + this.warnings.put("The following directory authorities did not " + + "return a consensus within a timeout of 60 seconds: " + + sb.toString().substring(2), 150L * 60L * 1000L); + } + } + + /* Check if all consensuses are fresh. */ + private void checkAllConsensusesFresh() { + long fresh = System.currentTimeMillis() - 60L * 60L * 1000L; + SortedSet<String> nonFresh = new TreeSet<String>(); + for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + String nickname = e.getKey(); + Status downloadedConsensus = e.getValue(); + if (downloadedConsensus.getValidAfterMillis() < fresh) { + nonFresh.add(nickname); + } + } + if (!nonFresh.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String nickname : nonFresh) { + sb.append(", " + nickname); + } + this.warnings.put("The consensuses published by the following " + + "directory authorities are more than 1 hour old and " + + "therefore not fresh anymore: " + + sb.toString().substring(2), 150L * 60L * 1000L); + } + } + /* Check if the most recent consensus is older than 1 hour. */ private boolean isConsensusFresh(Status consensus) { if (consensus.getValidAfterMillis() < System.currentTimeMillis() - 60L * 60L * 1000L) { - this.warnings.put("The last known consensus published at " - + dateTimeFormat.format(consensus.getValidAfterMillis()) - + " is more than 1 hour old and therefore not fresh anymore", - 0L); return false; } else { return true;