[tor-commits] [metrics-web/master] Rewrite the consensus-health checker.

commit 2f6c2bdc3d2b5393f8c884cf333a5e54ab4d45dc Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Oct 12 15:22:18 2011 +0200

Rewrite the consensus-health checker.

The major change in this rewrite is that the consensus-health checker is now independent from importing descriptors into the metrics database. It now downloads its own consensus and votes, triggered by its own cronjob.

This rewrite is the second in a series of refactorings to decouple the many metrics-web components. The first refactoring was to move ExoneraTor data into its own database and use a separate database importer for it. The goal will be to have separate component for each functionality without dependencies, so that every component can fail independent from the rest.

Another effect of the rewrite is that the consensus-health checker can now be extended more easily by adding new reports or new checks. --- build.xml | 9 + config.template | 8 - src/org/torproject/chc/Archiver.java | 120 +++ src/org/torproject/chc/Downloader.java | 190 ++++ src/org/torproject/chc/Main.java | 70 ++ src/org/torproject/chc/MetricsWebsiteReport.java | 841 +++++++++++++++++ src/org/torproject/chc/NagiosReport.java | 238 +++++ src/org/torproject/chc/Parser.java | 162 ++++ src/org/torproject/chc/Report.java | 23 + src/org/torproject/chc/Status.java | 185 ++++ src/org/torproject/chc/StatusEntry.java | 48 + src/org/torproject/chc/StdOutReport.java | 288 ++++++ src/org/torproject/ernie/cron/Configuration.java | 14 - .../ernie/cron/ConsensusHealthChecker.java | 993 -------------------- src/org/torproject/ernie/cron/Main.java | 17 +- .../ernie/cron/RelayDescriptorParser.java | 13 +- 16 files changed, 2178 insertions(+), 1041 deletions(-)

diff --git a/build.xml b/build.xml index 9194ccd..32a450f 100644 --- a/build.xml +++ b/build.xml @@ -60,6 +60,15 @@ </java> </target>

+ <!-- Run consensus-health checker. --> + <target name="chc" depends="compile"> + <java fork="true" + maxmemory="512m" + classname="org.torproject.chc.Main"> + <classpath refid="classpath"/> + </java> + </target> + <!-- Prepare data for being displayed on the website. --> <target name="run" depends="compile"> <java fork="true" diff --git a/config.template b/config.template index e0fa743..8c56909 100644 --- a/config.template +++ b/config.template @@ -36,14 +36,6 @@ ## files will be overwritten! #RelayDescriptorRawFilesDirectory pg-import/ # -## Write statistics about the current consensus and votes to the -## website -#WriteConsensusHealth 0 -# -## Write Nagios status file containing potential problems with the latest -## consensus -#WriteNagiosStatusFile 0 -# ## Write bridge stats to disk #WriteBridgeStats 0 # diff --git a/src/org/torproject/chc/Archiver.java b/src/org/torproject/chc/Archiver.java new file mode 100644 index 0000000..6271348 --- /dev/null +++ b/src/org/torproject/chc/Archiver.java @@ -0,0 +1,120 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.io.*; +import java.util.*; + +/* Load the last network status consensus and corresponding votes from + * disk and save newly downloaded ones to disk. */ +public class Archiver { + + /* Local directory containing cached consensuses and votes. */ + private File cachedFilesDirectory = new File("chc-cache"); + + /* Copy of the most recent cached consensus and votes. */ + private String loadedConsensus; + private List<String> loadedVotes = new ArrayList<String>(); + + /* Load the most recent cached consensus and votes to memory. */ + public void loadLastFromDisk() { + if (!this.cachedFilesDirectory.exists()) { + return; + } + String lastValidAfter = this.findLastConsensusPrefix(); + if (lastValidAfter != null) { + for (File file : this.cachedFilesDirectory.listFiles()) { + if (file.isDirectory() || + !file.getName().startsWith(lastValidAfter)) { + continue; + } + String content = null; + try { + FileInputStream fis = new FileInputStream(file); + BufferedInputStream bis = new BufferedInputStream(fis); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + bis.close(); + byte[] allData = baos.toByteArray(); + content = new String(allData); + } catch (IOException e) { + System.err.println("Could not read cached status from file '" + + file.getAbsolutePath() + "'. Skipping."); + continue; + } + if (file.getName().contains("-consensus")) { + this.loadedConsensus = content; + } else if (file.getName().contains("-vote-")) { + this.loadedVotes.add(content); + } + } + } + } + + /* Delete all cached consensuses and votes but the last consensus and + * corresponding votes. */ + public void deleteAllButLast() { + if (!this.cachedFilesDirectory.exists()) { + return; + } + String lastValidAfter = this.findLastConsensusPrefix(); + if (lastValidAfter != null) { + for (File file : this.cachedFilesDirectory.listFiles()) { + if (!file.getName().startsWith(lastValidAfter)) { + file.delete(); + } + } + } + } + + /* Save a status to disk under the given file name. */ + public void saveStatusStringToDisk(String content, String fileName) { + try { + this.cachedFilesDirectory.mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter(new File( + this.cachedFilesDirectory, fileName))); + bw.write(content); + bw.close(); + } catch (IOException e) { + System.err.println("Could not save status to file '" + + this.cachedFilesDirectory.getAbsolutePath() + "/" + fileName + + "'. Ignoring."); + } + } + + /* Find and return the timestamp prefix of the last published + * consensus. */ + private String findLastConsensusPrefix() { + String lastValidAfter = null; + for (File file : this.cachedFilesDirectory.listFiles()) { + if (file.isDirectory() || + file.getName().length() != + "yyyy-MM-dd-HH-mm-ss-consensus".length() || + !file.getName().endsWith("-consensus")) { + continue; + } + String prefix = file.getName().substring(0, + "yyyy-MM-dd-HH-mm-ss".length()); + if (lastValidAfter == null || + prefix.compareTo(lastValidAfter) > 0) { + lastValidAfter = prefix; + } + } + return lastValidAfter; + } + + /* Return the previously loaded (unparsed) consensus string. */ + public String getConsensusString() { + return this.loadedConsensus; + } + + /* Return the previously loaded (unparsed) vote strings. */ + public List<String> getVoteStrings() { + return this.loadedVotes; + } +} + diff --git a/src/org/torproject/chc/Downloader.java b/src/org/torproject/chc/Downloader.java new file mode 100644 index 0000000..94ed30f --- /dev/null +++ b/src/org/torproject/chc/Downloader.java @@ -0,0 +1,190 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.zip.*; + +/* Download the latest network status consensus and corresponding + * votes. */ +public class Downloader { + + /* List of directory authorities to download consensuses and votes + * from. */ + private static final List<String> AUTHORITIES = + new ArrayList<String>(Arrays.asList(("212.112.245.170,86.59.21.38," + + "216.224.124.114:9030,213.115.239.118:443,193.23.244.244," + + "208.83.223.34:443,128.31.0.34:9131,194.109.206.212"). + split(","))); + + /* Set the last known valid-after time to avoid downloading a new + * consensus if there cannot be a new one yet. */ + private long lastKnownValidAfterMillis; + public void setLastKnownValidAfterMillis( + long lastKnownValidAfterMillis) { + this.lastKnownValidAfterMillis = lastKnownValidAfterMillis; + } + + /* Download a new consensus and corresponding votes if we expect them to + * be newer than the ones we have. */ + public void downloadFromAuthorities() { + if (System.currentTimeMillis() - lastKnownValidAfterMillis < + 60L * 60L * 1000L) { + return; + } + this.downloadConsensus(); + if (this.downloadedConsensus != null) { + this.parseConsensusToFindReferencedVotes(); + this.downloadReferencedVotes(); + } + } + + /* Download the most recent consensus. */ + private String downloadedConsensus; + private void downloadConsensus() { + List<String> authorities = new ArrayList<String>(AUTHORITIES); + Collections.shuffle(authorities); + for (String authority : authorities) { + if (this.downloadedConsensus != null) { + break; + } + String resource = "/tor/status-vote/current/consensus.z"; + String fullUrl = "http://" + authority + resource; + String response = this.downloadFromAuthority(fullUrl); + if (response != null) { + this.downloadedConsensus = response; + } else { + System.err.println("Could not download consensus from directory " + + "authority " + authority + ". Ignoring."); + } + } + if (this.downloadedConsensus == null) { + System.err.println("Could not download consensus from any of the " + + "directory authorities. Ignoring."); + } + } + + private static class DownloadRunnable implements Runnable { + Thread mainThread; + String url; + String response; + boolean interrupted = false; + public DownloadRunnable(String url) { + this.mainThread = Thread.currentThread(); + this.url = url; + } + public void run() { + try { + URL u = new URL(url); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + int responseCode = huc.getResponseCode(); + if (responseCode == 200) { + BufferedInputStream in = new BufferedInputStream( + new InflaterInputStream(huc.getInputStream())); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while (!this.interrupted && + (len = in.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + if (this.interrupted) { + return; + } + in.close(); + byte[] allData = baos.toByteArray(); + this.response = new String(allData); + this.mainThread.interrupt(); + } + } catch (IOException e) { + /* Can't do much except leaving this.response at null. */ + } + } + } + + /* Download a consensus or vote from a directory authority using a + * timeout of 60 seconds. */ + private String downloadFromAuthority(final String url) { + DownloadRunnable downloadRunnable = new DownloadRunnable(url); + new Thread(downloadRunnable).start(); + try { + Thread.sleep(60L * 1000L); + } catch (InterruptedException e) { + /* Do nothing. */ + } + String response = downloadRunnable.response; + downloadRunnable.interrupted = true; + return response; + } + + /* Parse the downloaded consensus to find fingerprints of directory + * authorities publishing the corresponding votes. */ + private List<String> fingerprints = new ArrayList<String>(); + private void parseConsensusToFindReferencedVotes() { + if (this.downloadedConsensus != null) { + try { + BufferedReader br = new BufferedReader(new StringReader( + this.downloadedConsensus)); + String line; + while ((line = br.readLine()) != null) { + if (line.startsWith("dir-source ")) { + String[] parts = line.split(" "); + if (parts.length < 3) { + System.err.println("Bad dir-source line '" + line + + "' in downloaded consensus. Skipping."); + continue; + } + String nickname = parts[1]; + if (nickname.endsWith("-legacy")) { + continue; + } + String fingerprint = parts[2]; + this.fingerprints.add(fingerprint); + } + } + br.close(); + } catch (IOException e) { + System.err.println("Could not parse consensus to find referenced " + + "votes in it. Skipping."); + } + } + } + + /* Download the votes published by directory authorities listed in the + * consensus. */ + private List<String> downloadedVotes = new ArrayList<String>(); + private void downloadReferencedVotes() { + for (String fingerprint : this.fingerprints) { + String downloadedVote = null; + List<String> authorities = new ArrayList<String>(AUTHORITIES); + Collections.shuffle(authorities); + for (String authority : authorities) { + if (downloadedVote != null) { + break; + } + String resource = "/tor/status-vote/current/" + fingerprint + + ".z"; + String fullUrl = "http://" + authority + resource; + downloadedVote = this.downloadFromAuthority(fullUrl); + if (downloadedVote != null) { + this.downloadedVotes.add(downloadedVote); + } + } + } + } + + /* Return the previously downloaded (unparsed) consensus string. */ + public String getConsensusString() { + return this.downloadedConsensus; + } + + /* Return the previously downloaded (unparsed) vote strings. */ + public List<String> getVoteStrings() { + return this.downloadedVotes; + } +} + diff --git a/src/org/torproject/chc/Main.java b/src/org/torproject/chc/Main.java new file mode 100644 index 0000000..5aadc6e --- /dev/null +++ b/src/org/torproject/chc/Main.java @@ -0,0 +1,70 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.util.*; + +/* Coordinate the process of downloading consensus and votes to check + * Tor's consensus health. */ +public class Main { + public static void main(String[] args) { + + /* Initialize reports. */ + List<Report> reports = new ArrayList<Report>(); + reports.add(new MetricsWebsiteReport( + "website/consensus-health.html")); + reports.add(new NagiosReport("stats/consensus-health")); + reports.add(new StdOutReport()); + + /* Load last-known consensus and votes from disk and parse them. */ + Archiver archiver = new Archiver(); + archiver.loadLastFromDisk(); + Parser parser = new Parser(); + Status parsedCachedConsensus = parser.parse( + archiver.getConsensusString(), archiver.getVoteStrings()); + if (parsedCachedConsensus != null) { + for (Report report : reports) { + report.processCachedConsensus(parsedCachedConsensus); + } + } + + /* Download consensus and corresponding votes from the directory + * authorities and parse them, too. */ + Downloader downloader = new Downloader(); + if (parsedCachedConsensus != null) { + downloader.setLastKnownValidAfterMillis( + parsedCachedConsensus.getValidAfterMillis()); + } + downloader.downloadFromAuthorities(); + Status parsedDownloadedConsensus = parser.parse( + downloader.getConsensusString(), downloader.getVoteStrings()); + if (parsedDownloadedConsensus != null) { + for (Report report : reports) { + report.processDownloadedConsensus(parsedDownloadedConsensus); + } + } + + /* Save the new consensus and corresponding votes to disk and delete + * all previous ones. */ + if (parsedDownloadedConsensus != null) { + archiver.saveStatusStringToDisk( + parsedDownloadedConsensus.getUnparsedString(), + parsedDownloadedConsensus.getFileName()); + for (Status vote : parsedDownloadedConsensus.getVotes()) { + archiver.saveStatusStringToDisk(vote.getUnparsedString(), + vote.getFileName()); + } + archiver.deleteAllButLast(); + } + + /* Finish writing reports. */ + for (Report report : reports) { + report.writeReport(); + } + + /* Terminate the program including any download threads that may still + * be running. */ + System.exit(0); + } +} + diff --git a/src/org/torproject/chc/MetricsWebsiteReport.java b/src/org/torproject/chc/MetricsWebsiteReport.java new file mode 100644 index 0000000..2a54927 --- /dev/null +++ b/src/org/torproject/chc/MetricsWebsiteReport.java @@ -0,0 +1,841 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.io.*; +import java.text.*; +import java.util.*; + +/* Transform the most recent consensus and corresponding votes into an + * HTML page showing possible irregularities. */ +public class MetricsWebsiteReport implements Report { + + /* Date-time format to format timestamps. */ + private static SimpleDateFormat dateTimeFormat; + static { + dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + /* Output file to write report to. */ + private File htmlOutputFile; + + /* Initialize this report. */ + public MetricsWebsiteReport(String htmlOutputFilename) { + this.htmlOutputFile = new File(htmlOutputFilename); + } + + /* Process the last but one consensus (ignored in this report). */ + public void processCachedConsensus(Status cachedConsensus) { + /* Do nothing. */ + } + + /* Store the downloaded consensus and corresponding votes for later + * processing. */ + private Status downloadedConsensus; + private SortedSet<Status> downloadedVotes; + public void processDownloadedConsensus(Status downloadedConsensus) { + this.downloadedConsensus = downloadedConsensus; + this.downloadedVotes = downloadedConsensus.getVotes(); + } + + /* Writer to write all HTML output to. */ + private BufferedWriter bw; + + /* Write HTML output file for the metrics website. */ + public void writeReport() { + + if (this.downloadedConsensus != null) { + try { + this.htmlOutputFile.getParentFile().mkdirs(); + this.bw = new BufferedWriter(new FileWriter(this.htmlOutputFile)); + writePageHeader(); + writeValidAfterTime(); + writeKnownFlags(); + writeNumberOfRelaysVotedAbout(); + writeConsensusMethods(); + writeRecommendedVersions(); + writeConsensusParameters(); + writeAuthorityKeys(); + writeBandwidthScannerStatus(); + writeAuthorityVersions(); + writeRelayFlagsTable(); + writeRelayFlagsSummary(); + writePageFooter(); + this.bw.close(); + } catch (IOException e) { + System.err.println("Could not write HTML output file '" + + this.htmlOutputFile.getAbsolutePath() + "'. Ignoring."); + } + } + } + + /* Write the HTML page header including the metrics website + * navigation. */ + private void writePageHeader() throws IOException { + this.bw.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 " + + "Transitional//EN\">\n" + + "<html>\n" + + " <head>\n" + + " <title>Tor Metrics Portal: Consensus health</title>\n" + + " <meta http-equiv="content-type" content="text/html; " + + "charset=ISO-8859-1">\n" + + " <link href="/css/stylesheet-ltr.css" type="text/css" " + + "rel="stylesheet">\n" + + " <link href="/images/favicon.ico" " + + "type="image/x-icon" rel="shortcut icon">\n" + + " </head>\n" + + " <body>\n" + + " <div class="center">\n" + + " <table class="banner" border="0" cellpadding="0" " + + "cellspacing="0" summary="">\n" + + " <tr>\n" + + " <td class="banner-left"><a " + + "href="/index.html"><img src="/images/top-left.png" " + + "alt="Click to go to home page" width="193" " + + "height="79"></a></td>\n" + + " <td class="banner-middle">\n" + + " <a href="/">Home</a>\n" + + " <a href="graphs.html">Graphs</a>\n" + + " <a href="research.html">Research</a>\n" + + " <a href="status.html">Status</a>\n" + + " <br>\n" + + " <font size="2">\n" + + " <a href="networkstatus.html">Network " + + "Status</a>\n" + + " <a href="exonerator.html">ExoneraTor</a>\n" + + " <a href="relay-search.html">Relay Search</a>\n" + + " <a class="current">Consensus Health</a>\n" + + " </font>\n" + + " </td>\n" + + " <td class="banner-right"></td>\n" + + " </tr>\n" + + " </table>\n" + + " <div class="main-column">\n" + + " <h2>Tor Metrics Portal: Consensus Health</h2>\n" + + " <br>\n" + + " <p>This page shows statistics about the current " + + "consensus and votes to facilitate debugging of the " + + "directory consensus process.</p>\n"); + } + + /* Write the valid-after time of the downloaded consensus. */ + private void writeValidAfterTime() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Valid-after time</h3>\n" + + " <br>\n" + + " <p>Consensus was published "); + if (this.downloadedConsensus.getValidAfterMillis() < + System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { + this.bw.write("<font color="red">" + + dateTimeFormat.format( + this.downloadedConsensus.getValidAfterMillis()) + "</font>"); + } else { + this.bw.write(dateTimeFormat.format( + this.downloadedConsensus.getValidAfterMillis())); + } + this.bw.write(". <i>Note that it takes up to 15 to learn about new " + + "consensus and votes and process them.</i></p>\n"); + } + + /* Write the lists of known flags. */ + private void writeKnownFlags() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Known flags</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + for (Status vote : this.downloadedVotes) { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>known-flags"); + for (String knownFlag : vote.getKnownFlags()) { + this.bw.write(" " + knownFlag); + } + this.bw.write("</td>\n" + + " </tr>\n"); + } + } + this.bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">known-flags"); + for (String knownFlag : this.downloadedConsensus.getKnownFlags()) { + this.bw.write(" " + knownFlag); + } + this.bw.write("</font></td>\n" + + " </tr>\n" + + " </table>\n"); + } + + /* Write the number of relays voted about. */ + private void writeNumberOfRelaysVotedAbout() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Number of relays voted about</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="320">\n" + + " <col width="320">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td><td></td>" + + "</tr>\n"); + } else { + for (Status vote : this.downloadedVotes) { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>" + vote.getStatusEntries().size() + + " total</td>\n" + + " <td>" + vote.getRunningRelays() + + " Running</td>\n" + + " </tr>\n"); + } + } + this.bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">" + + this.downloadedConsensus.getStatusEntries().size() + + " total</font></td>\n" + + " <td><font color="blue">" + + this.downloadedConsensus.getRunningRelays() + + " Running</font></td>\n" + + " </tr>\n" + + " </table>\n"); + } + + /* Write the supported consensus methods of directory authorities and + * the resulting consensus method. */ + private void writeConsensusMethods() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Consensus methods</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + for (Status vote : this.downloadedVotes) { + SortedSet<Integer> consensusMethods = + vote.getConsensusMethods(); + if (consensusMethods.contains( + this.downloadedConsensus.getConsensusMethods().last())) { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>consensus-methods"); + for (int consensusMethod : consensusMethods) { + this.bw.write(" " + String.valueOf(consensusMethod)); + } + this.bw.write("</td>\n" + + " </tr>\n"); + } else { + this.bw.write(" <tr>\n" + + " <td><font color="red">" + + vote.getNickname() + "</font></td>\n" + + " <td><font color="red">" + + "consensus-methods"); + for (int consensusMethod : consensusMethods) { + this.bw.write(" " + String.valueOf(consensusMethod)); + } + this.bw.write("</font></td>\n" + + " </tr>\n"); + } + } + } + this.bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">consensus-method " + + this.downloadedConsensus.getConsensusMethods().last() + + "</font></td>\n" + + " </tr>\n" + + " </table>\n"); + } + + /* Write recommended versions. */ + private void writeRecommendedVersions() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Recommended versions</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + for (Status vote : this.downloadedVotes) { + SortedSet<String> voteRecommendedClientVersions = + vote.getRecommendedClientVersions(); + if (voteRecommendedClientVersions != null) { + if (downloadedConsensus.getRecommendedClientVersions().equals( + voteRecommendedClientVersions)) { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>client-versions "); + int i = 0; + for (String version : voteRecommendedClientVersions) { + this.bw.write((i++ > 0 ? "," : "") + version); + } + this.bw.write("</td>\n" + + " </tr>\n"); + } else { + this.bw.write(" <tr>\n" + + " <td><font color="red">" + + vote.getNickname() + + "</font></td>\n" + + " <td><font color="red">client-versions "); + int i = 0; + for (String version : voteRecommendedClientVersions) { + this.bw.write((i++ > 0 ? "," : "") + version); + } + this.bw.write("</font></td>\n" + + " </tr>\n"); + } + } + SortedSet<String> voteRecommendedServerVersions = + vote.getRecommendedServerVersions(); + if (voteRecommendedServerVersions != null) { + if (downloadedConsensus.getRecommendedServerVersions().equals( + voteRecommendedServerVersions)) { + this.bw.write(" <tr>\n" + + " <td></td>\n" + + " <td>server-versions "); + int i = 0; + for (String version : voteRecommendedServerVersions) { + this.bw.write((i++ > 0 ? "," : "") + version); + } + this.bw.write("</td>\n" + + " </tr>\n"); + } else { + this.bw.write(" <tr>\n" + + " <td></td>\n" + + " <td><font color="red">server-versions "); + int i = 0; + for (String version : voteRecommendedServerVersions) { + this.bw.write((i++ > 0 ? "," : "") + version); + } + this.bw.write("</font></td>\n" + + " </tr>\n"); + } + } + } + } + this.bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">client-versions "); + int i = 0; + for (String version : + downloadedConsensus.getRecommendedClientVersions()) { + this.bw.write((i++ > 0 ? "," : "") + version); + } + this.bw.write("</font></td>\n" + + " </tr>\n" + + " <tr>\n" + + " <td></td>\n" + + " <td><font color="blue">server-versions "); + i = 0; + for (String version : + downloadedConsensus.getRecommendedServerVersions()) { + this.bw.write((i++ > 0 ? "," : "") + version); + } + this.bw.write("</font></td>\n" + + " </tr>\n" + + " </table>\n"); + } + + /* Write consensus parameters. */ + private void writeConsensusParameters() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Consensus parameters</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + Set<String> validParameters = new HashSet<String>(Arrays.asList( + ("circwindow,CircuitPriorityHalflifeMsec,refuseunknownexits," + + "cbtdisabled,cbtnummodes,cbtrecentcount,cbtmaxtimeouts," + + "cbtmincircs,cbtquantile,cbtclosequantile,cbttestfreq," + + "cbtmintimeout,cbtinitialtimeout").split(","))); + Map<String, String> consensusConsensusParams = + downloadedConsensus.getConsensusParams(); + for (Status vote : this.downloadedVotes) { + Map<String, String> voteConsensusParams = + vote.getConsensusParams(); + boolean conflictOrInvalid = false; + if (voteConsensusParams == null) { + for (Map.Entry<String, String> e : + voteConsensusParams.entrySet()) { + if (!consensusConsensusParams.containsKey(e.getKey()) || + !consensusConsensusParams.get(e.getKey()).equals( + e.getValue()) || + !validParameters.contains(e.getKey())) { + conflictOrInvalid = true; + break; + } + } + } + if (conflictOrInvalid) { + this.bw.write(" <tr>\n" + + " <td><font color="red">" + + vote.getNickname() + "</font></td>\n" + + " <td><font color="red">params"); + for (Map.Entry<String, String> e : + voteConsensusParams.entrySet()) { + this.bw.write(" " + e.getKey() + "=" + e.getValue()); + } + this.bw.write("</font></td>\n" + + " </tr>\n"); + } else { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>params"); + for (Map.Entry<String, String> e : + voteConsensusParams.entrySet()) { + this.bw.write(" " + e.getKey() + "=" + e.getValue()); + } + this.bw.write("</td>\n" + + " </tr>\n"); + } + } + } + this.bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">params"); + for (Map.Entry<String, String> e : + this.downloadedConsensus.getConsensusParams().entrySet()) { + this.bw.write(" " + e.getKey() + "=" + e.getValue()); + } + this.bw.write("</font></td>\n" + + " </tr>\n" + + " </table>\n"); + } + + /* Write authority keys and their expiration dates. */ + private void writeAuthorityKeys() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Authority keys</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + for (Status vote : this.downloadedVotes) { + long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); + if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < + System.currentTimeMillis()) { + this.bw.write(" <tr>\n" + + " <td><font color="red">" + + vote.getNickname() + "</font></td>\n" + + " <td><font color="red">dir-key-expires " + + dateTimeFormat.format(voteDirKeyExpiresMillis) + + "</font></td>\n" + + " </tr>\n"); + } else { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>dir-key-expires " + + dateTimeFormat.format(voteDirKeyExpiresMillis) + + "</td>\n" + + " </tr>\n"); + } + } + } + this.bw.write(" </table>\n" + + " <br>\n" + + " <p><i>Note that expiration dates of legacy keys are " + + "not included in votes and therefore not listed here!</i>" + + "</p>\n"); + } + + /* Write the status of bandwidth scanners and results being contained + * in votes. */ + private void writeBandwidthScannerStatus() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Bandwidth scanner status</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (this.downloadedVotes.size() < 1) { + this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + for (Status vote : this.downloadedVotes) { + if (vote.getBandwidthWeights() > 0) { + this.bw.write(" <tr>\n" + + " <td>" + vote.getNickname() + "</td>\n" + + " <td>" + vote.getBandwidthWeights() + + " Measured values in w lines</td>\n" + + " </tr>\n"); + } + } + } + this.bw.write(" </table>\n"); + } + + /* Write directory authority versions. */ + private void writeAuthorityVersions() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Authority versions</h3>\n" + + " <br>\n"); + Map<String, String> authorityVersions = + this.downloadedConsensus.getAuthorityVersions(); + if (authorityVersions.size() < 1) { + this.bw.write(" <p>(No relays with Authority flag found.)" + + "</p>\n"); + } else { + this.bw.write(" <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + for (Map.Entry<String, String> e : authorityVersions.entrySet()) { + String nickname = e.getKey(); + String versionString = e.getValue(); + this.bw.write(" <tr>\n" + + " <td>" + nickname + "</td>\n" + + " <td>" + versionString + "</td>\n" + + " </tr>\n"); + } + this.bw.write(" </table>\n" + + " <br>\n" + + " <p><i>Note that this list of relays with the " + + "Authority flag may be different from the list of v3 " + + "directory authorities!</i></p>\n"); + } + } + + /* Write the (huge) table containing relay flags contained in votes and + * the consensus for each relay. */ + private void writeRelayFlagsTable() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Relay flags</h3>\n" + + " <br>\n" + + " <p>The semantics of flags written in the table is " + + "as follows:</p>\n" + + " <ul>\n" + + " <li><b>In vote and consensus:</b> Flag in vote " + + "matches flag in consensus, or relay is not listed in " + + "consensus (because it doesn't have the Running " + + "flag)</li>\n" + + " <li><b><font color="red">Only in " + + "vote:</font></b> Flag in vote, but missing in the " + + "consensus, because there was no majority for the flag or " + + "the flag was invalidated (e.g., Named gets invalidated by " + + "Unnamed)</li>\n" + + " <li><b><font color="gray"><s>Only in " + + "consensus:</s></font></b> Flag in consensus, but missing " + + "in a vote of a directory authority voting on this " + + "flag</li>\n" + + " <li><b><font color="blue">In " + + "consensus:</font></b> Flag in consensus</li>\n" + + " </ul>\n" + + " <br>\n" + + " <p>See also the summary below the table.</p>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="120">\n" + + " <col width="80">\n"); + for (int i = 0; i < this.downloadedVotes.size(); i++) { + this.bw.write(" <col width="" + + (640 / this.downloadedVotes.size()) + "">\n"); + } + this.bw.write(" </colgroup>\n"); + SortedMap<String, String> allRelays = new TreeMap<String, String>(); + for (Status vote : this.downloadedVotes) { + for (StatusEntry statusEntry : vote.getStatusEntries().values()) { + allRelays.put(statusEntry.getFingerprint(), + statusEntry.getNickname()); + } + } + for (StatusEntry statusEntry : + this.downloadedConsensus.getStatusEntries().values()) { + allRelays.put(statusEntry.getFingerprint(), + statusEntry.getNickname()); + } + int linesWritten = 0; + for (Map.Entry<String, String> e : allRelays.entrySet()) { + if (linesWritten++ % 10 == 0) { + this.writeRelayFlagsTableHeader(); + } + String fingerprint = e.getKey(); + String nickname = e.getValue(); + this.writeRelayFlagsTableRow(fingerprint, nickname); + } + this.bw.write(" </table>\n"); + } + + /* Write the table header that is repeated every ten relays and that + * contains the directory authority names. */ + private void writeRelayFlagsTableHeader() throws IOException { + this.bw.write(" <tr><td><br><b>Fingerprint</b></td>" + + "<td><br><b>Nickname</b></td>\n"); + for (Status vote : this.downloadedVotes) { + String shortDirName = vote.getNickname().length() > 6 ? + vote.getNickname().substring(0, 5) + "." : + vote.getNickname(); + this.bw.write("<td><br><b>" + shortDirName + "</b></td>"); + } + this.bw.write("<td><br><b>consensus</b></td></tr>\n"); + } + + /* Write a single row in the table of relay flags. */ + private void writeRelayFlagsTableRow(String fingerprint, + String nickname) throws IOException { + this.bw.write(" <tr>\n"); + if (this.downloadedConsensus.containsStatusEntry(fingerprint) && + this.downloadedConsensus.getStatusEntry(fingerprint).getFlags(). + contains("Named") && + !Character.isDigit(nickname.charAt(0))) { + this.bw.write(" <td id="" + nickname + + ""><a href="relay.html?fingerprint=" + + fingerprint + "" target="_blank">" + + fingerprint.substring(0, 8) + "</a></td>\n"); + } else { + this.bw.write(" <td><a href="relay.html?fingerprint=" + + fingerprint + "" target="_blank">" + + fingerprint.substring(0, 8) + "</a></td>\n"); + } + this.bw.write(" <td>" + nickname + "</td>\n"); + SortedSet<String> relevantFlags = new TreeSet<String>(); + for (Status vote : this.downloadedVotes) { + if (vote.containsStatusEntry(fingerprint)) { + relevantFlags.addAll(vote.getStatusEntry(fingerprint).getFlags()); + } + } + SortedSet<String> consensusFlags = null; + if (this.downloadedConsensus.containsStatusEntry(fingerprint)) { + consensusFlags = this.downloadedConsensus. + getStatusEntry(fingerprint).getFlags(); + relevantFlags.addAll(consensusFlags); + } + for (Status vote : this.downloadedVotes) { + if (vote.containsStatusEntry(fingerprint)) { + SortedSet<String> flags = vote.getStatusEntry(fingerprint). + getFlags(); + this.bw.write(" <td>"); + int flagsWritten = 0; + for (String flag : relevantFlags) { + this.bw.write(flagsWritten++ > 0 ? "<br>" : ""); + if (flags.contains(flag)) { + if (consensusFlags == null || + consensusFlags.contains(flag)) { + this.bw.write(flag); + } else { + this.bw.write("<font color="red">" + flag + "</font>"); + } + } else if (consensusFlags != null && + vote.getKnownFlags().contains(flag) && + consensusFlags.contains(flag)) { + this.bw.write("<font color="gray"><s>" + flag + + "</s></font>"); + } + } + this.bw.write("</td>\n"); + } else { + this.bw.write(" <td></td>\n"); + } + } + if (consensusFlags != null) { + this.bw.write(" <td>"); + int flagsWritten = 0; + for (String flag : relevantFlags) { + this.bw.write(flagsWritten++ > 0 ? "<br>" : ""); + if (consensusFlags.contains(flag)) { + this.bw.write("<font color="blue">" + flag + "</font>"); + } + } + this.bw.write("</td>\n"); + } else { + this.bw.write(" <td></td>\n"); + } + this.bw.write(" </tr>\n"); + } + + /* Write the relay flag summary. */ + private void writeRelayFlagsSummary() throws IOException { + this.bw.write(" <br>\n" + + " <h3>Overlap between votes and consensus</h3>\n" + + " <br>\n" + + " <p>The semantics of columns is similar to the " + + "table above:</p>\n" + + " <ul>\n" + + " <li><b>In vote and consensus:</b> Flag in vote " + + "matches flag in consensus, or relay is not listed in " + + "consensus (because it doesn't have the Running " + + "flag)</li>\n" + + " <li><b><font color="red">Only in " + + "vote:</font></b> Flag in vote, but missing in the " + + "consensus, because there was no majority for the flag or " + + "the flag was invalidated (e.g., Named gets invalidated by " + + "Unnamed)</li>\n" + + " <li><b><font color="gray"><s>Only in " + + "consensus:</s></font></b> Flag in consensus, but missing " + + "in a vote of a directory authority voting on this " + + "flag</li>\n" + + " </ul>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="210">\n" + + " <col width="210">\n" + + " <col width="210">\n" + + " </colgroup>\n" + + " <tr><td></td><td><b>Only in vote</b></td>" + + "<td><b>In vote and consensus</b></td>" + + "<td><b>Only in consensus</b></td>\n"); + Set<String> allFingerprints = new HashSet<String>(); + for (Status vote : this.downloadedVotes) { + allFingerprints.addAll(vote.getStatusEntries().keySet()); + } + allFingerprints.addAll(this.downloadedConsensus.getStatusEntries(). + keySet()); + SortedMap<String, SortedMap<String, Integer>> flagsAgree = + new TreeMap<String, SortedMap<String, Integer>>(); + SortedMap<String, SortedMap<String, Integer>> flagsLost = + new TreeMap<String, SortedMap<String, Integer>>(); + SortedMap<String, SortedMap<String, Integer>> flagsMissing = + new TreeMap<String, SortedMap<String, Integer>>(); + for (String fingerprint : allFingerprints) { + SortedSet<String> consensusFlags = + this.downloadedConsensus.containsStatusEntry(fingerprint) ? + this.downloadedConsensus.getStatusEntry(fingerprint).getFlags() : + null; + for (Status vote : this.downloadedVotes) { + String dir = vote.getNickname(); + if (vote.containsStatusEntry(fingerprint)) { + SortedSet<String> flags = vote.getStatusEntry(fingerprint). + getFlags(); + for (String flag : this.downloadedConsensus.getKnownFlags()) { + SortedMap<String, SortedMap<String, Integer>> sums = null; + if (flags.contains(flag)) { + if (consensusFlags == null || + consensusFlags.contains(flag)) { + sums = flagsAgree; + } else { + sums = flagsLost; + } + } else if (consensusFlags != null && + vote.getKnownFlags().contains(flag) && + consensusFlags.contains(flag)) { + sums = flagsMissing; + } + if (sums != null) { + SortedMap<String, Integer> sum = null; + if (sums.containsKey(dir)) { + sum = sums.get(dir); + } else { + sum = new TreeMap<String, Integer>(); + sums.put(dir, sum); + } + sum.put(flag, sum.containsKey(flag) ? + sum.get(flag) + 1 : 1); + } + } + } + } + } + for (Status vote : this.downloadedVotes) { + String dir = vote.getNickname(); + int i = 0; + for (String flag : vote.getKnownFlags()) { + this.bw.write(" <tr>\n" + + " <td>" + (i++ == 0 ? dir : "") + + "</td>\n"); + if (flagsLost.containsKey(dir) && + flagsLost.get(dir).containsKey(flag)) { + this.bw.write(" <td><font color="red"> " + + flagsLost.get(dir).get(flag) + " " + flag + + "</font></td>\n"); + } else { + this.bw.write(" <td></td>\n"); + } + if (flagsAgree.containsKey(dir) && + flagsAgree.get(dir).containsKey(flag)) { + this.bw.write(" <td>" + flagsAgree.get(dir).get(flag) + + " " + flag + "</td>\n"); + } else { + this.bw.write(" <td></td>\n"); + } + if (flagsMissing.containsKey(dir) && + flagsMissing.get(dir).containsKey(flag)) { + this.bw.write(" <td><font color="gray"><s>" + + flagsMissing.get(dir).get(flag) + " " + flag + + "</s></font></td>\n"); + } else { + this.bw.write(" <td></td>\n"); + } + this.bw.write(" </tr>\n"); + } + } + this.bw.write(" </table>\n"); + } + + /* Write the footer of the HTML page containing the blurb that is on + * every page of the metrics website. */ + private void writePageFooter() throws IOException { + this.bw.write(" </div>\n" + + " </div>\n" + + " <div class="bottom" id="bottom">\n" + + " <p>This material is supported in part by the " + + "National Science Foundation under Grant No. " + + "CNS-0959138. Any opinions, finding, and conclusions " + + "or recommendations expressed in this material are " + + "those of the author(s) and do not necessarily reflect " + + "the views of the National Science Foundation.</p>\n" + + " <p>"Tor" and the "Onion Logo" are <a " + + "href="https://www.torproject.org/docs/trademark-faq.html" + + ".en">" + + "registered trademarks</a> of The Tor Project, " + + "Inc.</p>\n" + + " <p>Data on this site is freely available under a " + + "<a href="http://creativecommons.org/publicdomain/" + + "zero/1.0/">CC0 no copyright declaration</a>: To the " + + "extent possible under law, the Tor Project has waived " + + "all copyright and related or neighboring rights in " + + "the data. Graphs are licensed under a <a " + + "href="http://creativecommons.org/licenses/by/3.0/" + + "us/">Creative Commons Attribution 3.0 United States " + + "License</a>.</p>\n" + + " </div>\n" + + " </body>\n" + + "</html>"); + } +} + diff --git a/src/org/torproject/chc/NagiosReport.java b/src/org/torproject/chc/NagiosReport.java new file mode 100644 index 0000000..26d61a7 --- /dev/null +++ b/src/org/torproject/chc/NagiosReport.java @@ -0,0 +1,238 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.io.*; +import java.text.*; +import java.util.*; + +/* Check a given consensus and votes for irregularities and writes results + * to a local text file for Nagios to print out warnings. */ +public class NagiosReport implements Report { + + /* Output file to write report to. */ + private File nagiosOutputFile; + + /* Initialize this report. */ + public NagiosReport(String nagiosOutputFilename) { + this.nagiosOutputFile = new File(nagiosOutputFilename); + } + + /* Store the cached consensus for processing. */ + private Status cachedConsensus; + public void processCachedConsensus(Status cachedConsensus) { + this.cachedConsensus = cachedConsensus; + } + + /* Store the current consensus and corresponding votes for + * processing. */ + private Status downloadedConsensus; + private SortedSet<Status> downloadedVotes; + public void processDownloadedConsensus(Status downloadedConsensus) { + this.downloadedConsensus = downloadedConsensus; + this.downloadedVotes = downloadedConsensus.getVotes(); + } + + /* Lists of output messages sorted by warnings, criticals, and + * unknowns (increasing severity). */ + private List<String> nagiosWarnings = new ArrayList<String>(), + nagiosCriticals = new ArrayList<String>(), + nagiosUnknowns = new ArrayList<String>(); + + /* Date-time format to format timestamps. */ + private static SimpleDateFormat dateTimeFormat; + static { + dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + /* Check consensus and votes and write any findings to the output + * file. */ + public void writeReport() { + if (this.downloadedConsensus != null) { + this.checkConsensusMethods(); + this.checkRecommendedVersions(); + this.checkConsensusParameters(); + this.checkAuthorityKeys(); + this.checkMissingVotes(); + this.checkBandwidthScanners(); + this.checkConsensusAge(this.downloadedConsensus); + } else if (this.cachedConsensus != null) { + this.checkConsensusAge(this.cachedConsensus); + } else { + this.nagiosUnknowns.add("No consensus known"); + } + this.writeNagiosStatusFile(); + } + + /* Check supported consensus methods of all votes. */ + private void checkConsensusMethods() { + for (Status vote : this.downloadedVotes) { + if (!this.downloadedConsensus.getConsensusMethods().contains( + this.downloadedConsensus.getConsensusMethods().last())) { + nagiosWarnings.add(vote.getNickname() + " does not support " + + "consensus method " + + this.downloadedConsensus.getConsensusMethods().last()); + } + } + } + + /* Check if the recommended client and server versions in a vote are + * different from the recommended versions in the consensus. */ + private void checkRecommendedVersions() { + for (Status vote : this.downloadedVotes) { + if (vote.getRecommendedClientVersions() != null && + !downloadedConsensus.getRecommendedClientVersions().equals( + vote.getRecommendedClientVersions())) { + nagiosWarnings.add(vote.getNickname() + " recommends other " + + "client versions than the consensus"); + } + if (vote.getRecommendedServerVersions() != null && + !downloadedConsensus.getRecommendedServerVersions().equals( + vote.getRecommendedServerVersions())) { + nagiosWarnings.add(vote.getNickname() + " recommends other " + + "server versions than the consensus"); + } + } + } + + /* Checks if a vote contains conflicting or invalid consensus + * parameters. */ + private void checkConsensusParameters() { + Set<String> validParameters = new HashSet<String>(Arrays.asList( + ("circwindow,CircuitPriorityHalflifeMsec,refuseunknownexits," + + "cbtdisabled,cbtnummodes,cbtrecentcount,cbtmaxtimeouts," + + "cbtmincircs,cbtquantile,cbtclosequantile,cbttestfreq," + + "cbtmintimeout,cbtinitialtimeout").split(","))); + for (Status vote : this.downloadedVotes) { + Map<String, String> voteConsensusParams = + vote.getConsensusParams(); + boolean conflictOrInvalid = false; + if (voteConsensusParams == null) { + for (Map.Entry<String, String> e : + voteConsensusParams.entrySet()) { + if (!downloadedConsensus.getConsensusParams().containsKey( + e.getKey()) || + !downloadedConsensus.getConsensusParams().get(e.getKey()). + equals(e.getValue()) || + !validParameters.contains(e.getKey())) { + StringBuilder message = new StringBuilder(); + message.append(vote.getNickname() + " sets conflicting " + + "or invalid consensus parameters:"); + for (Map.Entry<String, String> p : + voteConsensusParams.entrySet()) { + message.append(" " + p.getKey() + "=" + p.getValue()); + } + nagiosWarnings.add(message.toString()); + break; + } + } + } + } + } + + /* Check whether authority keys expire in the next 14 days. */ + private void checkAuthorityKeys() { + for (Status vote : this.downloadedVotes) { + long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); + if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < + System.currentTimeMillis()) { + nagiosWarnings.add(vote.getNickname() + "'s certificate " + + "expires in the next 14 days"); + } + } + } + + /* Check if any votes are missing. */ + private void checkMissingVotes() { + SortedSet<String> knownAuthorities = new TreeSet<String>( + Arrays.asList(("dannenberg,dizum,gabelmoo,ides,maatuska,moria1," + + "tor26,urras").split(","))); + SortedSet<String> missingVotes = + new TreeSet<String>(knownAuthorities); + for (Status vote : this.downloadedVotes) { + missingVotes.remove(vote.getNickname()); + } + if (!missingVotes.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String missingDir : missingVotes) { + sb.append(", " + missingDir); + } + nagiosWarnings.add("We're missing votes from the following " + + "directory authorities: " + sb.toString().substring(2)); + } + } + + /* Check if any bandwidth scanner results are missing. */ + private void checkBandwidthScanners() { + SortedSet<String> missingBandwidthScanners = new TreeSet<String>( + Arrays.asList("ides,urras,moria1,gabelmoo".split(","))); + SortedSet<String> runningBandwidthScanners = new TreeSet<String>(); + for (Status vote : this.downloadedVotes) { + if (vote.getBandwidthWeights() > 0) { + missingBandwidthScanners.remove(vote.getNickname()); + runningBandwidthScanners.add(vote.getNickname()); + } + } + if (!missingBandwidthScanners.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String dir : missingBandwidthScanners) { + sb.append(", " + dir); + } + String message = "The following directory authorities are not " + + "reporting bandwidth scanner results: " + + sb.toString().substring(2); + if (runningBandwidthScanners.size() >= 3) { + nagiosWarnings.add(message); + } else { + nagiosCriticals.add(message); + } + } + } + + /* Check that the most recent consensus is not more than 3 hours + * old. */ + public void checkConsensusAge(Status consensus) { + if (consensus.getValidAfterMillis() < + System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { + this.nagiosCriticals.add("The last known consensus published at " + + dateTimeFormat.format(consensus.getValidAfterMillis()) + + " is more than 3 hours old"); + } + } + + /* Write all output to the Nagios status file. The most severe status + * goes in the first line of the output file and the same status and all + * log messages in the second line. */ + private void writeNagiosStatusFile() { + File nagiosStatusFile = this.nagiosOutputFile; + try { + BufferedWriter bw = new BufferedWriter(new FileWriter( + nagiosStatusFile)); + if (!nagiosUnknowns.isEmpty()) { + bw.write("UNKNOWN\nUNKNOWN"); + } else if (!nagiosCriticals.isEmpty()) { + bw.write("CRITICAL\nCRITICAL"); + } else if (!nagiosWarnings.isEmpty()) { + bw.write("WARNING\nWARNING"); + } else { + bw.write("OK\nOK"); + } + for (String message : nagiosUnknowns) { + bw.write(" " + message + ";"); + } + for (String message : nagiosCriticals) { + bw.write(" " + message + ";"); + } + for (String message : nagiosWarnings) { + bw.write(" " + message + ";"); + } + bw.write("\n"); + bw.close(); + } catch (IOException e) { + System.err.println("Could not write Nagios output file to '" + + nagiosStatusFile.getAbsolutePath() + "'. Ignoring."); + } + } +} + diff --git a/src/org/torproject/chc/Parser.java b/src/org/torproject/chc/Parser.java new file mode 100644 index 0000000..4f27fbe --- /dev/null +++ b/src/org/torproject/chc/Parser.java @@ -0,0 +1,162 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.io.*; +import java.text.*; +import java.util.*; +import org.apache.commons.codec.binary.*; + +/* Parse a network status consensus or vote. */ +public class Parser { + + /* Parse and return a consensus and corresponding votes, or null if + * something goes wrong. */ + public Status parse(String consensusString, List<String> voteStrings) { + Status consensus = this.parseConsensusOrVote(consensusString, true); + if (consensus != null) { + for (String voteString : voteStrings) { + Status vote = this.parseConsensusOrVote(voteString, false); + if (consensus.getValidAfterMillis() == + vote.getValidAfterMillis()) { + consensus.addVote(vote); + } + } + } + return consensus; + } + + /* Date-time formats to parse and format timestamps. */ + private static SimpleDateFormat dateTimeFormat; + private static SimpleDateFormat fileNameFormat; + static { + dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + fileNameFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); + fileNameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + /* Parse a consensus or vote string into a Status instance. */ + private Status parseConsensusOrVote(String statusString, + boolean isConsensus) { + if (statusString == null) { + return null; + } + Status status = new Status(); + status.setUnparsedString(statusString); + try { + BufferedReader br = new BufferedReader(new StringReader( + statusString)); + String line, rLine = null, sLine = null; + int totalRelays = 0, runningRelays = 0, bandwidthWeights = 0; + while ((line = br.readLine()) != null) { + if (line.startsWith("consensus-method ") || + line.startsWith("consensus-methods ")) { + SortedSet<Integer> consensusMethods = new TreeSet<Integer>(); + String[] parts = line.split(" "); + for (int i = 1; i < parts.length; i++) { + consensusMethods.add(Integer.parseInt(parts[i])); + } + status.setConsensusMethods(consensusMethods); + } else if (line.startsWith("valid-after ")) { + try { + status.setValidAfterMillis(dateTimeFormat.parse( + line.substring("valid-after ".length())).getTime()); + } catch (ParseException e) { + System.err.println("Could not parse valid-after timestamp in " + + "line '" + line + "' of a " + + (isConsensus ? "consensus" : "vote") + ". Skipping."); + return null; + } + } else if (line.startsWith("client-versions ")) { + status.setRecommendedClientVersions( + new TreeSet<String>(Arrays.asList( + line.split(" ")[1].split(",")))); + } else if (line.startsWith("server-versions ")) { + status.setRecommendedServerVersions( + new TreeSet<String>(Arrays.asList( + line.split(" ")[1].split(",")))); + } else if (line.startsWith("known-flags ")) { + for (String flag : line.substring("known-flags ".length()). + split(" ")) { + status.addKnownFlag(flag); + } + } else if (line.startsWith("params ")) { + if (line.length() > "params ".length()) { + for (String param : + line.substring("params ".length()).split(" ")) { + String paramName = param.split("=")[0]; + String paramValue = param.split("=")[1]; + status.addConsensusParam(paramName, paramValue); + } + } + } else if (line.startsWith("dir-source ") && !isConsensus) { + status.setNickname(line.split(" ")[1]); + status.setFingerprint(line.split(" ")[2]); + } else if (line.startsWith("dir-key-expires ")) { + try { + status.setDirKeyExpiresMillis(dateTimeFormat.parse( + line.substring("dir-key-expires ".length())).getTime()); + } catch (ParseException e) { + System.err.println("Could not parse dir-key-expires " + + "timestamp in line '" + line + "' of a " + + (isConsensus ? "consensus" : "vote") + ". Skipping."); + return null; + } + } else if (line.startsWith("r ") || + line.equals("directory-footer")) { + if (rLine != null) { + StatusEntry statusEntry = new StatusEntry(); + statusEntry.setNickname(rLine.split(" ")[1]); + statusEntry.setFingerprint(Hex.encodeHexString( + Base64.decodeBase64(rLine.split(" ")[2] + "=")). + toUpperCase()); + SortedSet<String> flags = new TreeSet<String>(); + if (sLine.length() > 2) { + for (String flag : sLine.substring(2).split(" ")) { + flags.add(flag); + } + } + statusEntry.setFlags(flags); + status.addStatusEntry(statusEntry); + } + if (line.startsWith("r ")) { + rLine = line; + } else { + break; + } + } else if (line.startsWith("s ")) { + sLine = line; + if (line.contains(" Running")) { + runningRelays++; + } + } else if (line.startsWith("v ") && + sLine.contains(" Authority")) { + String nickname = rLine.split(" ")[1]; + String versionString = line.substring(2); + status.addAuthorityVersion(nickname, versionString); + } else if (line.startsWith("w ") && !isConsensus && + line.contains(" Measured")) { + bandwidthWeights++; + } + } + br.close(); + status.setRunningRelays(runningRelays); + status.setBandwidthWeights(bandwidthWeights); + if (isConsensus) { + status.setFileName(fileNameFormat.format( + status.getValidAfterMillis()) + "-consensus"); + } else { + status.setFileName(fileNameFormat.format( + status.getValidAfterMillis()) + "-vote-" + + status.getFingerprint()); + } + } catch (IOException e) { + System.err.println("Caught an IOException while parsing a " + + (isConsensus ? "consensus" : "vote") + " string. Skipping."); + return null; + } + return status; + } +} + diff --git a/src/org/torproject/chc/Report.java b/src/org/torproject/chc/Report.java new file mode 100644 index 0000000..28483cc --- /dev/null +++ b/src/org/torproject/chc/Report.java @@ -0,0 +1,23 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.util.*; + +/* Transform findings from parsing consensuses and votes into a report of + * some form. */ +public interface Report { + + /* Process the cached consensus and corresponding votes to compare them + * to the downloaded ones. */ + public abstract void processCachedConsensus(Status cachedConsensus); + + /* Process the downloaded current consensus and corresponding votes to + * find irregularities between them. */ + public abstract void processDownloadedConsensus( + Status downloadedConsensus); + + /* Finish writing report. */ + public abstract void writeReport(); +} + diff --git a/src/org/torproject/chc/Status.java b/src/org/torproject/chc/Status.java new file mode 100644 index 0000000..51a9052 --- /dev/null +++ b/src/org/torproject/chc/Status.java @@ -0,0 +1,185 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.util.*; + +/* Contains the unparsed string and parsed fields from a network status + * consensus or vote. */ +public class Status implements Comparable<Status> { + + /* Helper methods to implement the Comparable interface; Status + * instances are compared by nickname of the publishing directory + * authorities. */ + public int compareTo(Status o) { + return this.nickname.compareTo(o.nickname); + } + public boolean equals(Object o) { + return (o instanceof Status && + this.nickname.equals(((Status) o).nickname)); + } + + /* Unparsed string that was downloaded or read from disk and that can + * be written to disk. */ + private String unparsedString; + public void setUnparsedString(String unparsedString) { + this.unparsedString = unparsedString; + } + public String getUnparsedString() { + return this.unparsedString; + } + + /* File name when writing this status to disk. */ + private String fileName; + public void setFileName(String fileName) { + this.fileName = fileName; + } + public String getFileName() { + return this.fileName; + } + + /* Votes published at the same time as this consensus; votes don't + * reference any statuses. */ + private SortedSet<Status> votes = new TreeSet<Status>(); + public void addVote(Status vote) { + this.votes.add(vote); + } + public SortedSet<Status> getVotes() { + return this.votes; + } + + /* Fingerprint of the directory authority publishing this vote; left + * empty for consensuses. */ + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + public String getFingerprint() { + return this.fingerprint; + } + + /* Nickname of the directory authority publishing this vote; left empty + * for consensuses. */ + private String nickname; + public void setNickname(String nickname) { + this.nickname= nickname; + } + public String getNickname() { + return this.nickname; + } + + /* Valid-after time in milliseconds. */ + private long validAfterMillis; + public void setValidAfterMillis(long validAfterMillis) { + this.validAfterMillis = validAfterMillis; + } + public long getValidAfterMillis() { + return this.validAfterMillis; + } + + /* Consensus parameters. */ + private SortedMap<String, String> consensusParams = + new TreeMap<String, String>(); + public void addConsensusParam(String paramName, String paramValue) { + this.consensusParams.put(paramName, paramValue); + } + public SortedMap<String, String> getConsensusParams() { + return this.consensusParams; + } + + /* Consensus methods supported by the directory authority sending a vote + * or of the produced consensus. */ + private SortedSet<Integer> consensusMethods; + public void setConsensusMethods(SortedSet<Integer> consensusMethods) { + this.consensusMethods = consensusMethods; + } + public SortedSet<Integer> getConsensusMethods() { + return this.consensusMethods; + } + + /* Recommended server versions. */ + private SortedSet<String> recommendedServerVersions; + public void setRecommendedServerVersions( + SortedSet<String> recommendedServerVersions) { + this.recommendedServerVersions = recommendedServerVersions; + } + public SortedSet<String> getRecommendedServerVersions() { + return this.recommendedServerVersions; + } + + /* Recommended client versions. */ + private SortedSet<String> recommendedClientVersions; + public void setRecommendedClientVersions( + SortedSet<String> recommendedClientVersions) { + this.recommendedClientVersions = recommendedClientVersions; + } + public SortedSet<String> getRecommendedClientVersions() { + return this.recommendedClientVersions; + } + + /* Expiration times of directory signing keys. */ + private long dirKeyExpiresMillis; + public void setDirKeyExpiresMillis(long dirKeyExpiresMillis) { + this.dirKeyExpiresMillis = dirKeyExpiresMillis; + } + public long getDirKeyExpiresMillis() { + return this.dirKeyExpiresMillis; + } + + /* Known flags by the directory authority sending a vote or of the + * produced consensus. */ + private SortedSet<String> knownFlags = new TreeSet<String>(); + public void addKnownFlag(String knownFlag) { + this.knownFlags.add(knownFlag); + } + public SortedSet<String> getKnownFlags() { + return this.knownFlags; + } + + /* Number of status entries with the Running flag. */ + private int runningRelays; + public void setRunningRelays(int runningRelays) { + this.runningRelays = runningRelays; + } + public int getRunningRelays() { + return this.runningRelays; + } + + /* Number of status entries containing bandwidth weights (only relevant + * in votes). */ + private int bandwidthWeights; + public void setBandwidthWeights(int bandwidthWeights) { + this.bandwidthWeights = bandwidthWeights; + } + public int getBandwidthWeights() { + return this.bandwidthWeights; + } + + /* Status entries contained in this status. */ + private SortedMap<String, StatusEntry> statusEntries = + new TreeMap<String, StatusEntry>(); + public void addStatusEntry(StatusEntry statusEntry) { + this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); + } + public SortedMap<String, StatusEntry> getStatusEntries() { + return this.statusEntries; + } + public boolean containsStatusEntry(String fingerprint) { + return this.statusEntries.containsKey(fingerprint); + } + public StatusEntry getStatusEntry(String fingerprint) { + return this.statusEntries.get(fingerprint); + } + + /* Versions of directory authorities (only set in a consensus). */ + private SortedMap<String, String> authorityVersions = + new TreeMap<String, String>(); + public void addAuthorityVersion(String fingerprint, + String versionString) { + this.authorityVersions.put(fingerprint, versionString); + } + public SortedMap<String, String> getAuthorityVersions() { + return this.authorityVersions; + } +} + diff --git a/src/org/torproject/chc/StatusEntry.java b/src/org/torproject/chc/StatusEntry.java new file mode 100644 index 0000000..0377937 --- /dev/null +++ b/src/org/torproject/chc/StatusEntry.java @@ -0,0 +1,48 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.util.*; + +/* Contains the parsed data from a network status entry contained in a + * network status consensus or vote. */ +public class StatusEntry implements Comparable<StatusEntry> { + + /* Helper methods to implement the Comparable interface; StatusEntry + * instances are compared by fingerprint. */ + public int compareTo(StatusEntry o) { + return this.fingerprint.compareTo(o.fingerprint); + } + public boolean equals(Object o) { + return (o instanceof StatusEntry && + this.fingerprint.equals(((StatusEntry) o).fingerprint)); + } + + /* Relay fingerprint. */ + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + public String getFingerprint() { + return this.fingerprint; + } + + /* Relay nickname. */ + private String nickname; + public void setNickname(String nickname) { + this.nickname = nickname; + } + public String getNickname() { + return this.nickname; + } + + /* Relay flags. */ + private SortedSet<String> flags; + public void setFlags(SortedSet<String> flags) { + this.flags = flags; + } + public SortedSet<String> getFlags() { + return this.flags; + } +} + diff --git a/src/org/torproject/chc/StdOutReport.java b/src/org/torproject/chc/StdOutReport.java new file mode 100644 index 0000000..cf7665d --- /dev/null +++ b/src/org/torproject/chc/StdOutReport.java @@ -0,0 +1,288 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.chc; + +import java.io.*; +import java.text.*; +import java.util.*; + +/* Check a given consensus and votes for irregularities and write results + * to stdout while rate-limiting warnings based on severity. */ +public class StdOutReport implements Report { + + /* Warning messages and the time in millis that should have passed + * since sending them out. */ + private Map<String, Long> warnings = new HashMap<String, Long>(); + + /* Date-time format to format timestamps. */ + private static SimpleDateFormat dateTimeFormat; + static { + dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + /* Cached consensus for later processing. */ + private Status cachedConsensus; + public void processCachedConsensus(Status cachedConsensus) { + this.cachedConsensus = cachedConsensus; + } + + /* Downloaded consensus and corresponding votes for later + * processing. */ + private Status downloadedConsensus; + private SortedSet<Status> downloadedVotes; + public void processDownloadedConsensus(Status downloadedConsensus) { + this.downloadedConsensus = downloadedConsensus; + this.downloadedVotes = downloadedConsensus.getVotes(); + } + + /* Check consensuses and votes for irregularities and write output to + * stdout. */ + public void writeReport() { + this.readLastWarned(); + if (this.downloadedConsensus != null) { + this.checkConsensusMethods(); + this.checkRecommendedVersions(); + this.checkConsensusParameters(); + this.checkAuthorityKeys(); + this.checkMissingVotes(); + this.checkBandwidthScanners(); + this.checkConsensusAge(this.downloadedConsensus); + } else if (this.cachedConsensus != null) { + this.checkConsensusAge(this.cachedConsensus); + } else { + this.warnings.put("No consensus known", 0L); + } + this.prepareReport(); + this.writeReportToStdOut(); + this.writeLastWarned(); + } + + /* Warning messages of the last 24 hours that is used to implement + * rate limiting. */ + private Map<String, Long> lastWarned = new HashMap<String, Long>(); + + /* Read when we last emitted a warning to rate-limit some of them. */ + private void readLastWarned() { + long now = System.currentTimeMillis(); + File lastWarnedFile = new File("stats/chc-last-warned"); + try { + if (lastWarnedFile.exists()) { + BufferedReader br = new BufferedReader(new FileReader( + lastWarnedFile)); + String line; + while ((line = br.readLine()) != null) { + if (!line.contains(": ")) { + System.err.println("Bad line in stats/chc-last-warned: '" + line + + "'. Ignoring this line."); + continue; + } + long warnedMillis = Long.parseLong(line.substring(0, + line.indexOf(": "))); + if (warnedMillis < now - 24L * 60L * 60L * 1000L) { + /* Remove warnings that are older than 24 hours. */ + continue; + } + String message = line.substring(line.indexOf(": ") + 2); + lastWarned.put(message, warnedMillis); + } + } + } catch (IOException e) { + System.err.println("Could not read file '" + + lastWarnedFile.getAbsolutePath() + "' to learn which " + + "warnings have been sent out before. Ignoring."); + } + } + + /* Check supported consensus methods of all votes. */ + private void checkConsensusMethods() { + for (Status vote : this.downloadedVotes) { + if (!vote.getConsensusMethods().contains( + this.downloadedConsensus.getConsensusMethods().last())) { + this.warnings.put(vote.getNickname() + " does not " + + "support consensus method " + + this.downloadedConsensus.getConsensusMethods().last(), + 24L * 60L * 60L * 1000L); + } + } + } + + /* Check if the recommended versions in a vote are different from the + * recommended versions in the consensus. */ + private void checkRecommendedVersions() { + for (Status vote : this.downloadedVotes) { + if (vote.getRecommendedClientVersions() != null && + !downloadedConsensus.getRecommendedClientVersions().equals( + vote.getRecommendedClientVersions())) { + StringBuilder message = new StringBuilder(); + message.append(vote.getNickname() + " recommends other " + + "client versions than the consensus:"); + for (String version : vote.getRecommendedClientVersions()) { + message.append(" " + version); + } + this.warnings.put(message.toString(), 150L * 60L * 1000L); + } + if (vote.getRecommendedServerVersions() != null && + !downloadedConsensus.getRecommendedServerVersions().equals( + vote.getRecommendedServerVersions())) { + StringBuilder message = new StringBuilder(); + message.append(vote.getNickname() + " recommends other " + + "server versions than the consensus:"); + for (String version : vote.getRecommendedServerVersions()) { + message.append(" " + version); + } + this.warnings.put(message.toString(), 150L * 60L * 1000L); + } + } + } + + /* Check if a vote contains conflicting or invalid consensus + * parameters. */ + private void checkConsensusParameters() { + Set<String> validParameters = new HashSet<String>(Arrays.asList( + ("circwindow,CircuitPriorityHalflifeMsec,refuseunknownexits," + + "cbtdisabled,cbtnummodes,cbtrecentcount,cbtmaxtimeouts," + + "cbtmincircs,cbtquantile,cbtclosequantile,cbttestfreq," + + "cbtmintimeout,cbtinitialtimeout").split(","))); + for (Status vote : this.downloadedVotes) { + Map<String, String> voteConsensusParams = + vote.getConsensusParams(); + boolean conflictOrInvalid = false; + if (voteConsensusParams == null) { + for (Map.Entry<String, String> e : + voteConsensusParams.entrySet()) { + if (!downloadedConsensus.getConsensusParams().containsKey( + e.getKey()) || + !downloadedConsensus.getConsensusParams().get(e.getKey()). + equals(e.getValue()) || + !validParameters.contains(e.getKey())) { + StringBuilder message = new StringBuilder(); + message.append(vote.getNickname() + " sets conflicting or " + + "invalid consensus parameters:"); + for (Map.Entry<String, String> p : + voteConsensusParams.entrySet()) { + message.append(" " + p.getKey() + "=" + p.getValue()); + } + this.warnings.put(message.toString(), 150L * 60L * 1000L); + break; + } + } + } + } + } + + /* Check whether any of the authority keys expire in the next 14 + * days. */ + private void checkAuthorityKeys() { + for (Status vote : this.downloadedVotes) { + long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); + if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < + System.currentTimeMillis()) { + this.warnings.put(vote.getNickname() + "'s certificate " + + "expires in the next 14 days: " + + dateTimeFormat.format(voteDirKeyExpiresMillis), + 24L * 60L * 60L * 1000L); + } + } + } + + /* Check if any votes are missing. */ + private void checkMissingVotes() { + SortedSet<String> knownAuthorities = new TreeSet<String>( + Arrays.asList(("dannenberg,dizum,gabelmoo,ides,maatuska,moria1," + + "tor26,urras").split(","))); + SortedSet<String> missingVotes = + new TreeSet<String>(knownAuthorities); + for (Status vote : this.downloadedVotes) { + missingVotes.remove(vote.getNickname()); + } + if (!missingVotes.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String missingDir : missingVotes) { + sb.append(", " + missingDir); + } + this.warnings.put("We're missing votes from the following " + + "directory authorities: " + sb.toString().substring(2), + 150L * 60L * 1000L); + } + } + + /* Check if any bandwidth scanner results are missing. */ + private void checkBandwidthScanners() { + SortedSet<String> missingBandwidthScanners = new TreeSet<String>( + Arrays.asList("ides,urras,moria1,gabelmoo".split(","))); + for (Status vote : this.downloadedVotes) { + if (vote.getBandwidthWeights() > 0) { + missingBandwidthScanners.remove(vote.getNickname()); + } + } + if (!missingBandwidthScanners.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String dir : missingBandwidthScanners) { + sb.append(", " + dir); + } + this.warnings.put("The following directory authorities are not " + + "reporting bandwidth scanner results: " + + sb.toString().substring(2), 150L * 60L * 1000L); + } + } + + /* Check if the most recent consensus is older than 3 hours. */ + private void checkConsensusAge(Status consensus) { + if (consensus.getValidAfterMillis() < + System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { + this.warnings.put("The last known consensus published at " + + dateTimeFormat.format(consensus.getValidAfterMillis()) + + " is more than 3 hours old", 0L); + } + } + + /* Prepare a report to be written to stdout. */ + private String preparedReport = null; + private void prepareReport() { + long now = System.currentTimeMillis(); + boolean writeReport = false; + for (Map.Entry<String, Long> e : this.warnings.entrySet()) { + String message = e.getKey(); + long warnInterval = e.getValue(); + if (!lastWarned.containsKey(message) || + lastWarned.get(message) + warnInterval < now) { + writeReport = true; + } + } + if (writeReport) { + StringBuilder sb = new StringBuilder(); + for (String message : this.warnings.keySet()) { + this.lastWarned.put(message, now); + sb.append("\n\n" + message); + } + this.preparedReport = sb.toString().substring(2); + } + } + + /* Write report to stdout. */ + private void writeReportToStdOut() { + if (this.preparedReport != null) { + System.out.println(this.preparedReport); + } + } + + /* Write timestamps when warnings were last sent to disk. */ + private void writeLastWarned() { + File lastWarnedFile = new File("stats/chc-last-warned"); + try { + lastWarnedFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + lastWarnedFile)); + for (Map.Entry<String, Long> e : lastWarned.entrySet()) { + bw.write(String.valueOf(e.getValue()) + ": " + e.getKey() + "\n"); + } + bw.close(); + } catch (IOException e) { + System.err.println("Could not write file '" + + lastWarnedFile.getAbsolutePath() + "' to remember which " + + "warnings have been sent out before. Ignoring."); + } + } +} + diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java index 1c57392..b8f251e 100644 --- a/src/org/torproject/ernie/cron/Configuration.java +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -24,8 +24,6 @@ public class Configuration { "jdbc:postgresql://localhost/tordir?user=metrics&password=password"; private boolean writeRelayDescriptorsRawFiles = false; private String relayDescriptorRawFilesDirectory = "pg-import/"; - private boolean writeConsensusHealth = false; - private boolean writeNagiosStatusFile = false; private boolean writeBridgeStats = false; private boolean importWriteTorperfStats = false; private String torperfDirectory = "torperf/"; @@ -78,12 +76,6 @@ public class Configuration { line.split(" ")[1]) != 0; } else if (line.startsWith("RelayDescriptorRawFilesDirectory")) { this.relayDescriptorRawFilesDirectory = line.split(" ")[1]; - } else if (line.startsWith("WriteConsensusHealth")) { - this.writeConsensusHealth = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("WriteNagiosStatusFile")) { - this.writeNagiosStatusFile = Integer.parseInt( - line.split(" ")[1]) != 0; } else if (line.startsWith("WriteBridgeStats")) { this.writeBridgeStats = Integer.parseInt( line.split(" ")[1]) != 0; @@ -156,12 +148,6 @@ public class Configuration { public String getRelayDescriptorRawFilesDirectory() { return this.relayDescriptorRawFilesDirectory; } - public boolean getWriteConsensusHealth() { - return this.writeConsensusHealth; - } - public boolean getWriteNagiosStatusFile() { - return this.writeNagiosStatusFile; - } public boolean getWriteBridgeStats() { return this.writeBridgeStats; } diff --git a/src/org/torproject/ernie/cron/ConsensusHealthChecker.java b/src/org/torproject/ernie/cron/ConsensusHealthChecker.java deleted file mode 100644 index 4e2adc4..0000000 --- a/src/org/torproject/ernie/cron/ConsensusHealthChecker.java +++ /dev/null @@ -1,993 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.cron; - -import java.io.*; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.logging.*; -import org.apache.commons.codec.binary.*; - -/* - * TODO Possible extensions: - * - Include consensus signatures and tell by which Tor versions the - * consensus will be accepted (and by which not) - */ -public class ConsensusHealthChecker { - - private String mostRecentValidAfterTime = null; - - private byte[] mostRecentConsensus = null; - - /** - * Logger for this class. - */ - private Logger logger; - - private File statsDirectory; - - private List<String> nagiosWarnings = new ArrayList<String>(), - nagiosCriticals = new ArrayList<String>(), - nagiosUnknowns = new ArrayList<String>(); - - private SortedMap<String, byte[]> mostRecentVotes = - new TreeMap<String, byte[]>(); - - public ConsensusHealthChecker(File statsDirectory) { - - /* Initialize logger. */ - this.logger = Logger.getLogger( - ConsensusHealthChecker.class.getName()); - - this.statsDirectory = statsDirectory; - } - - public void processConsensus(String validAfterTime, byte[] data) { - /* Do we already know a consensus and/or vote(s)? */ - if (this.mostRecentValidAfterTime != null) { - int compareKnownToNew = - this.mostRecentValidAfterTime.compareTo(validAfterTime); - if (compareKnownToNew > 0) { - /* The consensus or vote(s) we know are more recent than this - * consensus. No need to store it. */ - return; - } else if (compareKnownToNew < 0) { - /* This consensus is newer than the known consensus or vote(s). - * Discard all known votes and overwrite the consensus below. */ - this.mostRecentVotes.clear(); - } - } - /* Store this consensus. */ - this.mostRecentValidAfterTime = validAfterTime; - this.mostRecentConsensus = data; - } - - public void processVote(String validAfterTime, String dirSource, - byte[] data) { - if (this.mostRecentValidAfterTime == null || - this.mostRecentValidAfterTime.compareTo(validAfterTime) < 0) { - /* This vote is more recent than the known consensus. Discard the - * consensus and all currently known votes. */ - this.mostRecentValidAfterTime = validAfterTime; - this.mostRecentVotes.clear(); - this.mostRecentConsensus = null; - } - if (this.mostRecentValidAfterTime.equals(validAfterTime)) { - /* Store this vote which belongs to the known consensus and/or - * other votes. */ - this.mostRecentVotes.put(dirSource, data); - } - } - - public void writeStatusWebsite() { - - /* If we don't have any consensus, we cannot write useful consensus - * health information to the website. Do not overwrite existing page - * with a warning, because we might just not have learned about a new - * consensus in this execution. */ - if (this.mostRecentConsensus == null) { - nagiosCriticals.add("No consensus known"); - return; - } - - /* Prepare parsing dates. */ - SimpleDateFormat dateTimeFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - StringBuilder knownFlagsResults = new StringBuilder(); - StringBuilder numRelaysVotesResults = new StringBuilder(); - StringBuilder consensusMethodsResults = new StringBuilder(); - StringBuilder versionsResults = new StringBuilder(); - StringBuilder paramsResults = new StringBuilder(); - StringBuilder authorityKeysResults = new StringBuilder(); - StringBuilder bandwidthScannersResults = new StringBuilder(); - StringBuilder authorityVersionsResults = new StringBuilder(); - SortedSet<String> allKnownFlags = new TreeSet<String>(); - SortedSet<String> allKnownVotes = new TreeSet<String>(); - SortedSet<String> runningBandwidthScanners = new TreeSet<String>(); - SortedMap<String, String> consensusAssignedFlags = - new TreeMap<String, String>(); - SortedMap<String, SortedSet<String>> votesAssignedFlags = - new TreeMap<String, SortedSet<String>>(); - SortedMap<String, String> votesKnownFlags = - new TreeMap<String, String>(); - SortedMap<String, SortedMap<String, Integer>> flagsAgree = - new TreeMap<String, SortedMap<String, Integer>>(); - SortedMap<String, SortedMap<String, Integer>> flagsLost = - new TreeMap<String, SortedMap<String, Integer>>(); - SortedMap<String, SortedMap<String, Integer>> flagsMissing = - new TreeMap<String, SortedMap<String, Integer>>(); - - /* Read consensus and parse all information that we want to compare to - * votes. */ - String consensusConsensusMethod = null, consensusKnownFlags = null, - consensusClientVersions = null, consensusServerVersions = null, - consensusParams = null, rLineTemp = null, sLineTemp = null; - int consensusTotalRelays = 0, consensusRunningRelays = 0; - try { - BufferedReader br = new BufferedReader(new StringReader(new String( - this.mostRecentConsensus))); - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("consensus-method ")) { - consensusConsensusMethod = line; - } else if (line.startsWith("client-versions ")) { - consensusClientVersions = line; - } else if (line.startsWith("server-versions ")) { - consensusServerVersions = line; - } else if (line.startsWith("known-flags ")) { - consensusKnownFlags = line; - } else if (line.startsWith("params ")) { - consensusParams = line; - } else if (line.startsWith("r ")) { - rLineTemp = line; - } else if (line.startsWith("s ")) { - sLineTemp = line; - consensusTotalRelays++; - if (line.contains(" Running")) { - consensusRunningRelays++; - } - consensusAssignedFlags.put(Hex.encodeHexString( - Base64.decodeBase64(rLineTemp.split(" ")[2] + "=")). - toUpperCase() + " " + rLineTemp.split(" ")[1], line); - } else if (line.startsWith("v ") && - sLineTemp.contains(" Authority")) { - authorityVersionsResults.append(" <tr>\n" - + " <td>" + rLineTemp.split(" ")[1] + "</td>\n" - + " <td>" + line.substring(2) + "</td>\n" - + " </tr>\n"); - } - } - br.close(); - } catch (IOException e) { - /* There should be no I/O taking place when reading a String. */ - } - - /* Read votes and parse all information to compare with the - * consensus. */ - for (byte[] voteBytes : this.mostRecentVotes.values()) { - String voteConsensusMethods = null, voteKnownFlags = null, - voteClientVersions = null, voteServerVersions = null, - voteParams = null, dirSource = null, voteDirKeyExpires = null; - int voteTotalRelays = 0, voteRunningRelays = 0, - voteContainsBandwidthWeights = 0; - try { - BufferedReader br = new BufferedReader(new StringReader( - new String(voteBytes))); - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("consensus-methods ")) { - voteConsensusMethods = line; - } else if (line.startsWith("client-versions ")) { - voteClientVersions = line; - } else if (line.startsWith("server-versions ")) { - voteServerVersions = line; - } else if (line.startsWith("known-flags ")) { - voteKnownFlags = line; - } else if (line.startsWith("params ")) { - voteParams = line; - } else if (line.startsWith("dir-source ")) { - dirSource = line.split(" ")[1]; - allKnownVotes.add(dirSource); - } else if (line.startsWith("dir-key-expires ")) { - voteDirKeyExpires = line; - } else if (line.startsWith("r ")) { - rLineTemp = line; - } else if (line.startsWith("s ")) { - voteTotalRelays++; - if (line.contains(" Running")) { - voteRunningRelays++; - } - String relayKey = Hex.encodeHexString(Base64.decodeBase64( - rLineTemp.split(" ")[2] + "=")).toUpperCase() + " " - + rLineTemp.split(" ")[1]; - SortedSet<String> sLines = null; - if (votesAssignedFlags.containsKey(relayKey)) { - sLines = votesAssignedFlags.get(relayKey); - } else { - sLines = new TreeSet<String>(); - votesAssignedFlags.put(relayKey, sLines); - } - sLines.add(dirSource + " " + line); - } else if (line.startsWith("w ")) { - if (line.contains(" Measured")) { - voteContainsBandwidthWeights++; - } - } - } - br.close(); - } catch (IOException e) { - /* There should be no I/O taking place when reading a String. */ - } - - /* Write known flags. */ - knownFlagsResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteKnownFlags + "</td>\n" - + " </tr>\n"); - votesKnownFlags.put(dirSource, voteKnownFlags); - for (String flag : voteKnownFlags.substring( - "known-flags ".length()).split(" ")) { - allKnownFlags.add(flag); - } - - /* Write number of relays voted about. */ - numRelaysVotesResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteTotalRelays + " total</td>\n" - + " <td>" + voteRunningRelays + " Running</td>\n" - + " </tr>\n"); - - /* Write supported consensus methods. */ - if (!voteConsensusMethods.contains(consensusConsensusMethod. - split(" ")[1])) { - consensusMethodsResults.append(" <tr>\n" - + " <td><font color="red">" + dirSource - + "</font></td>\n" - + " <td><font color="red">" - + voteConsensusMethods + "</font></td>\n" - + " </tr>\n"); - this.logger.warning(dirSource + " does not support consensus " - + "method " + consensusConsensusMethod.split(" ")[1] + ": " - + voteConsensusMethods); - nagiosWarnings.add(dirSource + " does not support consensus " - + "method " + consensusConsensusMethod.split(" ")[1]); - } else { - consensusMethodsResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteConsensusMethods + "</td>\n" - + " </tr>\n"); - this.logger.fine(dirSource + " supports consensus method " - + consensusConsensusMethod.split(" ")[1] + ": " - + voteConsensusMethods); - } - - /* Write recommended versions. */ - if (voteClientVersions == null) { - /* Not a versioning authority. */ - } else if (!voteClientVersions.equals(consensusClientVersions)) { - versionsResults.append(" <tr>\n" - + " <td><font color="red">" + dirSource - + "</font></td>\n" - + " <td><font color="red">" - + voteClientVersions + "</font></td>\n" - + " </tr>\n"); - this.logger.warning(dirSource + " recommends other client " - + "versions than the consensus: " + voteClientVersions); - nagiosWarnings.add(dirSource + " recommends other client " - + "versions than the consensus"); - } else { - versionsResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteClientVersions + "</td>\n" - + " </tr>\n"); - this.logger.fine(dirSource + " recommends the same client " - + "versions as the consensus: " + voteClientVersions); - } - if (voteServerVersions == null) { - /* Not a versioning authority. */ - } else if (!voteServerVersions.equals(consensusServerVersions)) { - versionsResults.append(" <tr>\n" - + " <td></td>\n" - + " <td><font color="red">" - + voteServerVersions + "</font></td>\n" - + " </tr>\n"); - this.logger.warning(dirSource + " recommends other server " - + "versions than the consensus: " + voteServerVersions); - nagiosWarnings.add(dirSource + " recommends other server " - + "versions than the consensus"); - } else { - versionsResults.append(" <tr>\n" - + " <td></td>\n" - + " <td>" + voteServerVersions + "</td>\n" - + " </tr>\n"); - this.logger.fine(dirSource + " recommends the same server " - + "versions as the consensus: " + voteServerVersions); - } - - /* Write consensus parameters. */ - boolean conflictOrInvalid = false; - Set<String> validParameters = new HashSet<String>(Arrays.asList( - ("circwindow,CircuitPriorityHalflifeMsec,refuseunknownexits," - + "cbtdisabled,cbtnummodes,cbtrecentcount,cbtmaxtimeouts," - + "cbtmincircs,cbtquantile,cbtclosequantile,cbttestfreq," - + "cbtmintimeout,cbtinitialtimeout").split(","))); - if (voteParams == null) { - /* Authority doesn't set consensus parameters. */ - } else { - for (String param : voteParams.split(" ")) { - if (!param.equals("params") && - (!consensusParams.contains(param) || - !validParameters.contains(param.split("=")[0]))) { - conflictOrInvalid = true; - break; - } - } - } - if (conflictOrInvalid) { - paramsResults.append(" <tr>\n" - + " <td><font color="red">" + dirSource - + "</font></td>\n" - + " <td><font color="red">" - + voteParams + "</font></td>\n" - + " </tr>\n"); - this.logger.warning(dirSource + " sets conflicting or invalid " - + "consensus parameters: " + voteParams); - nagiosWarnings.add(dirSource + " sets conflicting or invalid " - + "consensus parameters"); - } else { - paramsResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteParams + "</td>\n" - + " </tr>\n"); - this.logger.fine(dirSource + " sets only non-conflicting and " - + "valid consensus parameters: " + voteParams); - } - - /* Write authority key expiration date. */ - if (voteDirKeyExpires != null) { - boolean expiresIn14Days = false; - try { - expiresIn14Days = (System.currentTimeMillis() - + 14L * 24L * 60L * 60L * 1000L > - dateTimeFormat.parse(voteDirKeyExpires.substring( - "dir-key-expires ".length())).getTime()); - } catch (ParseException e) { - /* Can't parse the timestamp? Whatever. */ - } - if (expiresIn14Days) { - authorityKeysResults.append(" <tr>\n" - + " <td><font color="red">" + dirSource - + "</font></td>\n" - + " <td><font color="red">" - + voteDirKeyExpires + "</font></td>\n" - + " </tr>\n"); - this.logger.warning(dirSource + "'s certificate expires in the " - + "next 14 days: " + voteDirKeyExpires); - nagiosWarnings.add(dirSource + "'s certificate expires in the " - + "next 14 days"); - } else { - authorityKeysResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteDirKeyExpires + "</td>\n" - + " </tr>\n"); - this.logger.fine(dirSource + "'s certificate does not " - + "expire in the next 14 days: " + voteDirKeyExpires); - } - } - - /* Write results for bandwidth scanner status. */ - if (voteContainsBandwidthWeights > 0) { - bandwidthScannersResults.append(" <tr>\n" - + " <td>" + dirSource + "</td>\n" - + " <td>" + voteContainsBandwidthWeights - + " Measured values in w lines</td>\n" - + " </tr>\n"); - runningBandwidthScanners.add(dirSource); - } - } - - /* Check if we're missing a vote. TODO make this configurable */ - SortedSet<String> knownAuthorities = new TreeSet<String>( - Arrays.asList(("dannenberg,dizum,gabelmoo,ides,maatuska,moria1," - + "tor26,urras").split(","))); - for (String dir : allKnownVotes) { - knownAuthorities.remove(dir); - } - if (!knownAuthorities.isEmpty()) { - StringBuilder sb = new StringBuilder(); - for (String dir : knownAuthorities) { - sb.append(", " + dir); - } - this.logger.warning("We're missing votes from the following " - + "directory authorities: " + sb.toString().substring(2)); - nagiosWarnings.add("We're missing votes from the following " - + "directory authorities: " + sb.toString().substring(2)); - } - - /* Check if less than 4 bandwidth scanners are running. TODO make this - * configurable */ - SortedSet<String> knownBandwidthScanners = new TreeSet<String>( - Arrays.asList("ides,urras,moria1,gabelmoo".split(","))); - for (String dir : runningBandwidthScanners) { - knownBandwidthScanners.remove(dir); - } - if (!knownBandwidthScanners.isEmpty()) { - StringBuilder sb = new StringBuilder(); - for (String dir : knownBandwidthScanners) { - sb.append(", " + dir); - } - String message = "The following directory authorities are not " - + "reporting bandwidth scanner results: " - + sb.toString().substring(2); - this.logger.warning(message); - if (runningBandwidthScanners.size() >= 3) { - nagiosWarnings.add(message); - } else { - nagiosCriticals.add(message); - } - } - - try { - - /* Keep the past two consensus health statuses. */ - File file0 = new File("website/consensus-health.html"); - File file1 = new File("website/consensus-health-1.html"); - File file2 = new File("website/consensus-health-2.html"); - if (file2.exists()) { - file2.delete(); - } - if (file1.exists()) { - file1.renameTo(file2); - } - if (file0.exists()) { - file0.renameTo(file1); - } - - /* Start writing web page. */ - BufferedWriter bw = new BufferedWriter( - new FileWriter("website/consensus-health.html")); - bw.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 " - + "Transitional//EN\">\n" - + "<html>\n" - + " <head>\n" - + " <title>Tor Metrics Portal: Consensus health</title>\n" - + " <meta http-equiv="content-type" content="text/html; " - + "charset=ISO-8859-1">\n" - + " <link href="/css/stylesheet-ltr.css" type="text/css" " - + "rel="stylesheet">\n" - + " <link href="/images/favicon.ico" " - + "type="image/x-icon" rel="shortcut icon">\n" - + " </head>\n" - + " <body>\n" - + " <div class="center">\n" - + " <table class="banner" border="0" " - + "cellpadding="0" cellspacing="0" summary="">\n" - + " <tr>\n" - + " <td class="banner-left"><a " - + "href="/index.html"><img src="/images/top-left.png" " - + "alt="Click to go to home page" width="193" " - + "height="79"></a></td>\n" - + " <td class="banner-middle">\n" - + " <a href="/">Home</a>\n" - + " <a href="graphs.html">Graphs</a>\n" - + " <a href="research.html">Research</a>\n" - + " <a href="status.html">Status</a>\n" - + " <br>\n" - + " <font size="2">\n" - + " <a href="networkstatus.html">Network Status</a>\n" - + " <a href="exonerator.html">ExoneraTor</a>\n" - + " <a href="relay-search.html">Relay Search</a>\n" - + " <a class="current">Consensus Health</a>\n" - + " </font>\n" - + " </td>\n" - + " <td class="banner-right"></td>\n" - + " </tr>\n" - + " </table>\n" - + " <div class="main-column">\n" - + " <h2>Tor Metrics Portal: Consensus Health</h2>\n" - + " <br>\n" - + " <p>This page shows statistics about the current " - + "consensus and votes to facilitate debugging of the " - + "directory consensus process.</p>\n"); - - /* Write valid-after time. */ - bw.write(" <br>\n" - + " <h3>Valid-after time</h3>\n" - + " <br>\n" - + " <p>Consensus was published "); - boolean consensusIsStale = false; - try { - consensusIsStale = System.currentTimeMillis() - - 3L * 60L * 60L * 1000L > - dateTimeFormat.parse(this.mostRecentValidAfterTime).getTime(); - } catch (ParseException e) { - /* Can't parse the timestamp? Whatever. */ - } - if (consensusIsStale) { - bw.write("<font color="red">" + this.mostRecentValidAfterTime - + "</font>"); - this.logger.warning("The last consensus published at " - + this.mostRecentValidAfterTime + " is more than 3 hours " - + "old."); - nagiosCriticals.add("The last consensus published at " - + this.mostRecentValidAfterTime + " is more than 3 hours " - + "old"); - } else { - bw.write(this.mostRecentValidAfterTime); - this.logger.fine("The last consensus published at " - + this.mostRecentValidAfterTime + " is less than 3 hours " - + "old."); - } - bw.write(". <i>Note that it takes " - + "15 to 30 minutes for the metrics portal to learn about " - + "new consensus and votes and process them.</i></p>\n"); - - /* Write known flags. */ - bw.write(" <br>\n" - + " <h3>Known flags</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - if (knownFlagsResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); - } else { - bw.write(knownFlagsResults.toString()); - } - bw.write(" <tr>\n" - + " <td><font color="blue">consensus</font>" - + "</td>\n" - + " <td><font color="blue">" - + consensusKnownFlags + "</font></td>\n" - + " </tr>\n"); - bw.write(" </table>\n"); - - /* Write number of relays voted about. */ - bw.write(" <br>\n" - + " <h3>Number of relays voted about</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="320">\n" - + " <col width="320">\n" - + " </colgroup>\n"); - if (numRelaysVotesResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td><td></td></tr>\n"); - } else { - bw.write(numRelaysVotesResults.toString()); - } - bw.write(" <tr>\n" - + " <td><font color="blue">consensus</font>" - + "</td>\n" - + " <td><font color="blue">" - + consensusTotalRelays + " total</font></td>\n" - + " <td><font color="blue">" - + consensusRunningRelays + " Running</font></td>\n" - + " </tr>\n"); - bw.write(" </table>\n"); - - /* Write consensus methods. */ - bw.write(" <br>\n" - + " <h3>Consensus methods</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - if (consensusMethodsResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); - } else { - bw.write(consensusMethodsResults.toString()); - } - bw.write(" <tr>\n" - + " <td><font color="blue">consensus</font>" - + "</td>\n" - + " <td><font color="blue">" - + consensusConsensusMethod + "</font></td>\n" - + " </tr>\n"); - bw.write(" </table>\n"); - - /* Write recommended versions. */ - bw.write(" <br>\n" - + " <h3>Recommended versions</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - if (versionsResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); - } else { - bw.write(versionsResults.toString()); - } - bw.write(" <tr>\n" - + " <td><font color="blue">consensus</font>" - + "</td>\n" - + " <td><font color="blue">" - + consensusClientVersions + "</font></td>\n" - + " </tr>\n"); - bw.write(" <tr>\n" - + " <td></td>\n" - + " <td><font color="blue">" - + consensusServerVersions + "</font></td>\n" - + " </tr>\n"); - bw.write(" </table>\n"); - - /* Write consensus parameters. */ - bw.write(" <br>\n" - + " <h3>Consensus parameters</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - if (paramsResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); - } else { - bw.write(paramsResults.toString()); - } - bw.write(" <tr>\n" - + " <td><font color="blue">consensus</font>" - + "</td>\n" - + " <td><font color="blue">" - + consensusParams + "</font></td>\n" - + " </tr>\n"); - bw.write(" </table>\n"); - - /* Write authority keys. */ - bw.write(" <br>\n" - + " <h3>Authority keys</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - if (authorityKeysResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); - } else { - bw.write(authorityKeysResults.toString()); - } - bw.write(" </table>\n" - + " <br>\n" - + " <p><i>Note that expiration dates of legacy keys are " - + "not included in votes and therefore not listed here!</i>" - + "</p>\n"); - - /* Write bandwidth scanner status. */ - bw.write(" <br>\n" - + " <h3>Bandwidth scanner status</h3>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - if (bandwidthScannersResults.length() < 1) { - bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); - } else { - bw.write(bandwidthScannersResults.toString()); - } - bw.write(" </table>\n"); - - /* Write authority versions. */ - bw.write(" <br>\n" - + " <h3>Authority versions</h3>\n" - + " <br>\n"); - if (authorityVersionsResults.length() < 1) { - bw.write(" <p>(No relays with Authority flag found.)" - + "</p>\n"); - } else { - bw.write(" <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="640">\n" - + " </colgroup>\n"); - bw.write(authorityVersionsResults.toString()); - bw.write(" </table>\n" - + " <br>\n" - + " <p><i>Note that this list of relays with the " - + "Authority flag may be different from the list of v3 " - + "directory authorities!</i></p>\n"); - } - - /* Write (huge) table with all flags. */ - bw.write(" <br>\n" - + " <h3>Relay flags</h3>\n" - + " <br>\n" - + " <p>The semantics of flags written in the table is " - + "as follows:</p>\n" - + " <ul>\n" - + " <li><b>In vote and consensus:</b> Flag in vote " - + "matches flag in consensus, or relay is not listed in " - + "consensus (because it doesn't have the Running " - + "flag)</li>\n" - + " <li><b><font color="red">Only in " - + "vote:</font></b> Flag in vote, but missing in the " - + "consensus, because there was no majority for the flag or " - + "the flag was invalidated (e.g., Named gets invalidated by " - + "Unnamed)</li>\n" - + " <li><b><font color="gray"><s>Only in " - + "consensus:</s></font></b> Flag in consensus, but missing " - + "in a vote of a directory authority voting on this " - + "flag</li>\n" - + " <li><b><font color="blue">In " - + "consensus:</font></b> Flag in consensus</li>\n" - + " </ul>\n" - + " <br>\n" - + " <p>See also the summary below the table.</p>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="120">\n" - + " <col width="80">\n"); - for (int i = 0; i < allKnownVotes.size(); i++) { - bw.write(" <col width="" - + (640 / allKnownVotes.size()) + "">\n"); - } - bw.write(" </colgroup>\n"); - int linesWritten = 0; - for (Map.Entry<String, SortedSet<String>> e : - votesAssignedFlags.entrySet()) { - if (linesWritten++ % 10 == 0) { - bw.write(" <tr><td><br><b>Fingerprint</b></td>" - + "<td><br><b>Nickname</b></td>\n"); - for (String dir : allKnownVotes) { - String shortDirName = dir.length() > 6 ? - dir.substring(0, 5) + "." : dir; - bw.write("<td><br><b>" + shortDirName + "</b></td>"); - } - bw.write("<td><br><b>consensus</b></td></tr>\n"); - } - String relayKey = e.getKey(); - SortedSet<String> votes = e.getValue(); - String fingerprint = relayKey.split(" ")[0].substring(0, 8); - String nickname = relayKey.split(" ")[1]; - bw.write(" <tr>\n"); - if (consensusAssignedFlags.containsKey(relayKey) && - consensusAssignedFlags.get(relayKey).contains(" Named") && - !Character.isDigit(nickname.charAt(0))) { - bw.write(" <td id="" + nickname - + ""><a href="relay.html?fingerprint=" - + relayKey.split(" ")[0] + "" target="_blank">" - + fingerprint + "</a></td>\n"); - } else { - bw.write(" <td><a href="relay.html?fingerprint=" - + fingerprint + "" target="_blank">" + fingerprint - + "</a></td>\n"); - } - bw.write(" <td>" + nickname + "</td>\n"); - SortedSet<String> relevantFlags = new TreeSet<String>(); - for (String vote : votes) { - String[] parts = vote.split(" "); - for (int j = 2; j < parts.length; j++) { - relevantFlags.add(parts[j]); - } - } - String consensusFlags = null; - if (consensusAssignedFlags.containsKey(relayKey)) { - consensusFlags = consensusAssignedFlags.get(relayKey); - String[] parts = consensusFlags.split(" "); - for (int j = 1; j < parts.length; j++) { - relevantFlags.add(parts[j]); - } - } - for (String dir : allKnownVotes) { - String flags = null; - for (String vote : votes) { - if (vote.startsWith(dir)) { - flags = vote; - break; - } - } - if (flags != null) { - votes.remove(flags); - bw.write(" <td>"); - int flagsWritten = 0; - for (String flag : relevantFlags) { - bw.write(flagsWritten++ > 0 ? "<br>" : ""); - SortedMap<String, SortedMap<String, Integer>> sums = null; - if (flags.contains(" " + flag)) { - if (consensusFlags == null || - consensusFlags.contains(" " + flag)) { - bw.write(flag); - sums = flagsAgree; - } else { - bw.write("<font color="red">" + flag + "</font>"); - sums = flagsLost; - } - } else if (consensusFlags != null && - votesKnownFlags.get(dir).contains(" " + flag) && - consensusFlags.contains(" " + flag)) { - bw.write("<font color="gray"><s>" + flag - + "</s></font>"); - sums = flagsMissing; - } - if (sums != null) { - SortedMap<String, Integer> sum = null; - if (sums.containsKey(dir)) { - sum = sums.get(dir); - } else { - sum = new TreeMap<String, Integer>(); - sums.put(dir, sum); - } - sum.put(flag, sum.containsKey(flag) ? - sum.get(flag) + 1 : 1); - } - } - bw.write("</td>\n"); - } else { - bw.write(" <td></td>\n"); - } - } - if (consensusFlags != null) { - bw.write(" <td>"); - int flagsWritten = 0; - for (String flag : relevantFlags) { - bw.write(flagsWritten++ > 0 ? "<br>" : ""); - if (consensusFlags.contains(" " + flag)) { - bw.write("<font color="blue">" + flag + "</font>"); - } - } - bw.write("</td>\n"); - } else { - bw.write(" <td></td>\n"); - } - bw.write(" </tr>\n"); - } - bw.write(" </table>\n"); - - /* Write summary of overlap between votes and consensus. */ - bw.write(" <br>\n" - + " <h3>Overlap between votes and consensus</h3>\n" - + " <br>\n" - + " <p>The semantics of columns is similar to the " - + "table above:</p>\n" - + " <ul>\n" - + " <li><b>In vote and consensus:</b> Flag in vote " - + "matches flag in consensus, or relay is not listed in " - + "consensus (because it doesn't have the Running " - + "flag)</li>\n" - + " <li><b><font color="red">Only in " - + "vote:</font></b> Flag in vote, but missing in the " - + "consensus, because there was no majority for the flag or " - + "the flag was invalidated (e.g., Named gets invalidated by " - + "Unnamed)</li>\n" - + " <li><b><font color="gray"><s>Only in " - + "consensus:</s></font></b> Flag in consensus, but missing " - + "in a vote of a directory authority voting on this " - + "flag</li>\n" - + " </ul>\n" - + " <br>\n" - + " <table border="0" cellpadding="4" " - + "cellspacing="0" summary="">\n" - + " <colgroup>\n" - + " <col width="160">\n" - + " <col width="210">\n" - + " <col width="210">\n" - + " <col width="210">\n" - + " </colgroup>\n"); - bw.write(" <tr><td></td><td><b>Only in vote</b></td>" - + "<td><b>In vote and consensus</b></td>" - + "<td><b>Only in consensus</b></td>\n"); - for (String dir : allKnownVotes) { - boolean firstFlagWritten = false; - String[] flags = votesKnownFlags.get(dir).substring( - "known-flags ".length()).split(" "); - for (String flag : flags) { - bw.write(" <tr>\n"); - if (firstFlagWritten) { - bw.write(" <td></td>\n"); - } else { - bw.write(" <td>" + dir + "</td>\n"); - firstFlagWritten = true; - } - if (flagsLost.containsKey(dir) && - flagsLost.get(dir).containsKey(flag)) { - bw.write(" <td><font color="red"> " - + flagsLost.get(dir).get(flag) + " " + flag - + "</font></td>\n"); - } else { - bw.write(" <td></td>\n"); - } - if (flagsAgree.containsKey(dir) && - flagsAgree.get(dir).containsKey(flag)) { - bw.write(" <td>" + flagsAgree.get(dir).get(flag) - + " " + flag + "</td>\n"); - } else { - bw.write(" <td></td>\n"); - } - if (flagsMissing.containsKey(dir) && - flagsMissing.get(dir).containsKey(flag)) { - bw.write(" <td><font color="gray"><s>" - + flagsMissing.get(dir).get(flag) + " " + flag - + "</s></font></td>\n"); - } else { - bw.write(" <td></td>\n"); - } - bw.write(" </tr>\n"); - } - } - bw.write(" </table>\n"); - - /* Finish writing. */ - bw.write(" </div>\n" - + " </div>\n" - + " <div class="bottom" id="bottom">\n" - + " <p>This material is supported in part by the " - + "National Science Foundation under Grant No. " - + "CNS-0959138. Any opinions, finding, and conclusions " - + "or recommendations expressed in this material are " - + "those of the author(s) and do not necessarily reflect " - + "the views of the National Science Foundation.</p>\n" - + " <p>"Tor" and the "Onion Logo" are <a " - + "href="https://www.torproject.org/docs/trademark-faq.html" - + ".en">" - + "registered trademarks</a> of The Tor Project, " - + "Inc.</p>\n" - + " <p>Data on this site is freely available under a " - + "<a href="http://creativecommons.org/publicdomain/" - + "zero/1.0/">CC0 no copyright declaration</a>: To the " - + "extent possible under law, the Tor Project has waived " - + "all copyright and related or neighboring rights in " - + "the data. Graphs are licensed under a <a " - + "href="http://creativecommons.org/licenses/by/3.0/" - + "us/">Creative Commons Attribution 3.0 United States " - + "License</a>.</p>\n" - + " </div>\n" - + " </body>\n" - + "</html>"); - bw.close(); - - } catch (IOException e) { - } - } - - public void writeNagiosStatusFile() { - try { - statsDirectory.mkdirs(); - File nagiosStatusFile = new File(statsDirectory, - "consensus-health"); - BufferedWriter bw = new BufferedWriter(new FileWriter( - nagiosStatusFile)); - if (!nagiosUnknowns.isEmpty()) { - bw.write("UNKNOWN\nUNKNOWN"); - } else if (!nagiosCriticals.isEmpty()) { - bw.write("CRITICAL\nCRITICAL"); - } else if (!nagiosWarnings.isEmpty()) { - bw.write("WARNING\nWARNING"); - } else { - bw.write("OK\nOK"); - } - for (String message : nagiosUnknowns) { - bw.write(" " + message + ";"); - } - for (String message : nagiosCriticals) { - bw.write(" " + message + ";"); - } - for (String message : nagiosWarnings) { - bw.write(" " + message + ";"); - } - bw.write("\n"); - bw.close(); - } catch (IOException e) { - } - } -} - diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java index b24df37..4889fc2 100644 --- a/src/org/torproject/ernie/cron/Main.java +++ b/src/org/torproject/ernie/cron/Main.java @@ -38,10 +38,6 @@ public class Main { new BridgeStatsFileHandler( config.getRelayDescriptorDatabaseJDBC()) : null;

- // Prepare consensus health checker - ConsensusHealthChecker chc = config.getWriteConsensusHealth() ? - new ConsensusHealthChecker(statsDirectory) : null; - // Prepare writing relay descriptors to database RelayDescriptorDatabaseImporter rddi = config.getWriteRelayDescriptorDatabase() || @@ -54,8 +50,8 @@ public class Main {

// Prepare relay descriptor parser (only if we are writing the // consensus-health page to disk) - RelayDescriptorParser rdp = chc != null || rddi != null ? - new RelayDescriptorParser(chc, rddi, bsfh) : null; + RelayDescriptorParser rdp = rddi != null ? + new RelayDescriptorParser(rddi, bsfh) : null;

// Import relay descriptors if (rdp != null) { @@ -72,15 +68,6 @@ public class Main { rddi.closeConnection(); }

- // Write consensus health website - if (chc != null) { - chc.writeStatusWebsite(); - if (config.getWriteNagiosStatusFile()) { - chc.writeNagiosStatusFile(); - } - chc = null; - } - // Prepare consensus stats file handler (used for stats on running // bridges only) ConsensusStatsFileHandler csfh = config.getWriteBridgeStats() ? diff --git a/src/org/torproject/ernie/cron/RelayDescriptorParser.java b/src/org/torproject/ernie/cron/RelayDescriptorParser.java index 4b8a16c..f2ae02d 100644 --- a/src/org/torproject/ernie/cron/RelayDescriptorParser.java +++ b/src/org/torproject/ernie/cron/RelayDescriptorParser.java @@ -28,8 +28,6 @@ public class RelayDescriptorParser { */ private RelayDescriptorDatabaseImporter rddi;

- private ConsensusHealthChecker chc; - /** * Logger for this class. */ @@ -40,9 +38,8 @@ public class RelayDescriptorParser { /** * Initializes this class. */ - public RelayDescriptorParser(ConsensusHealthChecker chc, - RelayDescriptorDatabaseImporter rddi, BridgeStatsFileHandler bsfh) { - this.chc = chc; + public RelayDescriptorParser(RelayDescriptorDatabaseImporter rddi, + BridgeStatsFileHandler bsfh) { this.rddi = rddi; this.bsfh = bsfh;

@@ -205,9 +202,6 @@ public class RelayDescriptorParser { this.bsfh.addHashedRelay(hashedRelayIdentity); } } - if (this.chc != null) { - this.chc.processConsensus(validAfterTime, data); - } if (this.rddi != null) { this.rddi.addConsensus(validAfter, data); if (relayIdentity != null) { @@ -219,9 +213,6 @@ public class RelayDescriptorParser { } } } else { - if (this.chc != null) { - this.chc.processVote(validAfterTime, dirSource, data); - } if (this.rddi != null) { this.rddi.addVote(validAfter, dirSource, data); }