commit bbdfc6f942e58aa9191acc5609f73660f608051a Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Dec 13 14:22:22 2011 +0100

Clean up a comments and use saner output filenames. --- .gitignore | 1 + README | 36 +++++++++++++ src/org/torproject/doctor/Checker.java | 15 +++--- src/org/torproject/doctor/DownloadStatistics.java | 16 ++++++- src/org/torproject/doctor/Downloader.java | 24 ++++++--- src/org/torproject/doctor/Main.java | 11 ++-- .../torproject/doctor/MetricsWebsiteReport.java | 8 +-- src/org/torproject/doctor/StatusFileReport.java | 54 +++++++++++-------- src/org/torproject/doctor/Warning.java | 2 +- 9 files changed, 114 insertions(+), 53 deletions(-)

diff --git a/.gitignore b/.gitignore index 64253cb..14a8cb2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ classes/ lib/ +out/

diff --git a/README b/README index 0a9a9b4..1b900d2 100644 --- a/README +++ b/README @@ -8,3 +8,39 @@ from the Tor directory authorities and checks them for consensus problems. DocTor writes its findings to local files which can then be sent to a mailing list or IRC bot, or which can be served by an HTTP server.

+ +Howto +----- + +Create a lib/ directory. + +Download Apache Commons Codec 1.4 or higher and put it in the lib/ +directory. If the filename is not commons-codec-1.4.jar, update the +build.xml file. + +Clone metrics-lib, create a .jar file using `ant jar`, and put it in the +lib/ directory, too. + +Compile the Java classes using `ant compile`. + +Run the application using `ant run`. + +Output files are: + + - out/status/new-warnings: Consensus warnings that have been found in + this execution or that were found long enough before to not be + rate-limited anymore. + + - out/status/all-warnings: All warnings found in this execution, but only + if at least one of them is in new-warnings, too. + + - out/website/consensus-health.html: HTML file containing a full + comparison of consensuses to its votes, marking problems in red. + +Generated temp files (read: don't mess with them) are: + + - out/stats/download-stats.csv: Raw consensus download times. + + - out/stats/last-warned: Warning messages and when they were last + contained in new-warnings or all-warnings. + diff --git a/src/org/torproject/doctor/Checker.java b/src/org/torproject/doctor/Checker.java index ee4e29e..c78ebf6 100644 --- a/src/org/torproject/doctor/Checker.java +++ b/src/org/torproject/doctor/Checker.java @@ -14,7 +14,6 @@ public class Checker { /* Warning messages consisting of type and details. */ private SortedMap<Warning, String> warnings = new TreeMap<Warning, String>(); - public SortedMap<Warning, String> getWarnings() { return this.warnings; } @@ -26,13 +25,7 @@ public class Checker { dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); }

- /* Downloaded consensus and corresponding votes for processing. */ - private SortedMap<String, RelayNetworkStatusConsensus> - downloadedConsensuses = new TreeMap<String, - RelayNetworkStatusConsensus>(); - private RelayNetworkStatusConsensus downloadedConsensus; - private List<RelayNetworkStatusVote> downloadedVotes = - new ArrayList<RelayNetworkStatusVote>(); + /* Check consensuses and votes. */ public void processDownloadedConsensuses( List<DescriptorRequest> downloads) { this.storeDownloads(downloads); @@ -57,6 +50,12 @@ public class Checker {

/* Store consensuses and votes in a way that we can process them more * easily. */ + private SortedMap<String, RelayNetworkStatusConsensus> + downloadedConsensuses = new TreeMap<String, + RelayNetworkStatusConsensus>(); + private RelayNetworkStatusConsensus downloadedConsensus; + private List<RelayNetworkStatusVote> downloadedVotes = + new ArrayList<RelayNetworkStatusVote>(); private void storeDownloads(List<DescriptorRequest> downloads) { for (DescriptorRequest request : downloads) { for (Descriptor descriptor : request.getDescriptors()) { diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java index 9df0333..e30aed8 100644 --- a/src/org/torproject/doctor/DownloadStatistics.java +++ b/src/org/torproject/doctor/DownloadStatistics.java @@ -6,9 +6,15 @@ import java.io.*; import java.util.*; import org.torproject.descriptor.*;

+/* Provide simple statistics about consensus download times. */ public class DownloadStatistics { + + /* Add a new set of download times by append them to the history + * file. */ + private File statisticsFile = new File("out/state/download-stats.csv"); public void memorizeFetchTimes(List<DescriptorRequest> downloads) { try { + this.statisticsFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( this.statisticsFile, true)); for (DescriptorRequest request : downloads) { @@ -31,10 +37,12 @@ public class DownloadStatistics { + this.statisticsFile.getAbsolutePath() + ". Ignoring."); } } + + /* Prepare statistics by reading the download history and sorting to + * calculate percentiles more easily. */ private SortedMap<String, List<Long>> downloadData = new TreeMap<String, List<Long>>(); private int maxDownloadsPerAuthority = 0; - private File statisticsFile = new File("download-stats.csv"); public void prepareStatistics() { if (this.statisticsFile.exists()) { long cutOffMillis = System.currentTimeMillis() @@ -71,9 +79,13 @@ public class DownloadStatistics { } } } + + /* Return the list of authorities that we have statistics for. */ public SortedSet<String> getKnownAuthorities() { return new TreeSet<String>(this.downloadData.keySet()); } + + /* Return the download time percentile for a directory authority. */ public String getPercentile(String authority, int percentile) { if (percentile < 0 || percentile > 100 || !this.downloadData.containsKey(authority)) { @@ -84,6 +96,8 @@ public class DownloadStatistics { return String.valueOf(fetchTimes.get(index)); } } + + /* Return the number of NAs (timeouts) for a directory authority. */ public String getNAs(String authority) { if (!this.downloadData.containsKey(authority)) { return "NA"; diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java index 317b3b6..f20885f 100644 --- a/src/org/torproject/doctor/Downloader.java +++ b/src/org/torproject/doctor/Downloader.java @@ -2,23 +2,22 @@ * See LICENSE for licensing information */ package org.torproject.doctor;

-import java.io.*; -import java.net.*; -import java.text.*; import java.util.*; -import java.util.zip.*; import org.torproject.descriptor.*;

/* Download the latest network status consensus and corresponding - * votes. */ + * votes using metrics-lib. */ public class Downloader {

/* Download the current consensus and corresponding votes. */ public List<DescriptorRequest> downloadFromAuthorities() {

+ /* Create a descriptor downloader instance that will do all the hard + * download work for us. */ RelayDescriptorDownloader downloader = DescriptorSourceFactory.createRelayDescriptorDownloader();

+ /* Configure the currently known directory authorities. */ downloader.addDirectoryAuthority("gabelmoo", "212.112.245.170", 80); downloader.addDirectoryAuthority("tor26", "86.59.21.38", 80); downloader.addDirectoryAuthority("ides", "216.224.124.114", 9030); @@ -28,20 +27,28 @@ public class Downloader { downloader.addDirectoryAuthority("moria1", "128.31.0.34", 9131); downloader.addDirectoryAuthority("dizum", "194.109.206.212", 80);

+ /* Instruct the downloader to include the current consensus and all + * referenced votes in the downloads. The consensus shall be + * downloaded from all directory authorities, not just from one. */ downloader.setIncludeCurrentConsensusFromAllDirectoryAuthorities(); downloader.setIncludeCurrentReferencedVotes();

+ /* Set a per-request timeout of 60 seconds. */ downloader.setRequestTimeout(60L * 1000L);

- List<DescriptorRequest> allRequests = - new ArrayList<DescriptorRequest>(); + /* Iterate over the finished (or aborted) requests and memorize the + * included consensuses or votes. The processing will take place + * later. */ Iterator<DescriptorRequest> descriptorRequests = downloader.downloadDescriptors(); + List<DescriptorRequest> allRequests = + new ArrayList<DescriptorRequest>(); while (descriptorRequests.hasNext()) { try { allRequests.add(descriptorRequests.next()); } catch (NoSuchElementException e) { - /* TODO In theory, this exception shouldn't be thrown. */ + /* TODO In theory, this exception shouldn't be thrown. This is a + * bug in metrics-lib. */ System.err.println("Internal error: next() doesn't provide an " + "element even though hasNext() returned true. Got " + allRequests.size() + " elements so far. Stopping to " @@ -50,6 +57,7 @@ public class Downloader { } }

+ /* We downloaded everything we wanted. */ return allRequests; } } diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java index 40973b4..f723e0b 100644 --- a/src/org/torproject/doctor/Main.java +++ b/src/org/torproject/doctor/Main.java @@ -5,13 +5,13 @@ package org.torproject.doctor; import java.util.*; import org.torproject.descriptor.*;

-/* Coordinate the process of downloading consensus and votes to check - * Tor's consensus health. */ +/* Coordinate the process of downloading the current consensus and votes + * to check Tor's consensus health. */ public class Main { public static void main(String[] args) {

- /* Download consensus and corresponding votes from the directory - * authorities. */ + /* Download the current consensus from all directory authorities and + * all referenced votes from any directory authority. */ Downloader downloader = new Downloader(); List<DescriptorRequest> downloads = downloader.downloadFromAuthorities(); @@ -26,8 +26,7 @@ public class Main { statusFile.writeReport();

/* Write a complete consensus-health report to an HTML file. */ - MetricsWebsiteReport website = - new MetricsWebsiteReport("website/consensus-health.html"); + MetricsWebsiteReport website = new MetricsWebsiteReport(); website.processDownloadedConsensuses(downloads); DownloadStatistics fetchStatistics = new DownloadStatistics(); fetchStatistics.memorizeFetchTimes(downloads); diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java index 8d0bb0b..ac60ae7 100644 --- a/src/org/torproject/doctor/MetricsWebsiteReport.java +++ b/src/org/torproject/doctor/MetricsWebsiteReport.java @@ -19,12 +19,8 @@ public class MetricsWebsiteReport { }

/* Output file to write report to. */ - private File htmlOutputFile; - - /* Initialize this report. */ - public MetricsWebsiteReport(String htmlOutputFilename) { - this.htmlOutputFile = new File(htmlOutputFilename); - } + private File htmlOutputFile = + new File("out/website/consensus-health.html");

/* Store the downloaded consensus and corresponding votes for later * processing. */ diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java index 7890005..a8838b6 100644 --- a/src/org/torproject/doctor/StatusFileReport.java +++ b/src/org/torproject/doctor/StatusFileReport.java @@ -7,7 +7,9 @@ import java.text.*; import java.util.*;

/* Check a given consensus and votes for irregularities and write results - * to stdout while rate-limiting warnings based on severity. */ + * to status files while rate-limiting warnings based on severity. There + * will be a 'all-warnings' file with all warnings and a 'new-warnings' + * file with only the warnings that haven't been emitted recently. */ public class StatusFileReport {

/* Date-time format to format timestamps. */ @@ -23,32 +25,32 @@ public class StatusFileReport { this.warnings = warnings; }

- /* Check consensuses and votes for irregularities and write output to - * stdout. */ + /* Write warnings to the status files. */ public void writeReport() { this.readLastWarned(); - this.prepareReports(); + this.prepareStatusFiles(); this.writeStatusFiles(); this.writeLastWarned(); }

- /* Warning messages of the last 24 hours that is used to implement - * rate limiting. */ + /* Map of warning message strings of the last 24 hours and when they + * were last included in the 'new-warnings' file. This map is used to + * implement rate limiting. */ private Map<String, Long> lastWarned = new HashMap<String, Long>();

/* Read when we last emitted a warning to rate-limit some of them. */ + private File lastWarnedFile = new File("out/state/last-warned"); private void readLastWarned() { long now = System.currentTimeMillis(); - File lastWarnedFile = new File("stats/chc-last-warned"); try { - if (lastWarnedFile.exists()) { + if (this.lastWarnedFile.exists()) { BufferedReader br = new BufferedReader(new FileReader( - lastWarnedFile)); + this.lastWarnedFile)); String line; while ((line = br.readLine()) != null) { if (!line.contains(": ")) { - System.err.println("Bad line in stats/chc-last-warned: '" + line - + "'. Ignoring this line."); + System.err.println("Bad line in stats/chc-last-warned: '" + + line + "'. Ignoring this line."); continue; } long warnedMillis = Long.parseLong(line.substring(0, @@ -63,20 +65,21 @@ public class StatusFileReport { } } catch (IOException e) { System.err.println("Could not read file '" - + lastWarnedFile.getAbsolutePath() + "' to learn which " + + this.lastWarnedFile.getAbsolutePath() + "' to learn which " + "warnings have been sent out before. Ignoring."); } }

- /* Prepare a report to be written to stdout. */ + /* Prepare status files to be written. */ private String allWarnings = null, newWarnings = null; - private void prepareReports() { + private void prepareStatusFiles() { SortedMap<String, Long> warningStrings = new TreeMap<String, Long>(); for (Map.Entry<Warning, String> e : this.warnings.entrySet()) { Warning type = e.getKey(); String details = e.getValue(); switch (type) { case NoConsensusKnown: + warningStrings.put("No consensus known.", 0L); break; case ConsensusDownloadTimeout: warningStrings.put("The following directory authorities did " @@ -158,12 +161,17 @@ public class StatusFileReport { } }

- /* Write report to stdout. */ + /* Write status files to disk. */ + private File allWarningsFile = new File("out/status/all-warnings"); + private File newWarningsFile = new File("out/status/new-warnings"); private void writeStatusFiles() { try { + this.allWarningsFile.getParentFile().mkdirs(); + this.newWarningsFile.getParentFile().mkdirs(); BufferedWriter allBw = new BufferedWriter(new FileWriter( - "all-warnings")), newBw = new BufferedWriter(new FileWriter( - "new-warnings")); + this.allWarningsFile)); + BufferedWriter newBw = new BufferedWriter(new FileWriter( + this.newWarningsFile)); if (this.allWarnings != null) { allBw.write(this.allWarnings); } @@ -173,25 +181,25 @@ public class StatusFileReport { allBw.close(); newBw.close(); } catch (IOException e) { - System.err.println("Could not write status files 'all-warnings' " - + "and/or 'new-warnings'. Ignoring."); + System.err.println("Could not write status files '" + + this.allWarningsFile.getAbsolutePath() + "' and/or '" + + this.newWarningsFile.getAbsolutePath() + "'. Ignoring."); } }

/* Write timestamps when warnings were last sent to disk. */ private void writeLastWarned() { - File lastWarnedFile = new File("stats/chc-last-warned"); try { - lastWarnedFile.getParentFile().mkdirs(); + this.lastWarnedFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( - lastWarnedFile)); + this.lastWarnedFile)); for (Map.Entry<String, Long> e : lastWarned.entrySet()) { bw.write(String.valueOf(e.getValue()) + ": " + e.getKey() + "\n"); } bw.close(); } catch (IOException e) { System.err.println("Could not write file '" - + lastWarnedFile.getAbsolutePath() + "' to remember which " + + this.lastWarnedFile.getAbsolutePath() + "' to remember which " + "warnings have been sent out before. Ignoring."); } } diff --git a/src/org/torproject/doctor/Warning.java b/src/org/torproject/doctor/Warning.java index 0cb8cd8..1684f89 100644 --- a/src/org/torproject/doctor/Warning.java +++ b/src/org/torproject/doctor/Warning.java @@ -49,7 +49,7 @@ public enum Warning { ConsensusMissingVotes,

/* The consensuses downloaded from one or more authorities are missing - * signatures from other, previously voting authorities. */ + * signatures from previously voting authorities. */ ConsensusMissingSignatures }