[tor-commits] [doctor/master] Clean up a comments and use saner output filenames.
karsten at torproject.org
karsten at torproject.org
Tue Dec 13 13:40:15 UTC 2011
commit bbdfc6f942e58aa9191acc5609f73660f608051a
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Dec 13 14:22:22 2011 +0100
Clean up a comments and use saner output filenames.
---
.gitignore | 1 +
README | 36 +++++++++++++
src/org/torproject/doctor/Checker.java | 15 +++---
src/org/torproject/doctor/DownloadStatistics.java | 16 ++++++-
src/org/torproject/doctor/Downloader.java | 24 ++++++---
src/org/torproject/doctor/Main.java | 11 ++--
.../torproject/doctor/MetricsWebsiteReport.java | 8 +--
src/org/torproject/doctor/StatusFileReport.java | 54 +++++++++++--------
src/org/torproject/doctor/Warning.java | 2 +-
9 files changed, 114 insertions(+), 53 deletions(-)
diff --git a/.gitignore b/.gitignore
index 64253cb..14a8cb2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
classes/
lib/
+out/
diff --git a/README b/README
index 0a9a9b4..1b900d2 100644
--- a/README
+++ b/README
@@ -8,3 +8,39 @@ from the Tor directory authorities and checks them for consensus problems.
DocTor writes its findings to local files which can then be sent to a
mailing list or IRC bot, or which can be served by an HTTP server.
+
+Howto
+-----
+
+Create a lib/ directory.
+
+Download Apache Commons Codec 1.4 or higher and put it in the lib/
+directory. If the filename is not commons-codec-1.4.jar, update the
+build.xml file.
+
+Clone metrics-lib, create a .jar file using `ant jar`, and put it in the
+lib/ directory, too.
+
+Compile the Java classes using `ant compile`.
+
+Run the application using `ant run`.
+
+Output files are:
+
+ - out/status/new-warnings: Consensus warnings that have been found in
+ this execution or that were found long enough before to not be
+ rate-limited anymore.
+
+ - out/status/all-warnings: All warnings found in this execution, but only
+ if at least one of them is in new-warnings, too.
+
+ - out/website/consensus-health.html: HTML file containing a full
+ comparison of consensuses to its votes, marking problems in red.
+
+Generated temp files (read: don't mess with them) are:
+
+ - out/stats/download-stats.csv: Raw consensus download times.
+
+ - out/stats/last-warned: Warning messages and when they were last
+ contained in new-warnings or all-warnings.
+
diff --git a/src/org/torproject/doctor/Checker.java b/src/org/torproject/doctor/Checker.java
index ee4e29e..c78ebf6 100644
--- a/src/org/torproject/doctor/Checker.java
+++ b/src/org/torproject/doctor/Checker.java
@@ -14,7 +14,6 @@ public class Checker {
/* Warning messages consisting of type and details. */
private SortedMap<Warning, String> warnings =
new TreeMap<Warning, String>();
-
public SortedMap<Warning, String> getWarnings() {
return this.warnings;
}
@@ -26,13 +25,7 @@ public class Checker {
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
}
- /* Downloaded consensus and corresponding votes for processing. */
- private SortedMap<String, RelayNetworkStatusConsensus>
- downloadedConsensuses = new TreeMap<String,
- RelayNetworkStatusConsensus>();
- private RelayNetworkStatusConsensus downloadedConsensus;
- private List<RelayNetworkStatusVote> downloadedVotes =
- new ArrayList<RelayNetworkStatusVote>();
+ /* Check consensuses and votes. */
public void processDownloadedConsensuses(
List<DescriptorRequest> downloads) {
this.storeDownloads(downloads);
@@ -57,6 +50,12 @@ public class Checker {
/* Store consensuses and votes in a way that we can process them more
* easily. */
+ private SortedMap<String, RelayNetworkStatusConsensus>
+ downloadedConsensuses = new TreeMap<String,
+ RelayNetworkStatusConsensus>();
+ private RelayNetworkStatusConsensus downloadedConsensus;
+ private List<RelayNetworkStatusVote> downloadedVotes =
+ new ArrayList<RelayNetworkStatusVote>();
private void storeDownloads(List<DescriptorRequest> downloads) {
for (DescriptorRequest request : downloads) {
for (Descriptor descriptor : request.getDescriptors()) {
diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java
index 9df0333..e30aed8 100644
--- a/src/org/torproject/doctor/DownloadStatistics.java
+++ b/src/org/torproject/doctor/DownloadStatistics.java
@@ -6,9 +6,15 @@ import java.io.*;
import java.util.*;
import org.torproject.descriptor.*;
+/* Provide simple statistics about consensus download times. */
public class DownloadStatistics {
+
+ /* Add a new set of download times by append them to the history
+ * file. */
+ private File statisticsFile = new File("out/state/download-stats.csv");
public void memorizeFetchTimes(List<DescriptorRequest> downloads) {
try {
+ this.statisticsFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
this.statisticsFile, true));
for (DescriptorRequest request : downloads) {
@@ -31,10 +37,12 @@ public class DownloadStatistics {
+ this.statisticsFile.getAbsolutePath() + ". Ignoring.");
}
}
+
+ /* Prepare statistics by reading the download history and sorting to
+ * calculate percentiles more easily. */
private SortedMap<String, List<Long>> downloadData =
new TreeMap<String, List<Long>>();
private int maxDownloadsPerAuthority = 0;
- private File statisticsFile = new File("download-stats.csv");
public void prepareStatistics() {
if (this.statisticsFile.exists()) {
long cutOffMillis = System.currentTimeMillis()
@@ -71,9 +79,13 @@ public class DownloadStatistics {
}
}
}
+
+ /* Return the list of authorities that we have statistics for. */
public SortedSet<String> getKnownAuthorities() {
return new TreeSet<String>(this.downloadData.keySet());
}
+
+ /* Return the download time percentile for a directory authority. */
public String getPercentile(String authority, int percentile) {
if (percentile < 0 || percentile > 100 ||
!this.downloadData.containsKey(authority)) {
@@ -84,6 +96,8 @@ public class DownloadStatistics {
return String.valueOf(fetchTimes.get(index));
}
}
+
+ /* Return the number of NAs (timeouts) for a directory authority. */
public String getNAs(String authority) {
if (!this.downloadData.containsKey(authority)) {
return "NA";
diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java
index 317b3b6..f20885f 100644
--- a/src/org/torproject/doctor/Downloader.java
+++ b/src/org/torproject/doctor/Downloader.java
@@ -2,23 +2,22 @@
* See LICENSE for licensing information */
package org.torproject.doctor;
-import java.io.*;
-import java.net.*;
-import java.text.*;
import java.util.*;
-import java.util.zip.*;
import org.torproject.descriptor.*;
/* Download the latest network status consensus and corresponding
- * votes. */
+ * votes using metrics-lib. */
public class Downloader {
/* Download the current consensus and corresponding votes. */
public List<DescriptorRequest> downloadFromAuthorities() {
+ /* Create a descriptor downloader instance that will do all the hard
+ * download work for us. */
RelayDescriptorDownloader downloader =
DescriptorSourceFactory.createRelayDescriptorDownloader();
+ /* Configure the currently known directory authorities. */
downloader.addDirectoryAuthority("gabelmoo", "212.112.245.170", 80);
downloader.addDirectoryAuthority("tor26", "86.59.21.38", 80);
downloader.addDirectoryAuthority("ides", "216.224.124.114", 9030);
@@ -28,20 +27,28 @@ public class Downloader {
downloader.addDirectoryAuthority("moria1", "128.31.0.34", 9131);
downloader.addDirectoryAuthority("dizum", "194.109.206.212", 80);
+ /* Instruct the downloader to include the current consensus and all
+ * referenced votes in the downloads. The consensus shall be
+ * downloaded from all directory authorities, not just from one. */
downloader.setIncludeCurrentConsensusFromAllDirectoryAuthorities();
downloader.setIncludeCurrentReferencedVotes();
+ /* Set a per-request timeout of 60 seconds. */
downloader.setRequestTimeout(60L * 1000L);
- List<DescriptorRequest> allRequests =
- new ArrayList<DescriptorRequest>();
+ /* Iterate over the finished (or aborted) requests and memorize the
+ * included consensuses or votes. The processing will take place
+ * later. */
Iterator<DescriptorRequest> descriptorRequests =
downloader.downloadDescriptors();
+ List<DescriptorRequest> allRequests =
+ new ArrayList<DescriptorRequest>();
while (descriptorRequests.hasNext()) {
try {
allRequests.add(descriptorRequests.next());
} catch (NoSuchElementException e) {
- /* TODO In theory, this exception shouldn't be thrown. */
+ /* TODO In theory, this exception shouldn't be thrown. This is a
+ * bug in metrics-lib. */
System.err.println("Internal error: next() doesn't provide an "
+ "element even though hasNext() returned true. Got "
+ allRequests.size() + " elements so far. Stopping to "
@@ -50,6 +57,7 @@ public class Downloader {
}
}
+ /* We downloaded everything we wanted. */
return allRequests;
}
}
diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java
index 40973b4..f723e0b 100644
--- a/src/org/torproject/doctor/Main.java
+++ b/src/org/torproject/doctor/Main.java
@@ -5,13 +5,13 @@ package org.torproject.doctor;
import java.util.*;
import org.torproject.descriptor.*;
-/* Coordinate the process of downloading consensus and votes to check
- * Tor's consensus health. */
+/* Coordinate the process of downloading the current consensus and votes
+ * to check Tor's consensus health. */
public class Main {
public static void main(String[] args) {
- /* Download consensus and corresponding votes from the directory
- * authorities. */
+ /* Download the current consensus from all directory authorities and
+ * all referenced votes from any directory authority. */
Downloader downloader = new Downloader();
List<DescriptorRequest> downloads =
downloader.downloadFromAuthorities();
@@ -26,8 +26,7 @@ public class Main {
statusFile.writeReport();
/* Write a complete consensus-health report to an HTML file. */
- MetricsWebsiteReport website =
- new MetricsWebsiteReport("website/consensus-health.html");
+ MetricsWebsiteReport website = new MetricsWebsiteReport();
website.processDownloadedConsensuses(downloads);
DownloadStatistics fetchStatistics = new DownloadStatistics();
fetchStatistics.memorizeFetchTimes(downloads);
diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java
index 8d0bb0b..ac60ae7 100644
--- a/src/org/torproject/doctor/MetricsWebsiteReport.java
+++ b/src/org/torproject/doctor/MetricsWebsiteReport.java
@@ -19,12 +19,8 @@ public class MetricsWebsiteReport {
}
/* Output file to write report to. */
- private File htmlOutputFile;
-
- /* Initialize this report. */
- public MetricsWebsiteReport(String htmlOutputFilename) {
- this.htmlOutputFile = new File(htmlOutputFilename);
- }
+ private File htmlOutputFile =
+ new File("out/website/consensus-health.html");
/* Store the downloaded consensus and corresponding votes for later
* processing. */
diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java
index 7890005..a8838b6 100644
--- a/src/org/torproject/doctor/StatusFileReport.java
+++ b/src/org/torproject/doctor/StatusFileReport.java
@@ -7,7 +7,9 @@ import java.text.*;
import java.util.*;
/* Check a given consensus and votes for irregularities and write results
- * to stdout while rate-limiting warnings based on severity. */
+ * to status files while rate-limiting warnings based on severity. There
+ * will be a 'all-warnings' file with all warnings and a 'new-warnings'
+ * file with only the warnings that haven't been emitted recently. */
public class StatusFileReport {
/* Date-time format to format timestamps. */
@@ -23,32 +25,32 @@ public class StatusFileReport {
this.warnings = warnings;
}
- /* Check consensuses and votes for irregularities and write output to
- * stdout. */
+ /* Write warnings to the status files. */
public void writeReport() {
this.readLastWarned();
- this.prepareReports();
+ this.prepareStatusFiles();
this.writeStatusFiles();
this.writeLastWarned();
}
- /* Warning messages of the last 24 hours that is used to implement
- * rate limiting. */
+ /* Map of warning message strings of the last 24 hours and when they
+ * were last included in the 'new-warnings' file. This map is used to
+ * implement rate limiting. */
private Map<String, Long> lastWarned = new HashMap<String, Long>();
/* Read when we last emitted a warning to rate-limit some of them. */
+ private File lastWarnedFile = new File("out/state/last-warned");
private void readLastWarned() {
long now = System.currentTimeMillis();
- File lastWarnedFile = new File("stats/chc-last-warned");
try {
- if (lastWarnedFile.exists()) {
+ if (this.lastWarnedFile.exists()) {
BufferedReader br = new BufferedReader(new FileReader(
- lastWarnedFile));
+ this.lastWarnedFile));
String line;
while ((line = br.readLine()) != null) {
if (!line.contains(": ")) {
- System.err.println("Bad line in stats/chc-last-warned: '" + line
- + "'. Ignoring this line.");
+ System.err.println("Bad line in stats/chc-last-warned: '"
+ + line + "'. Ignoring this line.");
continue;
}
long warnedMillis = Long.parseLong(line.substring(0,
@@ -63,20 +65,21 @@ public class StatusFileReport {
}
} catch (IOException e) {
System.err.println("Could not read file '"
- + lastWarnedFile.getAbsolutePath() + "' to learn which "
+ + this.lastWarnedFile.getAbsolutePath() + "' to learn which "
+ "warnings have been sent out before. Ignoring.");
}
}
- /* Prepare a report to be written to stdout. */
+ /* Prepare status files to be written. */
private String allWarnings = null, newWarnings = null;
- private void prepareReports() {
+ private void prepareStatusFiles() {
SortedMap<String, Long> warningStrings = new TreeMap<String, Long>();
for (Map.Entry<Warning, String> e : this.warnings.entrySet()) {
Warning type = e.getKey();
String details = e.getValue();
switch (type) {
case NoConsensusKnown:
+ warningStrings.put("No consensus known.", 0L);
break;
case ConsensusDownloadTimeout:
warningStrings.put("The following directory authorities did "
@@ -158,12 +161,17 @@ public class StatusFileReport {
}
}
- /* Write report to stdout. */
+ /* Write status files to disk. */
+ private File allWarningsFile = new File("out/status/all-warnings");
+ private File newWarningsFile = new File("out/status/new-warnings");
private void writeStatusFiles() {
try {
+ this.allWarningsFile.getParentFile().mkdirs();
+ this.newWarningsFile.getParentFile().mkdirs();
BufferedWriter allBw = new BufferedWriter(new FileWriter(
- "all-warnings")), newBw = new BufferedWriter(new FileWriter(
- "new-warnings"));
+ this.allWarningsFile));
+ BufferedWriter newBw = new BufferedWriter(new FileWriter(
+ this.newWarningsFile));
if (this.allWarnings != null) {
allBw.write(this.allWarnings);
}
@@ -173,25 +181,25 @@ public class StatusFileReport {
allBw.close();
newBw.close();
} catch (IOException e) {
- System.err.println("Could not write status files 'all-warnings' "
- + "and/or 'new-warnings'. Ignoring.");
+ System.err.println("Could not write status files '"
+ + this.allWarningsFile.getAbsolutePath() + "' and/or '"
+ + this.newWarningsFile.getAbsolutePath() + "'. Ignoring.");
}
}
/* Write timestamps when warnings were last sent to disk. */
private void writeLastWarned() {
- File lastWarnedFile = new File("stats/chc-last-warned");
try {
- lastWarnedFile.getParentFile().mkdirs();
+ this.lastWarnedFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
- lastWarnedFile));
+ this.lastWarnedFile));
for (Map.Entry<String, Long> e : lastWarned.entrySet()) {
bw.write(String.valueOf(e.getValue()) + ": " + e.getKey() + "\n");
}
bw.close();
} catch (IOException e) {
System.err.println("Could not write file '"
- + lastWarnedFile.getAbsolutePath() + "' to remember which "
+ + this.lastWarnedFile.getAbsolutePath() + "' to remember which "
+ "warnings have been sent out before. Ignoring.");
}
}
diff --git a/src/org/torproject/doctor/Warning.java b/src/org/torproject/doctor/Warning.java
index 0cb8cd8..1684f89 100644
--- a/src/org/torproject/doctor/Warning.java
+++ b/src/org/torproject/doctor/Warning.java
@@ -49,7 +49,7 @@ public enum Warning {
ConsensusMissingVotes,
/* The consensuses downloaded from one or more authorities are missing
- * signatures from other, previously voting authorities. */
+ * signatures from previously voting authorities. */
ConsensusMissingSignatures
}
More information about the tor-commits
mailing list