commit 29dbe7fbb197c7c22f85d3fcd610daa8613a2de2 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Oct 20 14:30:47 2011 +0200
Stop archiving consensuses and votes after checking them.
The idea of archiving consensus and votes was that we could compare them over time to see if, e.g., a consensus parameter gets added or removed. This works in theory, but in practice, this approach makes the code more complex than it needs to be. We can as well change valid consensus parameters in the code. --- src/org/torproject/chc/Archiver.java | 120 ---------------------- src/org/torproject/chc/Downloader.java | 15 +--- src/org/torproject/chc/Main.java | 34 +------ src/org/torproject/chc/MetricsWebsiteReport.java | 5 - src/org/torproject/chc/NagiosReport.java | 19 ---- src/org/torproject/chc/Parser.java | 11 -- src/org/torproject/chc/Report.java | 4 - src/org/torproject/chc/Status.java | 9 -- src/org/torproject/chc/StdOutReport.java | 19 ---- 9 files changed, 5 insertions(+), 231 deletions(-)
diff --git a/src/org/torproject/chc/Archiver.java b/src/org/torproject/chc/Archiver.java deleted file mode 100644 index 6271348..0000000 --- a/src/org/torproject/chc/Archiver.java +++ /dev/null @@ -1,120 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.chc; - -import java.io.*; -import java.util.*; - -/* Load the last network status consensus and corresponding votes from - * disk and save newly downloaded ones to disk. */ -public class Archiver { - - /* Local directory containing cached consensuses and votes. */ - private File cachedFilesDirectory = new File("chc-cache"); - - /* Copy of the most recent cached consensus and votes. */ - private String loadedConsensus; - private List<String> loadedVotes = new ArrayList<String>(); - - /* Load the most recent cached consensus and votes to memory. */ - public void loadLastFromDisk() { - if (!this.cachedFilesDirectory.exists()) { - return; - } - String lastValidAfter = this.findLastConsensusPrefix(); - if (lastValidAfter != null) { - for (File file : this.cachedFilesDirectory.listFiles()) { - if (file.isDirectory() || - !file.getName().startsWith(lastValidAfter)) { - continue; - } - String content = null; - try { - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - content = new String(allData); - } catch (IOException e) { - System.err.println("Could not read cached status from file '" - + file.getAbsolutePath() + "'. Skipping."); - continue; - } - if (file.getName().contains("-consensus")) { - this.loadedConsensus = content; - } else if (file.getName().contains("-vote-")) { - this.loadedVotes.add(content); - } - } - } - } - - /* Delete all cached consensuses and votes but the last consensus and - * corresponding votes. */ - public void deleteAllButLast() { - if (!this.cachedFilesDirectory.exists()) { - return; - } - String lastValidAfter = this.findLastConsensusPrefix(); - if (lastValidAfter != null) { - for (File file : this.cachedFilesDirectory.listFiles()) { - if (!file.getName().startsWith(lastValidAfter)) { - file.delete(); - } - } - } - } - - /* Save a status to disk under the given file name. */ - public void saveStatusStringToDisk(String content, String fileName) { - try { - this.cachedFilesDirectory.mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(new File( - this.cachedFilesDirectory, fileName))); - bw.write(content); - bw.close(); - } catch (IOException e) { - System.err.println("Could not save status to file '" - + this.cachedFilesDirectory.getAbsolutePath() + "/" + fileName - + "'. Ignoring."); - } - } - - /* Find and return the timestamp prefix of the last published - * consensus. */ - private String findLastConsensusPrefix() { - String lastValidAfter = null; - for (File file : this.cachedFilesDirectory.listFiles()) { - if (file.isDirectory() || - file.getName().length() != - "yyyy-MM-dd-HH-mm-ss-consensus".length() || - !file.getName().endsWith("-consensus")) { - continue; - } - String prefix = file.getName().substring(0, - "yyyy-MM-dd-HH-mm-ss".length()); - if (lastValidAfter == null || - prefix.compareTo(lastValidAfter) > 0) { - lastValidAfter = prefix; - } - } - return lastValidAfter; - } - - /* Return the previously loaded (unparsed) consensus string. */ - public String getConsensusString() { - return this.loadedConsensus; - } - - /* Return the previously loaded (unparsed) vote strings. */ - public List<String> getVoteStrings() { - return this.loadedVotes; - } -} - diff --git a/src/org/torproject/chc/Downloader.java b/src/org/torproject/chc/Downloader.java index bdd23fe..cf451ee 100644 --- a/src/org/torproject/chc/Downloader.java +++ b/src/org/torproject/chc/Downloader.java @@ -20,21 +20,8 @@ public class Downloader { + "208.83.223.34:443,128.31.0.34:9131,194.109.206.212"). split(",")));
- /* Set the last known valid-after time to avoid downloading a new - * consensus if there cannot be a new one yet. */ - private long lastKnownValidAfterMillis; - public void setLastKnownValidAfterMillis( - long lastKnownValidAfterMillis) { - this.lastKnownValidAfterMillis = lastKnownValidAfterMillis; - } - - /* Download a new consensus and corresponding votes if we expect them to - * be newer than the ones we have. */ + /* Download a new consensus and corresponding votes. */ public void downloadFromAuthorities() { - if (System.currentTimeMillis() - lastKnownValidAfterMillis < - 60L * 60L * 1000L) { - return; - } this.downloadConsensus(); if (this.downloadedConsensus != null) { this.parseConsensusToFindReferencedVotes(); diff --git a/src/org/torproject/chc/Main.java b/src/org/torproject/chc/Main.java index 5aadc6e..d42ee44 100644 --- a/src/org/torproject/chc/Main.java +++ b/src/org/torproject/chc/Main.java @@ -16,26 +16,13 @@ public class Main { reports.add(new NagiosReport("stats/consensus-health")); reports.add(new StdOutReport());
- /* Load last-known consensus and votes from disk and parse them. */ - Archiver archiver = new Archiver(); - archiver.loadLastFromDisk(); - Parser parser = new Parser(); - Status parsedCachedConsensus = parser.parse( - archiver.getConsensusString(), archiver.getVoteStrings()); - if (parsedCachedConsensus != null) { - for (Report report : reports) { - report.processCachedConsensus(parsedCachedConsensus); - } - } - /* Download consensus and corresponding votes from the directory - * authorities and parse them, too. */ + * authorities. */ Downloader downloader = new Downloader(); - if (parsedCachedConsensus != null) { - downloader.setLastKnownValidAfterMillis( - parsedCachedConsensus.getValidAfterMillis()); - } downloader.downloadFromAuthorities(); + + /* Parse consensus and votes and pass them to the reports. */ + Parser parser = new Parser(); Status parsedDownloadedConsensus = parser.parse( downloader.getConsensusString(), downloader.getVoteStrings()); if (parsedDownloadedConsensus != null) { @@ -44,19 +31,6 @@ public class Main { } }
- /* Save the new consensus and corresponding votes to disk and delete - * all previous ones. */ - if (parsedDownloadedConsensus != null) { - archiver.saveStatusStringToDisk( - parsedDownloadedConsensus.getUnparsedString(), - parsedDownloadedConsensus.getFileName()); - for (Status vote : parsedDownloadedConsensus.getVotes()) { - archiver.saveStatusStringToDisk(vote.getUnparsedString(), - vote.getFileName()); - } - archiver.deleteAllButLast(); - } - /* Finish writing reports. */ for (Report report : reports) { report.writeReport(); diff --git a/src/org/torproject/chc/MetricsWebsiteReport.java b/src/org/torproject/chc/MetricsWebsiteReport.java index 2a54927..f77b89f 100644 --- a/src/org/torproject/chc/MetricsWebsiteReport.java +++ b/src/org/torproject/chc/MetricsWebsiteReport.java @@ -25,11 +25,6 @@ public class MetricsWebsiteReport implements Report { this.htmlOutputFile = new File(htmlOutputFilename); }
- /* Process the last but one consensus (ignored in this report). */ - public void processCachedConsensus(Status cachedConsensus) { - /* Do nothing. */ - } - /* Store the downloaded consensus and corresponding votes for later * processing. */ private Status downloadedConsensus; diff --git a/src/org/torproject/chc/NagiosReport.java b/src/org/torproject/chc/NagiosReport.java index 11bdaa3..2b5a6d3 100644 --- a/src/org/torproject/chc/NagiosReport.java +++ b/src/org/torproject/chc/NagiosReport.java @@ -18,12 +18,6 @@ public class NagiosReport implements Report { this.nagiosOutputFile = new File(nagiosOutputFilename); }
- /* Store the cached consensus for processing. */ - private Status cachedConsensus; - public void processCachedConsensus(Status cachedConsensus) { - this.cachedConsensus = cachedConsensus; - } - /* Store the current consensus and corresponding votes for * processing. */ private Status downloadedConsensus; @@ -58,8 +52,6 @@ public class NagiosReport implements Report { this.checkMissingVotes(); this.checkBandwidthScanners(); } - } else if (this.cachedConsensus != null) { - this.checkConsensusValid(this.cachedConsensus); } else { this.nagiosUnknowns.add("No consensus known"); } @@ -204,17 +196,6 @@ public class NagiosReport implements Report { } }
- /* Check that the most recent consensus is not more than 3 hours - * old. */ - public void checkConsensusValid(Status consensus) { - if (consensus.getValidAfterMillis() < - System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { - this.nagiosCriticals.add("The last known consensus published at " - + dateTimeFormat.format(consensus.getValidAfterMillis()) - + " is more than 3 hours old and therefore not valid anymore"); - } - } - /* Write all output to the Nagios status file. The most severe status * goes in the first line of the output file and the same status and all * log messages in the second line. */ diff --git a/src/org/torproject/chc/Parser.java b/src/org/torproject/chc/Parser.java index f7e9b57..47609a8 100644 --- a/src/org/torproject/chc/Parser.java +++ b/src/org/torproject/chc/Parser.java @@ -28,12 +28,9 @@ public class Parser {
/* Date-time formats to parse and format timestamps. */ private static SimpleDateFormat dateTimeFormat; - private static SimpleDateFormat fileNameFormat; static { dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - fileNameFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - fileNameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); }
/* Parse a consensus or vote string into a Status instance. */ @@ -143,14 +140,6 @@ public class Parser { br.close(); status.setRunningRelays(runningRelays); status.setBandwidthWeights(bandwidthWeights); - if (isConsensus) { - status.setFileName(fileNameFormat.format( - status.getValidAfterMillis()) + "-consensus"); - } else { - status.setFileName(fileNameFormat.format( - status.getValidAfterMillis()) + "-vote-" - + status.getFingerprint()); - } } catch (IOException e) { System.err.println("Caught an IOException while parsing a " + (isConsensus ? "consensus" : "vote") + " string. Skipping."); diff --git a/src/org/torproject/chc/Report.java b/src/org/torproject/chc/Report.java index 28483cc..c3937e0 100644 --- a/src/org/torproject/chc/Report.java +++ b/src/org/torproject/chc/Report.java @@ -8,10 +8,6 @@ import java.util.*; * some form. */ public interface Report {
- /* Process the cached consensus and corresponding votes to compare them - * to the downloaded ones. */ - public abstract void processCachedConsensus(Status cachedConsensus); - /* Process the downloaded current consensus and corresponding votes to * find irregularities between them. */ public abstract void processDownloadedConsensus( diff --git a/src/org/torproject/chc/Status.java b/src/org/torproject/chc/Status.java index 51a9052..cbf7395 100644 --- a/src/org/torproject/chc/Status.java +++ b/src/org/torproject/chc/Status.java @@ -29,15 +29,6 @@ public class Status implements Comparable<Status> { return this.unparsedString; }
- /* File name when writing this status to disk. */ - private String fileName; - public void setFileName(String fileName) { - this.fileName = fileName; - } - public String getFileName() { - return this.fileName; - } - /* Votes published at the same time as this consensus; votes don't * reference any statuses. */ private SortedSet<Status> votes = new TreeSet<Status>(); diff --git a/src/org/torproject/chc/StdOutReport.java b/src/org/torproject/chc/StdOutReport.java index f99c2eb..15a9148 100644 --- a/src/org/torproject/chc/StdOutReport.java +++ b/src/org/torproject/chc/StdOutReport.java @@ -21,12 +21,6 @@ public class StdOutReport implements Report { dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); }
- /* Cached consensus for later processing. */ - private Status cachedConsensus; - public void processCachedConsensus(Status cachedConsensus) { - this.cachedConsensus = cachedConsensus; - } - /* Downloaded consensus and corresponding votes for later * processing. */ private Status downloadedConsensus; @@ -49,8 +43,6 @@ public class StdOutReport implements Report { this.checkMissingVotes(); this.checkBandwidthScanners(); } - } else if (this.cachedConsensus != null) { - this.checkConsensusValid(this.cachedConsensus); } else { this.warnings.put("No consensus known", 0L); } @@ -242,17 +234,6 @@ public class StdOutReport implements Report { } }
- /* Check if the most recent consensus is older than 3 hours. */ - private void checkConsensusValid(Status consensus) { - if (consensus.getValidAfterMillis() < - System.currentTimeMillis() - 3L * 60L * 60L * 1000L) { - this.warnings.put("The last known consensus published at " - + dateTimeFormat.format(consensus.getValidAfterMillis()) - + " is more than 3 hours old and therefore not valid anymore", - 0L); - } - } - /* Prepare a report to be written to stdout. */ private String preparedReport = null; private void prepareReport() {