[tor-commits] [metrics-web/master] Generate consensus-health page as part of metrics-web.
karsten at torproject.org
karsten at torproject.org
Tue Mar 1 14:22:35 UTC 2011
commit 69f759b2ed4fa7be3837f44d6cc63d53278995b9
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Mon Feb 28 12:09:02 2011 +0100
Generate consensus-health page as part of metrics-web.
---
.gitignore | 3 +
build.xml | 19 +-
config.template | 16 +
run.sh | 5 +
src/org/torproject/ernie/cron/ArchiveReader.java | 123 +++
src/org/torproject/ernie/cron/Configuration.java | 86 ++
.../ernie/cron/ConsensusHealthChecker.java | 912 ++++++++++++++++++++
src/org/torproject/ernie/cron/LockFile.java | 47 +
.../ernie/cron/LoggingConfiguration.java | 88 ++
src/org/torproject/ernie/cron/Main.java | 67 ++
.../ernie/cron/RelayDescriptorParser.java | 91 ++
11 files changed, 1455 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore
index 2ebd1c3..aebf570 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,6 @@ classes/
# Possibly modified config file
etc/context.xml
+# Copied and possibly modified config file
+config
+
diff --git a/build.xml b/build.xml
index eb3b12a..86ce310 100644
--- a/build.xml
+++ b/build.xml
@@ -1,4 +1,4 @@
-<project name="metrics-web" basedir=".">
+<project default="run" name="metrics-web" basedir=".">
<!-- Define build paths. -->
<property name="sources" value="src"/>
@@ -10,15 +10,21 @@
value="${config}/context.xml.template"/>
<property name="contextxml" value="${config}/context.xml"/>
<property name="warfile" value="ernie.war"/>
+ <path id="classpath">
+ <pathelement path="${classes}"/>
+ <pathelement location="lib/commons-codec-1.4.jar"/>
+ </path>
<target name="init">
<copy file="${contextxmltemplate}" tofile="${contextxml}"/>
+ <copy file="config.template" tofile="config"/>
+ <mkdir dir="${classes}"/>
+ <mkdir dir="website"/>
</target>
<!-- Compile all servlets and plain Java classes. -->
<target name="compile"
depends="init">
- <mkdir dir="${classes}"/>
<javac destdir="${classes}"
srcdir="${sources}"
source="1.5"
@@ -34,6 +40,15 @@
</javac>
</target>
+ <!-- Prepare data for being displayed on the website. -->
+ <target name="run" depends="compile">
+ <java fork="true"
+ maxmemory="1024m"
+ classname="org.torproject.ernie.cron.Main">
+ <classpath refid="classpath"/>
+ </java>
+ </target>
+
<!-- Create a .war file for deployment. -->
<target name="make-war"
depends="compile">
diff --git a/config.template b/config.template
new file mode 100644
index 0000000..479d78c
--- /dev/null
+++ b/config.template
@@ -0,0 +1,16 @@
+## Import directory archives from disk, if available
+#ImportDirectoryArchives 0
+#
+## Relative path to directory to import directory archives from
+#DirectoryArchivesDirectory archives/
+#
+## Keep a history of imported directory archive files to know which files
+## have been imported before. This history can be useful when importing
+## from a changing source to avoid importing descriptors over and over
+## again, but it can be confusing to users who don't know about it.
+#KeepDirectoryArchiveImportHistory 0
+#
+## Write statistics about the current consensus and votes to the
+## website
+#WriteConsensusHealth 0
+
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..e804876
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# TODO is there a better way to suppress Ant's output?
+ant -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: "
+
diff --git a/src/org/torproject/ernie/cron/ArchiveReader.java b/src/org/torproject/ernie/cron/ArchiveReader.java
new file mode 100644
index 0000000..b21233d
--- /dev/null
+++ b/src/org/torproject/ernie/cron/ArchiveReader.java
@@ -0,0 +1,123 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.util.*;
+import java.util.logging.*;
+
+/**
+ * Read in all files in a given directory and pass buffered readers of
+ * them to the relay descriptor parser.
+ */
+public class ArchiveReader {
+ public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory,
+ File statsDirectory, boolean keepImportHistory) {
+
+ if (rdp == null || archivesDirectory == null ||
+ statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ int parsedFiles = 0, ignoredFiles = 0;
+ Logger logger = Logger.getLogger(ArchiveReader.class.getName());
+ SortedSet<String> lastArchivesImportHistory = new TreeSet<String>();
+ SortedSet<String> newArchivesImportHistory = new TreeSet<String>();
+ File archivesImportHistoryFile = new File(statsDirectory,
+ "archives-import-history");
+ if (keepImportHistory && archivesImportHistoryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ archivesImportHistoryFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ lastArchivesImportHistory.add(line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read in archives import "
+ + "history file. Skipping.");
+ }
+ }
+ if (archivesDirectory.exists()) {
+ logger.fine("Importing files in directory " + archivesDirectory
+ + "/...");
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(archivesDirectory);
+ List<File> problems = new ArrayList<File>();
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ filesInInputDir.add(f);
+ }
+ } else {
+ try {
+ if (keepImportHistory) {
+ newArchivesImportHistory.add(pop.getName());
+ }
+ if (keepImportHistory &&
+ lastArchivesImportHistory.contains(pop.getName())) {
+ ignoredFiles++;
+ continue;
+ } else if (pop.getName().endsWith(".tar.bz2")) {
+ logger.warning("Cannot parse compressed tarball "
+ + pop.getAbsolutePath() + ". Skipping.");
+ continue;
+ }
+ FileInputStream fis = new FileInputStream(pop);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ rdp.parse(allData);
+ parsedFiles++;
+ } catch (IOException e) {
+ problems.add(pop);
+ if (problems.size() > 3) {
+ break;
+ }
+ }
+ }
+ }
+ if (problems.isEmpty()) {
+ logger.fine("Finished importing files in directory "
+ + archivesDirectory + "/.");
+ } else {
+ StringBuilder sb = new StringBuilder("Failed importing files in "
+ + "directory " + archivesDirectory + "/:");
+ int printed = 0;
+ for (File f : problems) {
+ sb.append("\n " + f.getAbsolutePath());
+ if (++printed >= 3) {
+ sb.append("\n ... more");
+ break;
+ }
+ }
+ }
+ }
+ if (keepImportHistory) {
+ try {
+ archivesImportHistoryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ archivesImportHistoryFile));
+ for (String line : newArchivesImportHistory) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write archives import "
+ + "history file.");
+ }
+ }
+ logger.info("Finished importing relay descriptors from local "
+ + "directory:\nParsed " + parsedFiles + ", ignored "
+ + ignoredFiles + " files.");
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java
new file mode 100644
index 0000000..6b76dc7
--- /dev/null
+++ b/src/org/torproject/ernie/cron/Configuration.java
@@ -0,0 +1,86 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+/**
+ * Initialize configuration with hard-coded defaults, overwrite with
+ * configuration in config file, if exists, and answer Main.java about our
+ * configuration.
+ */
+public class Configuration {
+ private boolean importDirectoryArchives = false;
+ private String directoryArchivesDirectory = "archives/";
+ private boolean keepDirectoryArchiveImportHistory = false;
+ private boolean writeConsensusHealth = false;
+ public Configuration() {
+
+ /* Initialize logger. */
+ Logger logger = Logger.getLogger(Configuration.class.getName());
+
+ /* Read config file, if present. */
+ File configFile = new File("config");
+ if (!configFile.exists()) {
+ logger.warning("Could not find config file.");
+ return;
+ }
+ String line = null;
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(configFile));
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("#") || line.length() < 1) {
+ continue;
+ } else if (line.startsWith("ImportDirectoryArchives")) {
+ this.importDirectoryArchives = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DirectoryArchivesDirectory")) {
+ this.directoryArchivesDirectory = line.split(" ")[1];
+ } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
+ this.keepDirectoryArchiveImportHistory = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("WriteConsensusHealth")) {
+ this.writeConsensusHealth = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else {
+ logger.severe("Configuration file contains unrecognized "
+ + "configuration key in line '" + line + "'! Exiting!");
+ System.exit(1);
+ }
+ }
+ br.close();
+ } catch (ArrayIndexOutOfBoundsException e) {
+ logger.severe("Configuration file contains configuration key "
+ + "without value in line '" + line + "'. Exiting!");
+ System.exit(1);
+ } catch (MalformedURLException e) {
+ logger.severe("Configuration file contains illegal URL or IP:port "
+ + "pair in line '" + line + "'. Exiting!");
+ System.exit(1);
+ } catch (NumberFormatException e) {
+ logger.severe("Configuration file contains illegal value in line '"
+ + line + "' with legal values being 0 or 1. Exiting!");
+ System.exit(1);
+ } catch (IOException e) {
+ logger.log(Level.SEVERE, "Unknown problem while reading config "
+ + "file! Exiting!", e);
+ System.exit(1);
+ }
+ }
+ public boolean getImportDirectoryArchives() {
+ return this.importDirectoryArchives;
+ }
+ public String getDirectoryArchivesDirectory() {
+ return this.directoryArchivesDirectory;
+ }
+ public boolean getKeepDirectoryArchiveImportHistory() {
+ return this.keepDirectoryArchiveImportHistory;
+ }
+ public boolean getWriteConsensusHealth() {
+ return this.writeConsensusHealth;
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/ConsensusHealthChecker.java b/src/org/torproject/ernie/cron/ConsensusHealthChecker.java
new file mode 100644
index 0000000..b1ad6b6
--- /dev/null
+++ b/src/org/torproject/ernie/cron/ConsensusHealthChecker.java
@@ -0,0 +1,912 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.logging.*;
+import org.apache.commons.codec.binary.*;
+
+/*
+ * TODO Possible extensions:
+ * - Include consensus signatures and tell by which Tor versions the
+ * consensus will be accepted (and by which not)
+ */
+public class ConsensusHealthChecker {
+
+ private String mostRecentValidAfterTime = null;
+
+ private byte[] mostRecentConsensus = null;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ private SortedMap<String, byte[]> mostRecentVotes =
+ new TreeMap<String, byte[]>();
+
+ public ConsensusHealthChecker() {
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(
+ ConsensusHealthChecker.class.getName());
+ }
+
+ public void processConsensus(String validAfterTime, byte[] data) {
+ /* Do we already know a consensus and/or vote(s)? */
+ if (this.mostRecentValidAfterTime != null) {
+ int compareKnownToNew =
+ this.mostRecentValidAfterTime.compareTo(validAfterTime);
+ if (compareKnownToNew > 0) {
+ /* The consensus or vote(s) we know are more recent than this
+ * consensus. No need to store it. */
+ return;
+ } else if (compareKnownToNew < 0) {
+ /* This consensus is newer than the known consensus or vote(s).
+ * Discard all known votes and overwrite the consensus below. */
+ this.mostRecentVotes.clear();
+ }
+ }
+ /* Store this consensus. */
+ this.mostRecentValidAfterTime = validAfterTime;
+ this.mostRecentConsensus = data;
+ }
+
+ public void processVote(String validAfterTime, String dirSource,
+ byte[] data) {
+ if (this.mostRecentValidAfterTime == null ||
+ this.mostRecentValidAfterTime.compareTo(validAfterTime) < 0) {
+ /* This vote is more recent than the known consensus. Discard the
+ * consensus and all currently known votes. */
+ this.mostRecentValidAfterTime = validAfterTime;
+ this.mostRecentVotes.clear();
+ this.mostRecentConsensus = null;
+ }
+ if (this.mostRecentValidAfterTime.equals(validAfterTime)) {
+ /* Store this vote which belongs to the known consensus and/or
+ * other votes. */
+ this.mostRecentVotes.put(dirSource, data);
+ }
+ }
+
+ public void writeStatusWebsite() {
+
+ /* If we don't have any consensus, we cannot write useful consensus
+ * health information to the website. Do not overwrite existing page
+ * with a warning, because we might just not have learned about a new
+ * consensus in this execution. */
+ if (this.mostRecentConsensus == null) {
+ return;
+ }
+
+ /* Prepare parsing dates. */
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ StringBuilder knownFlagsResults = new StringBuilder();
+ StringBuilder numRelaysVotesResults = new StringBuilder();
+ StringBuilder consensusMethodsResults = new StringBuilder();
+ StringBuilder versionsResults = new StringBuilder();
+ StringBuilder paramsResults = new StringBuilder();
+ StringBuilder authorityKeysResults = new StringBuilder();
+ StringBuilder bandwidthScannersResults = new StringBuilder();
+ StringBuilder authorityVersionsResults = new StringBuilder();
+ SortedSet<String> allKnownFlags = new TreeSet<String>();
+ SortedSet<String> allKnownVotes = new TreeSet<String>();
+ SortedMap<String, String> consensusAssignedFlags =
+ new TreeMap<String, String>();
+ SortedMap<String, SortedSet<String>> votesAssignedFlags =
+ new TreeMap<String, SortedSet<String>>();
+ SortedMap<String, String> votesKnownFlags =
+ new TreeMap<String, String>();
+ SortedMap<String, SortedMap<String, Integer>> flagsAgree =
+ new TreeMap<String, SortedMap<String, Integer>>();
+ SortedMap<String, SortedMap<String, Integer>> flagsLost =
+ new TreeMap<String, SortedMap<String, Integer>>();
+ SortedMap<String, SortedMap<String, Integer>> flagsMissing =
+ new TreeMap<String, SortedMap<String, Integer>>();
+
+
+ /* Read consensus and parse all information that we want to compare to
+ * votes. */
+ String consensusConsensusMethod = null, consensusKnownFlags = null,
+ consensusClientVersions = null, consensusServerVersions = null,
+ consensusParams = null, rLineTemp = null, sLineTemp = null;
+ int consensusTotalRelays = 0, consensusRunningRelays = 0;
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ this.mostRecentConsensus)));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("consensus-method ")) {
+ consensusConsensusMethod = line;
+ } else if (line.startsWith("client-versions ")) {
+ consensusClientVersions = line;
+ } else if (line.startsWith("server-versions ")) {
+ consensusServerVersions = line;
+ } else if (line.startsWith("known-flags ")) {
+ consensusKnownFlags = line;
+ } else if (line.startsWith("params ")) {
+ consensusParams = line;
+ } else if (line.startsWith("r ")) {
+ rLineTemp = line;
+ } else if (line.startsWith("s ")) {
+ sLineTemp = line;
+ consensusTotalRelays++;
+ if (line.contains(" Running")) {
+ consensusRunningRelays++;
+ }
+ consensusAssignedFlags.put(Hex.encodeHexString(
+ Base64.decodeBase64(rLineTemp.split(" ")[2] + "=")).
+ toUpperCase() + " " + rLineTemp.split(" ")[1], line);
+ } else if (line.startsWith("v ") &&
+ sLineTemp.contains(" Authority")) {
+ authorityVersionsResults.append(" <tr>\n"
+ + " <td>" + rLineTemp.split(" ")[1] + "</td>\n"
+ + " <td>" + line.substring(2) + "</td>\n"
+ + " </tr>\n");
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ /* There should be no I/O taking place when reading a String. */
+ }
+
+ /* Read votes and parse all information to compare with the
+ * consensus. */
+ for (byte[] voteBytes : this.mostRecentVotes.values()) {
+ String voteConsensusMethods = null, voteKnownFlags = null,
+ voteClientVersions = null, voteServerVersions = null,
+ voteParams = null, dirSource = null, voteDirKeyExpires = null;
+ int voteTotalRelays = 0, voteRunningRelays = 0,
+ voteContainsBandwidthWeights = 0;
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(voteBytes)));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("consensus-methods ")) {
+ voteConsensusMethods = line;
+ } else if (line.startsWith("client-versions ")) {
+ voteClientVersions = line;
+ } else if (line.startsWith("server-versions ")) {
+ voteServerVersions = line;
+ } else if (line.startsWith("known-flags ")) {
+ voteKnownFlags = line;
+ } else if (line.startsWith("params ")) {
+ voteParams = line;
+ } else if (line.startsWith("dir-source ")) {
+ dirSource = line.split(" ")[1];
+ allKnownVotes.add(dirSource);
+ } else if (line.startsWith("dir-key-expires ")) {
+ voteDirKeyExpires = line;
+ } else if (line.startsWith("r ")) {
+ rLineTemp = line;
+ } else if (line.startsWith("s ")) {
+ voteTotalRelays++;
+ if (line.contains(" Running")) {
+ voteRunningRelays++;
+ }
+ String relayKey = Hex.encodeHexString(Base64.decodeBase64(
+ rLineTemp.split(" ")[2] + "=")).toUpperCase() + " "
+ + rLineTemp.split(" ")[1];
+ SortedSet<String> sLines = null;
+ if (votesAssignedFlags.containsKey(relayKey)) {
+ sLines = votesAssignedFlags.get(relayKey);
+ } else {
+ sLines = new TreeSet<String>();
+ votesAssignedFlags.put(relayKey, sLines);
+ }
+ sLines.add(dirSource + " " + line);
+ } else if (line.startsWith("w ")) {
+ if (line.contains(" Measured")) {
+ voteContainsBandwidthWeights++;
+ }
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ /* There should be no I/O taking place when reading a String. */
+ }
+
+ /* Write known flags. */
+ knownFlagsResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteKnownFlags + "</td>\n"
+ + " </tr>\n");
+ votesKnownFlags.put(dirSource, voteKnownFlags);
+ for (String flag : voteKnownFlags.substring(
+ "known-flags ".length()).split(" ")) {
+ allKnownFlags.add(flag);
+ }
+
+ /* Write number of relays voted about. */
+ numRelaysVotesResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteTotalRelays + " total</td>\n"
+ + " <td>" + voteRunningRelays + " Running</td>\n"
+ + " </tr>\n");
+
+ /* Write supported consensus methods. */
+ if (!voteConsensusMethods.contains(consensusConsensusMethod.
+ split(" ")[1])) {
+ consensusMethodsResults.append(" <tr>\n"
+ + " <td><font color=\"red\">" + dirSource
+ + "</font></td>\n"
+ + " <td><font color=\"red\">"
+ + voteConsensusMethods + "</font></td>\n"
+ + " </tr>\n");
+ this.logger.warning(dirSource + " does not support consensus "
+ + "method " + consensusConsensusMethod.split(" ")[1] + ": "
+ + voteConsensusMethods);
+ } else {
+ consensusMethodsResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteConsensusMethods + "</td>\n"
+ + " </tr>\n");
+ this.logger.fine(dirSource + " supports consensus method "
+ + consensusConsensusMethod.split(" ")[1] + ": "
+ + voteConsensusMethods);
+ }
+
+ /* Write recommended versions. */
+ if (voteClientVersions == null) {
+ /* Not a versioning authority. */
+ } else if (!voteClientVersions.equals(consensusClientVersions)) {
+ versionsResults.append(" <tr>\n"
+ + " <td><font color=\"red\">" + dirSource
+ + "</font></td>\n"
+ + " <td><font color=\"red\">"
+ + voteClientVersions + "</font></td>\n"
+ + " </tr>\n");
+ this.logger.warning(dirSource + " recommends other client "
+ + "versions than the consensus: " + voteClientVersions);
+ } else {
+ versionsResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteClientVersions + "</td>\n"
+ + " </tr>\n");
+ this.logger.fine(dirSource + " recommends the same client "
+ + "versions as the consensus: " + voteClientVersions);
+ }
+ if (voteServerVersions == null) {
+ /* Not a versioning authority. */
+ } else if (!voteServerVersions.equals(consensusServerVersions)) {
+ versionsResults.append(" <tr>\n"
+ + " <td></td>\n"
+ + " <td><font color=\"red\">"
+ + voteServerVersions + "</font></td>\n"
+ + " </tr>\n");
+ this.logger.warning(dirSource + " recommends other server "
+ + "versions than the consensus: " + voteServerVersions);
+ } else {
+ versionsResults.append(" <tr>\n"
+ + " <td></td>\n"
+ + " <td>" + voteServerVersions + "</td>\n"
+ + " </tr>\n");
+ this.logger.fine(dirSource + " recommends the same server "
+ + "versions as the consensus: " + voteServerVersions);
+ }
+
+ /* Write consensus parameters. */
+ boolean conflictOrInvalid = false;
+ Set<String> validParameters = new HashSet<String>(Arrays.asList(
+ ("circwindow,CircuitPriorityHalflifeMsec,refuseunknownexits,"
+ + "cbtdisabled,cbtnummodes,cbtrecentcount,cbtmaxtimeouts,"
+ + "cbtmincircs,cbtquantile,cbtclosequantile,cbttestfreq,"
+ + "cbtmintimeout,cbtinitialtimeout").split(",")));
+ if (voteParams == null) {
+ /* Authority doesn't set consensus parameters. */
+ } else {
+ for (String param : voteParams.split(" ")) {
+ if (!param.equals("params") &&
+ (!consensusParams.contains(param) ||
+ !validParameters.contains(param.split("=")[0]))) {
+ conflictOrInvalid = true;
+ break;
+ }
+ }
+ }
+ if (conflictOrInvalid) {
+ paramsResults.append(" <tr>\n"
+ + " <td><font color=\"red\">" + dirSource
+ + "</font></td>\n"
+ + " <td><font color=\"red\">"
+ + voteParams + "</font></td>\n"
+ + " </tr>\n");
+ this.logger.warning(dirSource + " sets conflicting or invalid "
+ + "consensus parameters: " + voteParams);
+ } else {
+ paramsResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteParams + "</td>\n"
+ + " </tr>\n");
+ this.logger.fine(dirSource + " sets only non-conflicting and "
+ + "valid consensus parameters: " + voteParams);
+ }
+
+ /* Write authority key expiration date. */
+ if (voteDirKeyExpires != null) {
+ boolean expiresIn14Days = false;
+ try {
+ expiresIn14Days = (System.currentTimeMillis()
+ + 14L * 24L * 60L * 60L * 1000L >
+ dateTimeFormat.parse(voteDirKeyExpires.substring(
+ "dir-key-expires ".length())).getTime());
+ } catch (ParseException e) {
+ /* Can't parse the timestamp? Whatever. */
+ }
+ if (expiresIn14Days) {
+ authorityKeysResults.append(" <tr>\n"
+ + " <td><font color=\"red\">" + dirSource
+ + "</font></td>\n"
+ + " <td><font color=\"red\">"
+ + voteDirKeyExpires + "</font></td>\n"
+ + " </tr>\n");
+ this.logger.warning(dirSource + "'s certificate expires in the "
+ + "next 14 days: " + voteDirKeyExpires);
+ } else {
+ authorityKeysResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteDirKeyExpires + "</td>\n"
+ + " </tr>\n");
+ this.logger.fine(dirSource + "'s certificate does not "
+ + "expire in the next 14 days: " + voteDirKeyExpires);
+ }
+ }
+
+ /* Write results for bandwidth scanner status. */
+ if (voteContainsBandwidthWeights > 0) {
+ bandwidthScannersResults.append(" <tr>\n"
+ + " <td>" + dirSource + "</td>\n"
+ + " <td>" + voteContainsBandwidthWeights
+ + " Measured values in w lines</td>\n"
+ + " </tr>\n");
+ }
+ }
+
+ /* Check if we're missing a vote. TODO make this configurable */
+ SortedSet<String> knownAuthorities = new TreeSet<String>(
+ Arrays.asList(("dannenberg,dizum,gabelmoo,ides,maatuska,moria1,"
+ + "tor26,urras").split(",")));
+ for (String dir : allKnownVotes) {
+ knownAuthorities.remove(dir);
+ }
+ if (!knownAuthorities.isEmpty()) {
+ StringBuilder sb = new StringBuilder();
+ for (String dir : knownAuthorities) {
+ sb.append(", " + dir);
+ }
+ this.logger.warning("We're missing votes from the following "
+ + "directory authorities: " + sb.toString().substring(2));
+ }
+
+ try {
+
+ /* Keep the past two consensus health statuses. */
+ File file0 = new File("website/consensus-health.html");
+ File file1 = new File("website/consensus-health-1.html");
+ File file2 = new File("website/consensus-health-2.html");
+ if (file2.exists()) {
+ file2.delete();
+ }
+ if (file1.exists()) {
+ file1.renameTo(file2);
+ }
+ if (file0.exists()) {
+ file0.renameTo(file1);
+ }
+
+ /* Start writing web page. */
+ BufferedWriter bw = new BufferedWriter(
+ new FileWriter("website/consensus-health.html"));
+ bw.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 "
+ + "Transitional//EN\">\n"
+ + "<html>\n"
+ + " <head>\n"
+ + " <title>Tor Metrics Portal: Consensus health</title>\n"
+ + " <meta http-equiv=\"content-type\" content=\"text/html; "
+ + "charset=ISO-8859-1\">\n"
+ + " <link href=\"/css/stylesheet-ltr.css\" type=\"text/css\" "
+ + "rel=\"stylesheet\">\n"
+ + " <link href=\"/images/favicon.ico\" "
+ + "type=\"image/x-icon\" rel=\"shortcut icon\">\n"
+ + " </head>\n"
+ + " <body>\n"
+ + " <div class=\"center\">\n"
+ + " <table class=\"banner\" border=\"0\" "
+ + "cellpadding=\"0\" cellspacing=\"0\" summary=\"\">\n"
+ + " <tr>\n"
+ + " <td class=\"banner-left\"><a "
+ + "href=\"/index.html\"><img src=\"/images/top-left.png\" "
+ + "alt=\"Click to go to home page\" width=\"193\" "
+ + "height=\"79\"></a></td>\n"
+ + " <td class=\"banner-middle\">\n"
+ + " <a href=\"/\">Home</a>\n"
+ + " <a href=\"graphs.html\">Graphs</a>\n"
+ + " <a href=\"research.html\">Research</a>\n"
+ + " <a href=\"status.html\">Status</a>\n"
+ + " <br>\n"
+ + " <font size=\"2\">\n"
+ + " <a href=\"exonerator.html\">ExoneraTor</a>\n"
+ + " <a href=\"relay-search.html\">Relay Search</a>\n"
+ + " <a class=\"current\">Consensus Health</a>\n"
+ + " </font>\n"
+ + " </td>\n"
+ + " <td class=\"banner-right\"></td>\n"
+ + " </tr>\n"
+ + " </table>\n"
+ + " <div class=\"main-column\">\n"
+ + " <h2>Tor Metrics Portal: Consensus Health</h2>\n"
+ + " <br>\n"
+ + " <p>This page shows statistics about the current "
+ + "consensus and votes to facilitate debugging of the "
+ + "directory consensus process.</p>\n");
+
+ /* Write valid-after time. */
+ bw.write(" <br>\n"
+ + " <h3>Valid-after time</h3>\n"
+ + " <br>\n"
+ + " <p>Consensus was published ");
+ boolean consensusIsStale = false;
+ try {
+ consensusIsStale = System.currentTimeMillis()
+ - 3L * 60L * 60L * 1000L >
+ dateTimeFormat.parse(this.mostRecentValidAfterTime).getTime();
+ } catch (ParseException e) {
+ /* Can't parse the timestamp? Whatever. */
+ }
+ if (consensusIsStale) {
+ bw.write("<font color=\"red\">" + this.mostRecentValidAfterTime
+ + "</font>");
+ this.logger.warning("The last consensus published at "
+ + this.mostRecentValidAfterTime + " is more than 3 hours "
+ + "old.");
+ } else {
+ bw.write(this.mostRecentValidAfterTime);
+ this.logger.fine("The last consensus published at "
+ + this.mostRecentValidAfterTime + " is less than 3 hours "
+ + "old.");
+ }
+ bw.write(". <i>Note that it takes "
+ + "15 to 30 minutes for the metrics portal to learn about "
+ + "new consensus and votes and process them.</i></p>\n");
+
+ /* Write known flags. */
+ bw.write(" <br>\n"
+ + " <h3>Known flags</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ if (knownFlagsResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n");
+ } else {
+ bw.write(knownFlagsResults.toString());
+ }
+ bw.write(" <tr>\n"
+ + " <td><font color=\"blue\">consensus</font>"
+ + "</td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusKnownFlags + "</font></td>\n"
+ + " </tr>\n");
+ bw.write(" </table>\n");
+
+ /* Write number of relays voted about. */
+ bw.write(" <br>\n"
+ + " <h3>Number of relays voted about</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"320\">\n"
+ + " <col width=\"320\">\n"
+ + " </colgroup>\n");
+ if (numRelaysVotesResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td><td></td></tr>\n");
+ } else {
+ bw.write(numRelaysVotesResults.toString());
+ }
+ bw.write(" <tr>\n"
+ + " <td><font color=\"blue\">consensus</font>"
+ + "</td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusTotalRelays + " total</font></td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusRunningRelays + " Running</font></td>\n"
+ + " </tr>\n");
+ bw.write(" </table>\n");
+
+ /* Write consensus methods. */
+ bw.write(" <br>\n"
+ + " <h3>Consensus methods</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ if (consensusMethodsResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n");
+ } else {
+ bw.write(consensusMethodsResults.toString());
+ }
+ bw.write(" <tr>\n"
+ + " <td><font color=\"blue\">consensus</font>"
+ + "</td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusConsensusMethod + "</font></td>\n"
+ + " </tr>\n");
+ bw.write(" </table>\n");
+
+ /* Write recommended versions. */
+ bw.write(" <br>\n"
+ + " <h3>Recommended versions</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ if (versionsResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n");
+ } else {
+ bw.write(versionsResults.toString());
+ }
+ bw.write(" <tr>\n"
+ + " <td><font color=\"blue\">consensus</font>"
+ + "</td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusClientVersions + "</font></td>\n"
+ + " </tr>\n");
+ bw.write(" <tr>\n"
+ + " <td></td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusServerVersions + "</font></td>\n"
+ + " </tr>\n");
+ bw.write(" </table>\n");
+
+ /* Write consensus parameters. */
+ bw.write(" <br>\n"
+ + " <h3>Consensus parameters</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ if (paramsResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n");
+ } else {
+ bw.write(paramsResults.toString());
+ }
+ bw.write(" <tr>\n"
+ + " <td><font color=\"blue\">consensus</font>"
+ + "</td>\n"
+ + " <td><font color=\"blue\">"
+ + consensusParams + "</font></td>\n"
+ + " </tr>\n");
+ bw.write(" </table>\n");
+
+ /* Write authority keys. */
+ bw.write(" <br>\n"
+ + " <h3>Authority keys</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ if (authorityKeysResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n");
+ } else {
+ bw.write(authorityKeysResults.toString());
+ }
+ bw.write(" </table>\n"
+ + " <br>\n"
+ + " <p><i>Note that expiration dates of legacy keys are "
+ + "not included in votes and therefore not listed here!</i>"
+ + "</p>\n");
+
+ /* Write bandwidth scanner status. */
+ bw.write(" <br>\n"
+ + " <h3>Bandwidth scanner status</h3>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ if (bandwidthScannersResults.length() < 1) {
+ bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n");
+ } else {
+ bw.write(bandwidthScannersResults.toString());
+ }
+ bw.write(" </table>\n");
+
+ /* Write authority versions. */
+ bw.write(" <br>\n"
+ + " <h3>Authority versions</h3>\n"
+ + " <br>\n");
+ if (authorityVersionsResults.length() < 1) {
+ bw.write(" <p>(No relays with Authority flag found.)"
+ + "</p>\n");
+ } else {
+ bw.write(" <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"640\">\n"
+ + " </colgroup>\n");
+ bw.write(authorityVersionsResults.toString());
+ bw.write(" </table>\n"
+ + " <br>\n"
+ + " <p><i>Note that this list of relays with the "
+ + "Authority flag may be different from the list of v3 "
+ + "directory authorities!</i></p>\n");
+ }
+
+ /* Write (huge) table with all flags. */
+ bw.write(" <br>\n"
+ + " <h3>Relay flags</h3>\n"
+ + " <br>\n"
+ + " <p>The semantics of flags written in the table is "
+ + "as follows:</p>\n"
+ + " <ul>\n"
+ + " <li><b>In vote and consensus:</b> Flag in vote "
+ + "matches flag in consensus, or relay is not listed in "
+ + "consensus (because it doesn't have the Running "
+ + "flag)</li>\n"
+ + " <li><b><font color=\"red\">Only in "
+ + "vote:</font></b> Flag in vote, but missing in the "
+ + "consensus, because there was no majority for the flag or "
+ + "the flag was invalidated (e.g., Named gets invalidated by "
+ + "Unnamed)</li>\n"
+ + " <li><b><font color=\"gray\"><s>Only in "
+ + "consensus:</s></font></b> Flag in consensus, but missing "
+ + "in a vote of a directory authority voting on this "
+ + "flag</li>\n"
+ + " <li><b><font color=\"blue\">In "
+ + "consensus:</font></b> Flag in consensus</li>\n"
+ + " </ul>\n"
+ + " <br>\n"
+ + " <p>See also the summary below the table.</p>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"120\">\n"
+ + " <col width=\"80\">\n");
+ for (int i = 0; i < allKnownVotes.size(); i++) {
+ bw.write(" <col width=\""
+ + (640 / allKnownVotes.size()) + "\">\n");
+ }
+ bw.write(" </colgroup>\n");
+ int linesWritten = 0;
+ for (Map.Entry<String, SortedSet<String>> e :
+ votesAssignedFlags.entrySet()) {
+ if (linesWritten++ % 10 == 0) {
+ bw.write(" <tr><td><br><b>Fingerprint</b></td>"
+ + "<td><br><b>Nickname</b></td>\n");
+ for (String dir : allKnownVotes) {
+ String shortDirName = dir.length() > 6 ?
+ dir.substring(0, 5) + "." : dir;
+ bw.write("<td><br><b>" + shortDirName + "</b></td>");
+ }
+ bw.write("<td><br><b>consensus</b></td></tr>\n");
+ }
+ String relayKey = e.getKey();
+ SortedSet<String> votes = e.getValue();
+ String fingerprint = relayKey.split(" ")[0].substring(0, 8);
+ String nickname = relayKey.split(" ")[1];
+ bw.write(" <tr>\n");
+ if (consensusAssignedFlags.containsKey(relayKey) &&
+ consensusAssignedFlags.get(relayKey).contains(" Named") &&
+ !Character.isDigit(nickname.charAt(0))) {
+ bw.write(" <td id=\"" + nickname
+ + "\"><a href=\"relay.html?fingerprint="
+ + relayKey.split(" ")[0] + "\" target=\"_blank\">"
+ + fingerprint + "</a></td>\n");
+ } else {
+ bw.write(" <td><a href=\"relay.html?fingerprint="
+ + fingerprint + "\" target=\"_blank\">" + fingerprint
+ + "</a></td>\n");
+ }
+ bw.write(" <td>" + nickname + "</td>\n");
+ SortedSet<String> relevantFlags = new TreeSet<String>();
+ for (String vote : votes) {
+ String[] parts = vote.split(" ");
+ for (int j = 2; j < parts.length; j++) {
+ relevantFlags.add(parts[j]);
+ }
+ }
+ String consensusFlags = null;
+ if (consensusAssignedFlags.containsKey(relayKey)) {
+ consensusFlags = consensusAssignedFlags.get(relayKey);
+ String[] parts = consensusFlags.split(" ");
+ for (int j = 1; j < parts.length; j++) {
+ relevantFlags.add(parts[j]);
+ }
+ }
+ for (String dir : allKnownVotes) {
+ String flags = null;
+ for (String vote : votes) {
+ if (vote.startsWith(dir)) {
+ flags = vote;
+ break;
+ }
+ }
+ if (flags != null) {
+ votes.remove(flags);
+ bw.write(" <td>");
+ int flagsWritten = 0;
+ for (String flag : relevantFlags) {
+ bw.write(flagsWritten++ > 0 ? "<br>" : "");
+ SortedMap<String, SortedMap<String, Integer>> sums = null;
+ if (flags.contains(" " + flag)) {
+ if (consensusFlags == null ||
+ consensusFlags.contains(" " + flag)) {
+ bw.write(flag);
+ sums = flagsAgree;
+ } else {
+ bw.write("<font color=\"red\">" + flag + "</font>");
+ sums = flagsLost;
+ }
+ } else if (consensusFlags != null &&
+ votesKnownFlags.get(dir).contains(" " + flag) &&
+ consensusFlags.contains(" " + flag)) {
+ bw.write("<font color=\"gray\"><s>" + flag
+ + "</s></font>");
+ sums = flagsMissing;
+ }
+ if (sums != null) {
+ SortedMap<String, Integer> sum = null;
+ if (sums.containsKey(dir)) {
+ sum = sums.get(dir);
+ } else {
+ sum = new TreeMap<String, Integer>();
+ sums.put(dir, sum);
+ }
+ sum.put(flag, sum.containsKey(flag) ?
+ sum.get(flag) + 1 : 1);
+ }
+ }
+ bw.write("</td>\n");
+ } else {
+ bw.write(" <td></td>\n");
+ }
+ }
+ if (consensusFlags != null) {
+ bw.write(" <td>");
+ int flagsWritten = 0;
+ for (String flag : relevantFlags) {
+ bw.write(flagsWritten++ > 0 ? "<br>" : "");
+ if (consensusFlags.contains(" " + flag)) {
+ bw.write("<font color=\"blue\">" + flag + "</font>");
+ }
+ }
+ bw.write("</td>\n");
+ } else {
+ bw.write(" <td></td>\n");
+ }
+ bw.write(" </tr>\n");
+ }
+ bw.write(" </table>\n");
+
+ /* Write summary of overlap between votes and consensus. */
+ bw.write(" <br>\n"
+ + " <h3>Overlap between votes and consensus</h3>\n"
+ + " <br>\n"
+ + " <p>The semantics of columns is similar to the "
+ + "table above:</p>\n"
+ + " <ul>\n"
+ + " <li><b>In vote and consensus:</b> Flag in vote "
+ + "matches flag in consensus, or relay is not listed in "
+ + "consensus (because it doesn't have the Running "
+ + "flag)</li>\n"
+ + " <li><b><font color=\"red\">Only in "
+ + "vote:</font></b> Flag in vote, but missing in the "
+ + "consensus, because there was no majority for the flag or "
+ + "the flag was invalidated (e.g., Named gets invalidated by "
+ + "Unnamed)</li>\n"
+ + " <li><b><font color=\"gray\"><s>Only in "
+ + "consensus:</s></font></b> Flag in consensus, but missing "
+ + "in a vote of a directory authority voting on this "
+ + "flag</li>\n"
+ + " </ul>\n"
+ + " <br>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"210\">\n"
+ + " <col width=\"210\">\n"
+ + " <col width=\"210\">\n"
+ + " </colgroup>\n");
+ bw.write(" <tr><td></td><td><b>Only in vote</b></td>"
+ + "<td><b>In vote and consensus</b></td>"
+ + "<td><b>Only in consensus</b></td>\n");
+ for (String dir : allKnownVotes) {
+ boolean firstFlagWritten = false;
+ String[] flags = votesKnownFlags.get(dir).substring(
+ "known-flags ".length()).split(" ");
+ for (String flag : flags) {
+ bw.write(" <tr>\n");
+ if (firstFlagWritten) {
+ bw.write(" <td></td>\n");
+ } else {
+ bw.write(" <td>" + dir + "</td>\n");
+ firstFlagWritten = true;
+ }
+ if (flagsLost.containsKey(dir) &&
+ flagsLost.get(dir).containsKey(flag)) {
+ bw.write(" <td><font color=\"red\"> "
+ + flagsLost.get(dir).get(flag) + " " + flag
+ + "</font></td>\n");
+ } else {
+ bw.write(" <td></td>\n");
+ }
+ if (flagsAgree.containsKey(dir) &&
+ flagsAgree.get(dir).containsKey(flag)) {
+ bw.write(" <td>" + flagsAgree.get(dir).get(flag)
+ + " " + flag + "</td>\n");
+ } else {
+ bw.write(" <td></td>\n");
+ }
+ if (flagsMissing.containsKey(dir) &&
+ flagsMissing.get(dir).containsKey(flag)) {
+ bw.write(" <td><font color=\"gray\"><s>"
+ + flagsMissing.get(dir).get(flag) + " " + flag
+ + "</s></font></td>\n");
+ } else {
+ bw.write(" <td></td>\n");
+ }
+ bw.write(" </tr>\n");
+ }
+ }
+ bw.write(" </table>\n");
+
+ /* Finish writing. */
+ bw.write(" </div>\n"
+ + " </div>\n"
+ + " <div class=\"bottom\" id=\"bottom\">\n"
+ + " <p>This material is supported in part by the "
+ + "National Science Foundation under Grant No. "
+ + "CNS-0959138. Any opinions, finding, and conclusions "
+ + "or recommendations expressed in this material are "
+ + "those of the author(s) and do not necessarily reflect "
+ + "the views of the National Science Foundation.</p>\n"
+ + " <p>\"Tor\" and the \"Onion Logo\" are <a "
+ + "href=\"https://www.torproject.org/docs/trademark-faq.html"
+ + ".en\">"
+ + "registered trademarks</a> of The Tor Project, "
+ + "Inc.</p>\n"
+ + " <p>Data on this site is freely available under a "
+ + "<a href=\"http://creativecommons.org/publicdomain/"
+ + "zero/1.0/\">CC0 no copyright declaration</a>: To the "
+ + "extent possible under law, the Tor Project has waived "
+ + "all copyright and related or neighboring rights in "
+ + "the data. Graphs are licensed under a <a "
+ + "href=\"http://creativecommons.org/licenses/by/3.0/"
+ + "us/\">Creative Commons Attribution 3.0 United States "
+ + "License</a>.</p>\n"
+ + " </div>\n"
+ + " </body>\n"
+ + "</html>");
+ bw.close();
+
+ } catch (IOException e) {
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/LockFile.java b/src/org/torproject/ernie/cron/LockFile.java
new file mode 100644
index 0000000..7a097f0
--- /dev/null
+++ b/src/org/torproject/ernie/cron/LockFile.java
@@ -0,0 +1,47 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.util.logging.*;
+
+public class LockFile {
+
+ private File lockFile;
+ private Logger logger;
+
+ public LockFile() {
+ this.lockFile = new File("lock");
+ this.logger = Logger.getLogger(LockFile.class.getName());
+ }
+
+ public boolean acquireLock() {
+ this.logger.fine("Trying to acquire lock...");
+ try {
+ if (this.lockFile.exists()) {
+ BufferedReader br = new BufferedReader(new FileReader("lock"));
+ long runStarted = Long.parseLong(br.readLine());
+ br.close();
+ if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) {
+ return false;
+ }
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter("lock"));
+ bw.append("" + System.currentTimeMillis() + "\n");
+ bw.close();
+ this.logger.fine("Acquired lock.");
+ return true;
+ } catch (IOException e) {
+ this.logger.warning("Caught exception while trying to acquire "
+ + "lock!");
+ return false;
+ }
+ }
+
+ public void releaseLock() {
+ this.logger.fine("Releasing lock...");
+ this.lockFile.delete();
+ this.logger.fine("Released lock.");
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/LoggingConfiguration.java b/src/org/torproject/ernie/cron/LoggingConfiguration.java
new file mode 100644
index 0000000..f14fc54
--- /dev/null
+++ b/src/org/torproject/ernie/cron/LoggingConfiguration.java
@@ -0,0 +1,88 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.text.*;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.logging.*;
+
+/**
+ * Initialize logging configuration.
+ *
+ * Log levels used by ERNIE:
+ *
+ * - SEVERE: An event made it impossible to continue program execution.
+ * - WARNING: A potential problem occurred that requires the operator to
+ * look after the otherwise unattended setup
+ * - INFO: Messages on INFO level are meant to help the operator in making
+ * sure that operation works as expected.
+ * - FINE: Debug messages that are used to identify problems and which are
+ * turned on by default.
+ * - FINER: More detailed debug messages to investigate problems in more
+ * detail. Not turned on by default. Increase log file limit when using
+ * FINER.
+ * - FINEST: Most detailed debug messages. Not used.
+ */
+public class LoggingConfiguration {
+
+ public LoggingConfiguration() {
+
+ /* Remove default console handler. */
+ for (Handler h : Logger.getLogger("").getHandlers()) {
+ Logger.getLogger("").removeHandler(h);
+ }
+
+ /* Disable logging of internal Sun classes. */
+ Logger.getLogger("sun").setLevel(Level.OFF);
+
+ /* Set minimum log level we care about from INFO to FINER. */
+ Logger.getLogger("").setLevel(Level.FINER);
+
+ /* Create log handler that writes messages on WARNING or higher to the
+ * console. */
+ final SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Formatter cf = new Formatter() {
+ public String format(LogRecord record) {
+ return dateTimeFormat.format(new Date(record.getMillis())) + " "
+ + record.getMessage() + "\n";
+ }
+ };
+ Handler ch = new ConsoleHandler();
+ ch.setFormatter(cf);
+ ch.setLevel(Level.WARNING);
+ Logger.getLogger("").addHandler(ch);
+
+ /* Initialize own logger for this class. */
+ Logger logger = Logger.getLogger(
+ LoggingConfiguration.class.getName());
+
+ /* Create log handler that writes all messages on FINE or higher to a
+ * local file. */
+ Formatter ff = new Formatter() {
+ public String format(LogRecord record) {
+ return dateTimeFormat.format(new Date(record.getMillis())) + " "
+ + record.getLevel() + " " + record.getSourceClassName() + " "
+ + record.getSourceMethodName() + " " + record.getMessage()
+ + (record.getThrown() != null ? " " + record.getThrown() : "")
+ + "\n";
+ }
+ };
+ try {
+ FileHandler fh = new FileHandler("log", 5000000, 5, true);
+ fh.setFormatter(ff);
+ fh.setLevel(Level.FINE);
+ Logger.getLogger("").addHandler(fh);
+ } catch (SecurityException e) {
+ logger.log(Level.WARNING, "No permission to create log file. "
+ + "Logging to file is disabled.", e);
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write to log file. Logging to "
+ + "file is disabled.", e);
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java
new file mode 100644
index 0000000..b95a133
--- /dev/null
+++ b/src/org/torproject/ernie/cron/Main.java
@@ -0,0 +1,67 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.util.*;
+import java.util.logging.*;
+
+/**
+ * Coordinate downloading and parsing of descriptors and extraction of
+ * statistically relevant data for later processing with R.
+ */
+public class Main {
+ public static void main(String[] args) {
+
+ /* Initialize logging configuration. */
+ new LoggingConfiguration();
+
+ Logger logger = Logger.getLogger(Main.class.getName());
+ logger.info("Starting ERNIE.");
+
+ // Initialize configuration
+ Configuration config = new Configuration();
+
+ // Use lock file to avoid overlapping runs
+ LockFile lf = new LockFile();
+ if (!lf.acquireLock()) {
+ logger.severe("Warning: ERNIE is already running or has not exited "
+ + "cleanly! Exiting!");
+ System.exit(1);
+ }
+
+ // Define stats directory for temporary files
+ File statsDirectory = new File("stats");
+
+ // Prepare consensus health checker
+ ConsensusHealthChecker chc = config.getWriteConsensusHealth() ?
+ new ConsensusHealthChecker() : null;
+
+ // Prepare relay descriptor parser (only if we are writing the
+ // consensus-health page to disk)
+ RelayDescriptorParser rdp = config.getWriteConsensusHealth() ?
+ new RelayDescriptorParser(chc) : null;
+
+ // Import relay descriptors
+ if (rdp != null) {
+ if (config.getImportDirectoryArchives()) {
+ new ArchiveReader(rdp,
+ new File(config.getDirectoryArchivesDirectory()),
+ statsDirectory,
+ config.getKeepDirectoryArchiveImportHistory());
+ }
+ }
+
+ // Write consensus health website
+ if (chc != null) {
+ chc.writeStatusWebsite();
+ chc = null;
+ }
+
+ // Remove lock file
+ lf.releaseLock();
+
+ logger.info("Terminating ERNIE.");
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/RelayDescriptorParser.java b/src/org/torproject/ernie/cron/RelayDescriptorParser.java
new file mode 100644
index 0000000..24b512b
--- /dev/null
+++ b/src/org/torproject/ernie/cron/RelayDescriptorParser.java
@@ -0,0 +1,91 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import java.util.logging.*;
+import org.apache.commons.codec.digest.*;
+import org.apache.commons.codec.binary.*;
+
+/**
+ * Parses relay descriptors including network status consensuses and
+ * votes, server and extra-info descriptors, and passes the results to the
+ * stats handlers, to the archive writer, or to the relay descriptor
+ * downloader.
+ */
+public class RelayDescriptorParser {
+
+ private ConsensusHealthChecker chc;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ private SimpleDateFormat dateTimeFormat;
+
+ /**
+ * Initializes this class.
+ */
+ public RelayDescriptorParser(ConsensusHealthChecker chc) {
+ this.chc = chc;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+
+ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ public void parse(byte[] data) {
+ try {
+ /* Convert descriptor to ASCII for parsing. This means we'll lose
+ * the non-ASCII chars, but we don't care about them for parsing
+ * anyway. */
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = br.readLine();
+ if (line == null) {
+ this.logger.fine("We were given an empty descriptor for "
+ + "parsing. Ignoring.");
+ return;
+ }
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (line.equals("network-status-version 3")) {
+ // TODO when parsing the current consensus, check the fresh-until
+ // time to see when we switch from hourly to half-hourly
+ // consensuses
+ boolean isConsensus = true;
+ String validAfterTime = null;
+ String dirSource = null;
+ while ((line = br.readLine()) != null) {
+ if (line.equals("vote-status vote")) {
+ isConsensus = false;
+ } else if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ } else if (line.startsWith("dir-source ")) {
+ dirSource = line.split(" ")[2];
+ break;
+ }
+ }
+ if (isConsensus) {
+ if (this.chc != null) {
+ this.chc.processConsensus(validAfterTime, data);
+ }
+ } else {
+ if (this.chc != null) {
+ this.chc.processVote(validAfterTime, dirSource, data);
+ }
+ }
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ }
+ }
+}
+
More information about the tor-commits
mailing list