commit 69f759b2ed4fa7be3837f44d6cc63d53278995b9 Author: Karsten Loesing karsten.loesing@gmx.net Date: Mon Feb 28 12:09:02 2011 +0100
Generate consensus-health page as part of metrics-web. --- .gitignore | 3 + build.xml | 19 +- config.template | 16 + run.sh | 5 + src/org/torproject/ernie/cron/ArchiveReader.java | 123 +++ src/org/torproject/ernie/cron/Configuration.java | 86 ++ .../ernie/cron/ConsensusHealthChecker.java | 912 ++++++++++++++++++++ src/org/torproject/ernie/cron/LockFile.java | 47 + .../ernie/cron/LoggingConfiguration.java | 88 ++ src/org/torproject/ernie/cron/Main.java | 67 ++ .../ernie/cron/RelayDescriptorParser.java | 91 ++ 11 files changed, 1455 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore index 2ebd1c3..aebf570 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ classes/ # Possibly modified config file etc/context.xml
+# Copied and possibly modified config file +config + diff --git a/build.xml b/build.xml index eb3b12a..86ce310 100644 --- a/build.xml +++ b/build.xml @@ -1,4 +1,4 @@ -<project name="metrics-web" basedir="."> +<project default="run" name="metrics-web" basedir=".">
<!-- Define build paths. --> <property name="sources" value="src"/> @@ -10,15 +10,21 @@ value="${config}/context.xml.template"/> <property name="contextxml" value="${config}/context.xml"/> <property name="warfile" value="ernie.war"/> + <path id="classpath"> + <pathelement path="${classes}"/> + <pathelement location="lib/commons-codec-1.4.jar"/> + </path>
<target name="init"> <copy file="${contextxmltemplate}" tofile="${contextxml}"/> + <copy file="config.template" tofile="config"/> + <mkdir dir="${classes}"/> + <mkdir dir="website"/> </target>
<!-- Compile all servlets and plain Java classes. --> <target name="compile" depends="init"> - <mkdir dir="${classes}"/> <javac destdir="${classes}" srcdir="${sources}" source="1.5" @@ -34,6 +40,15 @@ </javac> </target>
+ <!-- Prepare data for being displayed on the website. --> + <target name="run" depends="compile"> + <java fork="true" + maxmemory="1024m" + classname="org.torproject.ernie.cron.Main"> + <classpath refid="classpath"/> + </java> + </target> + <!-- Create a .war file for deployment. --> <target name="make-war" depends="compile"> diff --git a/config.template b/config.template new file mode 100644 index 0000000..479d78c --- /dev/null +++ b/config.template @@ -0,0 +1,16 @@ +## Import directory archives from disk, if available +#ImportDirectoryArchives 0 +# +## Relative path to directory to import directory archives from +#DirectoryArchivesDirectory archives/ +# +## Keep a history of imported directory archive files to know which files +## have been imported before. This history can be useful when importing +## from a changing source to avoid importing descriptors over and over +## again, but it can be confusing to users who don't know about it. +#KeepDirectoryArchiveImportHistory 0 +# +## Write statistics about the current consensus and votes to the +## website +#WriteConsensusHealth 0 + diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..e804876 --- /dev/null +++ b/run.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +# TODO is there a better way to suppress Ant's output? +ant -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " + diff --git a/src/org/torproject/ernie/cron/ArchiveReader.java b/src/org/torproject/ernie/cron/ArchiveReader.java new file mode 100644 index 0000000..b21233d --- /dev/null +++ b/src/org/torproject/ernie/cron/ArchiveReader.java @@ -0,0 +1,123 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.util.*; +import java.util.logging.*; + +/** + * Read in all files in a given directory and pass buffered readers of + * them to the relay descriptor parser. + */ +public class ArchiveReader { + public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory, + File statsDirectory, boolean keepImportHistory) { + + if (rdp == null || archivesDirectory == null || + statsDirectory == null) { + throw new IllegalArgumentException(); + } + + int parsedFiles = 0, ignoredFiles = 0; + Logger logger = Logger.getLogger(ArchiveReader.class.getName()); + SortedSet<String> lastArchivesImportHistory = new TreeSet<String>(); + SortedSet<String> newArchivesImportHistory = new TreeSet<String>(); + File archivesImportHistoryFile = new File(statsDirectory, + "archives-import-history"); + if (keepImportHistory && archivesImportHistoryFile.exists()) { + try { + BufferedReader br = new BufferedReader(new FileReader( + archivesImportHistoryFile)); + String line = null; + while ((line = br.readLine()) != null) { + lastArchivesImportHistory.add(line); + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not read in archives import " + + "history file. Skipping."); + } + } + if (archivesDirectory.exists()) { + logger.fine("Importing files in directory " + archivesDirectory + + "/..."); + Stack<File> filesInInputDir = new Stack<File>(); + filesInInputDir.add(archivesDirectory); + List<File> problems = new ArrayList<File>(); + while (!filesInInputDir.isEmpty()) { + File pop = filesInInputDir.pop(); + if (pop.isDirectory()) { + for (File f : pop.listFiles()) { + filesInInputDir.add(f); + } + } else { + try { + if (keepImportHistory) { + newArchivesImportHistory.add(pop.getName()); + } + if (keepImportHistory && + lastArchivesImportHistory.contains(pop.getName())) { + ignoredFiles++; + continue; + } else if (pop.getName().endsWith(".tar.bz2")) { + logger.warning("Cannot parse compressed tarball " + + pop.getAbsolutePath() + ". Skipping."); + continue; + } + FileInputStream fis = new FileInputStream(pop); + BufferedInputStream bis = new BufferedInputStream(fis); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + bis.close(); + byte[] allData = baos.toByteArray(); + rdp.parse(allData); + parsedFiles++; + } catch (IOException e) { + problems.add(pop); + if (problems.size() > 3) { + break; + } + } + } + } + if (problems.isEmpty()) { + logger.fine("Finished importing files in directory " + + archivesDirectory + "/."); + } else { + StringBuilder sb = new StringBuilder("Failed importing files in " + + "directory " + archivesDirectory + "/:"); + int printed = 0; + for (File f : problems) { + sb.append("\n " + f.getAbsolutePath()); + if (++printed >= 3) { + sb.append("\n ... more"); + break; + } + } + } + } + if (keepImportHistory) { + try { + archivesImportHistoryFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + archivesImportHistoryFile)); + for (String line : newArchivesImportHistory) { + bw.write(line + "\n"); + } + bw.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not write archives import " + + "history file."); + } + } + logger.info("Finished importing relay descriptors from local " + + "directory:\nParsed " + parsedFiles + ", ignored " + + ignoredFiles + " files."); + } +} + diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java new file mode 100644 index 0000000..6b76dc7 --- /dev/null +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -0,0 +1,86 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.logging.*; + +/** + * Initialize configuration with hard-coded defaults, overwrite with + * configuration in config file, if exists, and answer Main.java about our + * configuration. + */ +public class Configuration { + private boolean importDirectoryArchives = false; + private String directoryArchivesDirectory = "archives/"; + private boolean keepDirectoryArchiveImportHistory = false; + private boolean writeConsensusHealth = false; + public Configuration() { + + /* Initialize logger. */ + Logger logger = Logger.getLogger(Configuration.class.getName()); + + /* Read config file, if present. */ + File configFile = new File("config"); + if (!configFile.exists()) { + logger.warning("Could not find config file."); + return; + } + String line = null; + try { + BufferedReader br = new BufferedReader(new FileReader(configFile)); + while ((line = br.readLine()) != null) { + if (line.startsWith("#") || line.length() < 1) { + continue; + } else if (line.startsWith("ImportDirectoryArchives")) { + this.importDirectoryArchives = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DirectoryArchivesDirectory")) { + this.directoryArchivesDirectory = line.split(" ")[1]; + } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { + this.keepDirectoryArchiveImportHistory = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("WriteConsensusHealth")) { + this.writeConsensusHealth = Integer.parseInt( + line.split(" ")[1]) != 0; + } else { + logger.severe("Configuration file contains unrecognized " + + "configuration key in line '" + line + "'! Exiting!"); + System.exit(1); + } + } + br.close(); + } catch (ArrayIndexOutOfBoundsException e) { + logger.severe("Configuration file contains configuration key " + + "without value in line '" + line + "'. Exiting!"); + System.exit(1); + } catch (MalformedURLException e) { + logger.severe("Configuration file contains illegal URL or IP:port " + + "pair in line '" + line + "'. Exiting!"); + System.exit(1); + } catch (NumberFormatException e) { + logger.severe("Configuration file contains illegal value in line '" + + line + "' with legal values being 0 or 1. Exiting!"); + System.exit(1); + } catch (IOException e) { + logger.log(Level.SEVERE, "Unknown problem while reading config " + + "file! Exiting!", e); + System.exit(1); + } + } + public boolean getImportDirectoryArchives() { + return this.importDirectoryArchives; + } + public String getDirectoryArchivesDirectory() { + return this.directoryArchivesDirectory; + } + public boolean getKeepDirectoryArchiveImportHistory() { + return this.keepDirectoryArchiveImportHistory; + } + public boolean getWriteConsensusHealth() { + return this.writeConsensusHealth; + } +} + diff --git a/src/org/torproject/ernie/cron/ConsensusHealthChecker.java b/src/org/torproject/ernie/cron/ConsensusHealthChecker.java new file mode 100644 index 0000000..b1ad6b6 --- /dev/null +++ b/src/org/torproject/ernie/cron/ConsensusHealthChecker.java @@ -0,0 +1,912 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.logging.*; +import org.apache.commons.codec.binary.*; + +/* + * TODO Possible extensions: + * - Include consensus signatures and tell by which Tor versions the + * consensus will be accepted (and by which not) + */ +public class ConsensusHealthChecker { + + private String mostRecentValidAfterTime = null; + + private byte[] mostRecentConsensus = null; + + /** + * Logger for this class. + */ + private Logger logger; + + private SortedMap<String, byte[]> mostRecentVotes = + new TreeMap<String, byte[]>(); + + public ConsensusHealthChecker() { + /* Initialize logger. */ + this.logger = Logger.getLogger( + ConsensusHealthChecker.class.getName()); + } + + public void processConsensus(String validAfterTime, byte[] data) { + /* Do we already know a consensus and/or vote(s)? */ + if (this.mostRecentValidAfterTime != null) { + int compareKnownToNew = + this.mostRecentValidAfterTime.compareTo(validAfterTime); + if (compareKnownToNew > 0) { + /* The consensus or vote(s) we know are more recent than this + * consensus. No need to store it. */ + return; + } else if (compareKnownToNew < 0) { + /* This consensus is newer than the known consensus or vote(s). + * Discard all known votes and overwrite the consensus below. */ + this.mostRecentVotes.clear(); + } + } + /* Store this consensus. */ + this.mostRecentValidAfterTime = validAfterTime; + this.mostRecentConsensus = data; + } + + public void processVote(String validAfterTime, String dirSource, + byte[] data) { + if (this.mostRecentValidAfterTime == null || + this.mostRecentValidAfterTime.compareTo(validAfterTime) < 0) { + /* This vote is more recent than the known consensus. Discard the + * consensus and all currently known votes. */ + this.mostRecentValidAfterTime = validAfterTime; + this.mostRecentVotes.clear(); + this.mostRecentConsensus = null; + } + if (this.mostRecentValidAfterTime.equals(validAfterTime)) { + /* Store this vote which belongs to the known consensus and/or + * other votes. */ + this.mostRecentVotes.put(dirSource, data); + } + } + + public void writeStatusWebsite() { + + /* If we don't have any consensus, we cannot write useful consensus + * health information to the website. Do not overwrite existing page + * with a warning, because we might just not have learned about a new + * consensus in this execution. */ + if (this.mostRecentConsensus == null) { + return; + } + + /* Prepare parsing dates. */ + SimpleDateFormat dateTimeFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + StringBuilder knownFlagsResults = new StringBuilder(); + StringBuilder numRelaysVotesResults = new StringBuilder(); + StringBuilder consensusMethodsResults = new StringBuilder(); + StringBuilder versionsResults = new StringBuilder(); + StringBuilder paramsResults = new StringBuilder(); + StringBuilder authorityKeysResults = new StringBuilder(); + StringBuilder bandwidthScannersResults = new StringBuilder(); + StringBuilder authorityVersionsResults = new StringBuilder(); + SortedSet<String> allKnownFlags = new TreeSet<String>(); + SortedSet<String> allKnownVotes = new TreeSet<String>(); + SortedMap<String, String> consensusAssignedFlags = + new TreeMap<String, String>(); + SortedMap<String, SortedSet<String>> votesAssignedFlags = + new TreeMap<String, SortedSet<String>>(); + SortedMap<String, String> votesKnownFlags = + new TreeMap<String, String>(); + SortedMap<String, SortedMap<String, Integer>> flagsAgree = + new TreeMap<String, SortedMap<String, Integer>>(); + SortedMap<String, SortedMap<String, Integer>> flagsLost = + new TreeMap<String, SortedMap<String, Integer>>(); + SortedMap<String, SortedMap<String, Integer>> flagsMissing = + new TreeMap<String, SortedMap<String, Integer>>(); + + + /* Read consensus and parse all information that we want to compare to + * votes. */ + String consensusConsensusMethod = null, consensusKnownFlags = null, + consensusClientVersions = null, consensusServerVersions = null, + consensusParams = null, rLineTemp = null, sLineTemp = null; + int consensusTotalRelays = 0, consensusRunningRelays = 0; + try { + BufferedReader br = new BufferedReader(new StringReader(new String( + this.mostRecentConsensus))); + String line = null; + while ((line = br.readLine()) != null) { + if (line.startsWith("consensus-method ")) { + consensusConsensusMethod = line; + } else if (line.startsWith("client-versions ")) { + consensusClientVersions = line; + } else if (line.startsWith("server-versions ")) { + consensusServerVersions = line; + } else if (line.startsWith("known-flags ")) { + consensusKnownFlags = line; + } else if (line.startsWith("params ")) { + consensusParams = line; + } else if (line.startsWith("r ")) { + rLineTemp = line; + } else if (line.startsWith("s ")) { + sLineTemp = line; + consensusTotalRelays++; + if (line.contains(" Running")) { + consensusRunningRelays++; + } + consensusAssignedFlags.put(Hex.encodeHexString( + Base64.decodeBase64(rLineTemp.split(" ")[2] + "=")). + toUpperCase() + " " + rLineTemp.split(" ")[1], line); + } else if (line.startsWith("v ") && + sLineTemp.contains(" Authority")) { + authorityVersionsResults.append(" <tr>\n" + + " <td>" + rLineTemp.split(" ")[1] + "</td>\n" + + " <td>" + line.substring(2) + "</td>\n" + + " </tr>\n"); + } + } + br.close(); + } catch (IOException e) { + /* There should be no I/O taking place when reading a String. */ + } + + /* Read votes and parse all information to compare with the + * consensus. */ + for (byte[] voteBytes : this.mostRecentVotes.values()) { + String voteConsensusMethods = null, voteKnownFlags = null, + voteClientVersions = null, voteServerVersions = null, + voteParams = null, dirSource = null, voteDirKeyExpires = null; + int voteTotalRelays = 0, voteRunningRelays = 0, + voteContainsBandwidthWeights = 0; + try { + BufferedReader br = new BufferedReader(new StringReader( + new String(voteBytes))); + String line = null; + while ((line = br.readLine()) != null) { + if (line.startsWith("consensus-methods ")) { + voteConsensusMethods = line; + } else if (line.startsWith("client-versions ")) { + voteClientVersions = line; + } else if (line.startsWith("server-versions ")) { + voteServerVersions = line; + } else if (line.startsWith("known-flags ")) { + voteKnownFlags = line; + } else if (line.startsWith("params ")) { + voteParams = line; + } else if (line.startsWith("dir-source ")) { + dirSource = line.split(" ")[1]; + allKnownVotes.add(dirSource); + } else if (line.startsWith("dir-key-expires ")) { + voteDirKeyExpires = line; + } else if (line.startsWith("r ")) { + rLineTemp = line; + } else if (line.startsWith("s ")) { + voteTotalRelays++; + if (line.contains(" Running")) { + voteRunningRelays++; + } + String relayKey = Hex.encodeHexString(Base64.decodeBase64( + rLineTemp.split(" ")[2] + "=")).toUpperCase() + " " + + rLineTemp.split(" ")[1]; + SortedSet<String> sLines = null; + if (votesAssignedFlags.containsKey(relayKey)) { + sLines = votesAssignedFlags.get(relayKey); + } else { + sLines = new TreeSet<String>(); + votesAssignedFlags.put(relayKey, sLines); + } + sLines.add(dirSource + " " + line); + } else if (line.startsWith("w ")) { + if (line.contains(" Measured")) { + voteContainsBandwidthWeights++; + } + } + } + br.close(); + } catch (IOException e) { + /* There should be no I/O taking place when reading a String. */ + } + + /* Write known flags. */ + knownFlagsResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteKnownFlags + "</td>\n" + + " </tr>\n"); + votesKnownFlags.put(dirSource, voteKnownFlags); + for (String flag : voteKnownFlags.substring( + "known-flags ".length()).split(" ")) { + allKnownFlags.add(flag); + } + + /* Write number of relays voted about. */ + numRelaysVotesResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteTotalRelays + " total</td>\n" + + " <td>" + voteRunningRelays + " Running</td>\n" + + " </tr>\n"); + + /* Write supported consensus methods. */ + if (!voteConsensusMethods.contains(consensusConsensusMethod. + split(" ")[1])) { + consensusMethodsResults.append(" <tr>\n" + + " <td><font color="red">" + dirSource + + "</font></td>\n" + + " <td><font color="red">" + + voteConsensusMethods + "</font></td>\n" + + " </tr>\n"); + this.logger.warning(dirSource + " does not support consensus " + + "method " + consensusConsensusMethod.split(" ")[1] + ": " + + voteConsensusMethods); + } else { + consensusMethodsResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteConsensusMethods + "</td>\n" + + " </tr>\n"); + this.logger.fine(dirSource + " supports consensus method " + + consensusConsensusMethod.split(" ")[1] + ": " + + voteConsensusMethods); + } + + /* Write recommended versions. */ + if (voteClientVersions == null) { + /* Not a versioning authority. */ + } else if (!voteClientVersions.equals(consensusClientVersions)) { + versionsResults.append(" <tr>\n" + + " <td><font color="red">" + dirSource + + "</font></td>\n" + + " <td><font color="red">" + + voteClientVersions + "</font></td>\n" + + " </tr>\n"); + this.logger.warning(dirSource + " recommends other client " + + "versions than the consensus: " + voteClientVersions); + } else { + versionsResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteClientVersions + "</td>\n" + + " </tr>\n"); + this.logger.fine(dirSource + " recommends the same client " + + "versions as the consensus: " + voteClientVersions); + } + if (voteServerVersions == null) { + /* Not a versioning authority. */ + } else if (!voteServerVersions.equals(consensusServerVersions)) { + versionsResults.append(" <tr>\n" + + " <td></td>\n" + + " <td><font color="red">" + + voteServerVersions + "</font></td>\n" + + " </tr>\n"); + this.logger.warning(dirSource + " recommends other server " + + "versions than the consensus: " + voteServerVersions); + } else { + versionsResults.append(" <tr>\n" + + " <td></td>\n" + + " <td>" + voteServerVersions + "</td>\n" + + " </tr>\n"); + this.logger.fine(dirSource + " recommends the same server " + + "versions as the consensus: " + voteServerVersions); + } + + /* Write consensus parameters. */ + boolean conflictOrInvalid = false; + Set<String> validParameters = new HashSet<String>(Arrays.asList( + ("circwindow,CircuitPriorityHalflifeMsec,refuseunknownexits," + + "cbtdisabled,cbtnummodes,cbtrecentcount,cbtmaxtimeouts," + + "cbtmincircs,cbtquantile,cbtclosequantile,cbttestfreq," + + "cbtmintimeout,cbtinitialtimeout").split(","))); + if (voteParams == null) { + /* Authority doesn't set consensus parameters. */ + } else { + for (String param : voteParams.split(" ")) { + if (!param.equals("params") && + (!consensusParams.contains(param) || + !validParameters.contains(param.split("=")[0]))) { + conflictOrInvalid = true; + break; + } + } + } + if (conflictOrInvalid) { + paramsResults.append(" <tr>\n" + + " <td><font color="red">" + dirSource + + "</font></td>\n" + + " <td><font color="red">" + + voteParams + "</font></td>\n" + + " </tr>\n"); + this.logger.warning(dirSource + " sets conflicting or invalid " + + "consensus parameters: " + voteParams); + } else { + paramsResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteParams + "</td>\n" + + " </tr>\n"); + this.logger.fine(dirSource + " sets only non-conflicting and " + + "valid consensus parameters: " + voteParams); + } + + /* Write authority key expiration date. */ + if (voteDirKeyExpires != null) { + boolean expiresIn14Days = false; + try { + expiresIn14Days = (System.currentTimeMillis() + + 14L * 24L * 60L * 60L * 1000L > + dateTimeFormat.parse(voteDirKeyExpires.substring( + "dir-key-expires ".length())).getTime()); + } catch (ParseException e) { + /* Can't parse the timestamp? Whatever. */ + } + if (expiresIn14Days) { + authorityKeysResults.append(" <tr>\n" + + " <td><font color="red">" + dirSource + + "</font></td>\n" + + " <td><font color="red">" + + voteDirKeyExpires + "</font></td>\n" + + " </tr>\n"); + this.logger.warning(dirSource + "'s certificate expires in the " + + "next 14 days: " + voteDirKeyExpires); + } else { + authorityKeysResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteDirKeyExpires + "</td>\n" + + " </tr>\n"); + this.logger.fine(dirSource + "'s certificate does not " + + "expire in the next 14 days: " + voteDirKeyExpires); + } + } + + /* Write results for bandwidth scanner status. */ + if (voteContainsBandwidthWeights > 0) { + bandwidthScannersResults.append(" <tr>\n" + + " <td>" + dirSource + "</td>\n" + + " <td>" + voteContainsBandwidthWeights + + " Measured values in w lines</td>\n" + + " </tr>\n"); + } + } + + /* Check if we're missing a vote. TODO make this configurable */ + SortedSet<String> knownAuthorities = new TreeSet<String>( + Arrays.asList(("dannenberg,dizum,gabelmoo,ides,maatuska,moria1," + + "tor26,urras").split(","))); + for (String dir : allKnownVotes) { + knownAuthorities.remove(dir); + } + if (!knownAuthorities.isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String dir : knownAuthorities) { + sb.append(", " + dir); + } + this.logger.warning("We're missing votes from the following " + + "directory authorities: " + sb.toString().substring(2)); + } + + try { + + /* Keep the past two consensus health statuses. */ + File file0 = new File("website/consensus-health.html"); + File file1 = new File("website/consensus-health-1.html"); + File file2 = new File("website/consensus-health-2.html"); + if (file2.exists()) { + file2.delete(); + } + if (file1.exists()) { + file1.renameTo(file2); + } + if (file0.exists()) { + file0.renameTo(file1); + } + + /* Start writing web page. */ + BufferedWriter bw = new BufferedWriter( + new FileWriter("website/consensus-health.html")); + bw.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 " + + "Transitional//EN\">\n" + + "<html>\n" + + " <head>\n" + + " <title>Tor Metrics Portal: Consensus health</title>\n" + + " <meta http-equiv="content-type" content="text/html; " + + "charset=ISO-8859-1">\n" + + " <link href="/css/stylesheet-ltr.css" type="text/css" " + + "rel="stylesheet">\n" + + " <link href="/images/favicon.ico" " + + "type="image/x-icon" rel="shortcut icon">\n" + + " </head>\n" + + " <body>\n" + + " <div class="center">\n" + + " <table class="banner" border="0" " + + "cellpadding="0" cellspacing="0" summary="">\n" + + " <tr>\n" + + " <td class="banner-left"><a " + + "href="/index.html"><img src="/images/top-left.png" " + + "alt="Click to go to home page" width="193" " + + "height="79"></a></td>\n" + + " <td class="banner-middle">\n" + + " <a href="/">Home</a>\n" + + " <a href="graphs.html">Graphs</a>\n" + + " <a href="research.html">Research</a>\n" + + " <a href="status.html">Status</a>\n" + + " <br>\n" + + " <font size="2">\n" + + " <a href="exonerator.html">ExoneraTor</a>\n" + + " <a href="relay-search.html">Relay Search</a>\n" + + " <a class="current">Consensus Health</a>\n" + + " </font>\n" + + " </td>\n" + + " <td class="banner-right"></td>\n" + + " </tr>\n" + + " </table>\n" + + " <div class="main-column">\n" + + " <h2>Tor Metrics Portal: Consensus Health</h2>\n" + + " <br>\n" + + " <p>This page shows statistics about the current " + + "consensus and votes to facilitate debugging of the " + + "directory consensus process.</p>\n"); + + /* Write valid-after time. */ + bw.write(" <br>\n" + + " <h3>Valid-after time</h3>\n" + + " <br>\n" + + " <p>Consensus was published "); + boolean consensusIsStale = false; + try { + consensusIsStale = System.currentTimeMillis() + - 3L * 60L * 60L * 1000L > + dateTimeFormat.parse(this.mostRecentValidAfterTime).getTime(); + } catch (ParseException e) { + /* Can't parse the timestamp? Whatever. */ + } + if (consensusIsStale) { + bw.write("<font color="red">" + this.mostRecentValidAfterTime + + "</font>"); + this.logger.warning("The last consensus published at " + + this.mostRecentValidAfterTime + " is more than 3 hours " + + "old."); + } else { + bw.write(this.mostRecentValidAfterTime); + this.logger.fine("The last consensus published at " + + this.mostRecentValidAfterTime + " is less than 3 hours " + + "old."); + } + bw.write(". <i>Note that it takes " + + "15 to 30 minutes for the metrics portal to learn about " + + "new consensus and votes and process them.</i></p>\n"); + + /* Write known flags. */ + bw.write(" <br>\n" + + " <h3>Known flags</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (knownFlagsResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + bw.write(knownFlagsResults.toString()); + } + bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">" + + consensusKnownFlags + "</font></td>\n" + + " </tr>\n"); + bw.write(" </table>\n"); + + /* Write number of relays voted about. */ + bw.write(" <br>\n" + + " <h3>Number of relays voted about</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="320">\n" + + " <col width="320">\n" + + " </colgroup>\n"); + if (numRelaysVotesResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td><td></td></tr>\n"); + } else { + bw.write(numRelaysVotesResults.toString()); + } + bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">" + + consensusTotalRelays + " total</font></td>\n" + + " <td><font color="blue">" + + consensusRunningRelays + " Running</font></td>\n" + + " </tr>\n"); + bw.write(" </table>\n"); + + /* Write consensus methods. */ + bw.write(" <br>\n" + + " <h3>Consensus methods</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (consensusMethodsResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + bw.write(consensusMethodsResults.toString()); + } + bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">" + + consensusConsensusMethod + "</font></td>\n" + + " </tr>\n"); + bw.write(" </table>\n"); + + /* Write recommended versions. */ + bw.write(" <br>\n" + + " <h3>Recommended versions</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (versionsResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + bw.write(versionsResults.toString()); + } + bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">" + + consensusClientVersions + "</font></td>\n" + + " </tr>\n"); + bw.write(" <tr>\n" + + " <td></td>\n" + + " <td><font color="blue">" + + consensusServerVersions + "</font></td>\n" + + " </tr>\n"); + bw.write(" </table>\n"); + + /* Write consensus parameters. */ + bw.write(" <br>\n" + + " <h3>Consensus parameters</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (paramsResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + bw.write(paramsResults.toString()); + } + bw.write(" <tr>\n" + + " <td><font color="blue">consensus</font>" + + "</td>\n" + + " <td><font color="blue">" + + consensusParams + "</font></td>\n" + + " </tr>\n"); + bw.write(" </table>\n"); + + /* Write authority keys. */ + bw.write(" <br>\n" + + " <h3>Authority keys</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (authorityKeysResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + bw.write(authorityKeysResults.toString()); + } + bw.write(" </table>\n" + + " <br>\n" + + " <p><i>Note that expiration dates of legacy keys are " + + "not included in votes and therefore not listed here!</i>" + + "</p>\n"); + + /* Write bandwidth scanner status. */ + bw.write(" <br>\n" + + " <h3>Bandwidth scanner status</h3>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + if (bandwidthScannersResults.length() < 1) { + bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); + } else { + bw.write(bandwidthScannersResults.toString()); + } + bw.write(" </table>\n"); + + /* Write authority versions. */ + bw.write(" <br>\n" + + " <h3>Authority versions</h3>\n" + + " <br>\n"); + if (authorityVersionsResults.length() < 1) { + bw.write(" <p>(No relays with Authority flag found.)" + + "</p>\n"); + } else { + bw.write(" <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="640">\n" + + " </colgroup>\n"); + bw.write(authorityVersionsResults.toString()); + bw.write(" </table>\n" + + " <br>\n" + + " <p><i>Note that this list of relays with the " + + "Authority flag may be different from the list of v3 " + + "directory authorities!</i></p>\n"); + } + + /* Write (huge) table with all flags. */ + bw.write(" <br>\n" + + " <h3>Relay flags</h3>\n" + + " <br>\n" + + " <p>The semantics of flags written in the table is " + + "as follows:</p>\n" + + " <ul>\n" + + " <li><b>In vote and consensus:</b> Flag in vote " + + "matches flag in consensus, or relay is not listed in " + + "consensus (because it doesn't have the Running " + + "flag)</li>\n" + + " <li><b><font color="red">Only in " + + "vote:</font></b> Flag in vote, but missing in the " + + "consensus, because there was no majority for the flag or " + + "the flag was invalidated (e.g., Named gets invalidated by " + + "Unnamed)</li>\n" + + " <li><b><font color="gray"><s>Only in " + + "consensus:</s></font></b> Flag in consensus, but missing " + + "in a vote of a directory authority voting on this " + + "flag</li>\n" + + " <li><b><font color="blue">In " + + "consensus:</font></b> Flag in consensus</li>\n" + + " </ul>\n" + + " <br>\n" + + " <p>See also the summary below the table.</p>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="120">\n" + + " <col width="80">\n"); + for (int i = 0; i < allKnownVotes.size(); i++) { + bw.write(" <col width="" + + (640 / allKnownVotes.size()) + "">\n"); + } + bw.write(" </colgroup>\n"); + int linesWritten = 0; + for (Map.Entry<String, SortedSet<String>> e : + votesAssignedFlags.entrySet()) { + if (linesWritten++ % 10 == 0) { + bw.write(" <tr><td><br><b>Fingerprint</b></td>" + + "<td><br><b>Nickname</b></td>\n"); + for (String dir : allKnownVotes) { + String shortDirName = dir.length() > 6 ? + dir.substring(0, 5) + "." : dir; + bw.write("<td><br><b>" + shortDirName + "</b></td>"); + } + bw.write("<td><br><b>consensus</b></td></tr>\n"); + } + String relayKey = e.getKey(); + SortedSet<String> votes = e.getValue(); + String fingerprint = relayKey.split(" ")[0].substring(0, 8); + String nickname = relayKey.split(" ")[1]; + bw.write(" <tr>\n"); + if (consensusAssignedFlags.containsKey(relayKey) && + consensusAssignedFlags.get(relayKey).contains(" Named") && + !Character.isDigit(nickname.charAt(0))) { + bw.write(" <td id="" + nickname + + ""><a href="relay.html?fingerprint=" + + relayKey.split(" ")[0] + "" target="_blank">" + + fingerprint + "</a></td>\n"); + } else { + bw.write(" <td><a href="relay.html?fingerprint=" + + fingerprint + "" target="_blank">" + fingerprint + + "</a></td>\n"); + } + bw.write(" <td>" + nickname + "</td>\n"); + SortedSet<String> relevantFlags = new TreeSet<String>(); + for (String vote : votes) { + String[] parts = vote.split(" "); + for (int j = 2; j < parts.length; j++) { + relevantFlags.add(parts[j]); + } + } + String consensusFlags = null; + if (consensusAssignedFlags.containsKey(relayKey)) { + consensusFlags = consensusAssignedFlags.get(relayKey); + String[] parts = consensusFlags.split(" "); + for (int j = 1; j < parts.length; j++) { + relevantFlags.add(parts[j]); + } + } + for (String dir : allKnownVotes) { + String flags = null; + for (String vote : votes) { + if (vote.startsWith(dir)) { + flags = vote; + break; + } + } + if (flags != null) { + votes.remove(flags); + bw.write(" <td>"); + int flagsWritten = 0; + for (String flag : relevantFlags) { + bw.write(flagsWritten++ > 0 ? "<br>" : ""); + SortedMap<String, SortedMap<String, Integer>> sums = null; + if (flags.contains(" " + flag)) { + if (consensusFlags == null || + consensusFlags.contains(" " + flag)) { + bw.write(flag); + sums = flagsAgree; + } else { + bw.write("<font color="red">" + flag + "</font>"); + sums = flagsLost; + } + } else if (consensusFlags != null && + votesKnownFlags.get(dir).contains(" " + flag) && + consensusFlags.contains(" " + flag)) { + bw.write("<font color="gray"><s>" + flag + + "</s></font>"); + sums = flagsMissing; + } + if (sums != null) { + SortedMap<String, Integer> sum = null; + if (sums.containsKey(dir)) { + sum = sums.get(dir); + } else { + sum = new TreeMap<String, Integer>(); + sums.put(dir, sum); + } + sum.put(flag, sum.containsKey(flag) ? + sum.get(flag) + 1 : 1); + } + } + bw.write("</td>\n"); + } else { + bw.write(" <td></td>\n"); + } + } + if (consensusFlags != null) { + bw.write(" <td>"); + int flagsWritten = 0; + for (String flag : relevantFlags) { + bw.write(flagsWritten++ > 0 ? "<br>" : ""); + if (consensusFlags.contains(" " + flag)) { + bw.write("<font color="blue">" + flag + "</font>"); + } + } + bw.write("</td>\n"); + } else { + bw.write(" <td></td>\n"); + } + bw.write(" </tr>\n"); + } + bw.write(" </table>\n"); + + /* Write summary of overlap between votes and consensus. */ + bw.write(" <br>\n" + + " <h3>Overlap between votes and consensus</h3>\n" + + " <br>\n" + + " <p>The semantics of columns is similar to the " + + "table above:</p>\n" + + " <ul>\n" + + " <li><b>In vote and consensus:</b> Flag in vote " + + "matches flag in consensus, or relay is not listed in " + + "consensus (because it doesn't have the Running " + + "flag)</li>\n" + + " <li><b><font color="red">Only in " + + "vote:</font></b> Flag in vote, but missing in the " + + "consensus, because there was no majority for the flag or " + + "the flag was invalidated (e.g., Named gets invalidated by " + + "Unnamed)</li>\n" + + " <li><b><font color="gray"><s>Only in " + + "consensus:</s></font></b> Flag in consensus, but missing " + + "in a vote of a directory authority voting on this " + + "flag</li>\n" + + " </ul>\n" + + " <br>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="210">\n" + + " <col width="210">\n" + + " <col width="210">\n" + + " </colgroup>\n"); + bw.write(" <tr><td></td><td><b>Only in vote</b></td>" + + "<td><b>In vote and consensus</b></td>" + + "<td><b>Only in consensus</b></td>\n"); + for (String dir : allKnownVotes) { + boolean firstFlagWritten = false; + String[] flags = votesKnownFlags.get(dir).substring( + "known-flags ".length()).split(" "); + for (String flag : flags) { + bw.write(" <tr>\n"); + if (firstFlagWritten) { + bw.write(" <td></td>\n"); + } else { + bw.write(" <td>" + dir + "</td>\n"); + firstFlagWritten = true; + } + if (flagsLost.containsKey(dir) && + flagsLost.get(dir).containsKey(flag)) { + bw.write(" <td><font color="red"> " + + flagsLost.get(dir).get(flag) + " " + flag + + "</font></td>\n"); + } else { + bw.write(" <td></td>\n"); + } + if (flagsAgree.containsKey(dir) && + flagsAgree.get(dir).containsKey(flag)) { + bw.write(" <td>" + flagsAgree.get(dir).get(flag) + + " " + flag + "</td>\n"); + } else { + bw.write(" <td></td>\n"); + } + if (flagsMissing.containsKey(dir) && + flagsMissing.get(dir).containsKey(flag)) { + bw.write(" <td><font color="gray"><s>" + + flagsMissing.get(dir).get(flag) + " " + flag + + "</s></font></td>\n"); + } else { + bw.write(" <td></td>\n"); + } + bw.write(" </tr>\n"); + } + } + bw.write(" </table>\n"); + + /* Finish writing. */ + bw.write(" </div>\n" + + " </div>\n" + + " <div class="bottom" id="bottom">\n" + + " <p>This material is supported in part by the " + + "National Science Foundation under Grant No. " + + "CNS-0959138. Any opinions, finding, and conclusions " + + "or recommendations expressed in this material are " + + "those of the author(s) and do not necessarily reflect " + + "the views of the National Science Foundation.</p>\n" + + " <p>"Tor" and the "Onion Logo" are <a " + + "href="https://www.torproject.org/docs/trademark-faq.html" + + ".en">" + + "registered trademarks</a> of The Tor Project, " + + "Inc.</p>\n" + + " <p>Data on this site is freely available under a " + + "<a href="http://creativecommons.org/publicdomain/" + + "zero/1.0/">CC0 no copyright declaration</a>: To the " + + "extent possible under law, the Tor Project has waived " + + "all copyright and related or neighboring rights in " + + "the data. Graphs are licensed under a <a " + + "href="http://creativecommons.org/licenses/by/3.0/" + + "us/">Creative Commons Attribution 3.0 United States " + + "License</a>.</p>\n" + + " </div>\n" + + " </body>\n" + + "</html>"); + bw.close(); + + } catch (IOException e) { + } + } +} + diff --git a/src/org/torproject/ernie/cron/LockFile.java b/src/org/torproject/ernie/cron/LockFile.java new file mode 100644 index 0000000..7a097f0 --- /dev/null +++ b/src/org/torproject/ernie/cron/LockFile.java @@ -0,0 +1,47 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.util.logging.*; + +public class LockFile { + + private File lockFile; + private Logger logger; + + public LockFile() { + this.lockFile = new File("lock"); + this.logger = Logger.getLogger(LockFile.class.getName()); + } + + public boolean acquireLock() { + this.logger.fine("Trying to acquire lock..."); + try { + if (this.lockFile.exists()) { + BufferedReader br = new BufferedReader(new FileReader("lock")); + long runStarted = Long.parseLong(br.readLine()); + br.close(); + if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) { + return false; + } + } + BufferedWriter bw = new BufferedWriter(new FileWriter("lock")); + bw.append("" + System.currentTimeMillis() + "\n"); + bw.close(); + this.logger.fine("Acquired lock."); + return true; + } catch (IOException e) { + this.logger.warning("Caught exception while trying to acquire " + + "lock!"); + return false; + } + } + + public void releaseLock() { + this.logger.fine("Releasing lock..."); + this.lockFile.delete(); + this.logger.fine("Released lock."); + } +} + diff --git a/src/org/torproject/ernie/cron/LoggingConfiguration.java b/src/org/torproject/ernie/cron/LoggingConfiguration.java new file mode 100644 index 0000000..f14fc54 --- /dev/null +++ b/src/org/torproject/ernie/cron/LoggingConfiguration.java @@ -0,0 +1,88 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.text.*; +import java.util.Date; +import java.util.TimeZone; +import java.util.logging.*; + +/** + * Initialize logging configuration. + * + * Log levels used by ERNIE: + * + * - SEVERE: An event made it impossible to continue program execution. + * - WARNING: A potential problem occurred that requires the operator to + * look after the otherwise unattended setup + * - INFO: Messages on INFO level are meant to help the operator in making + * sure that operation works as expected. + * - FINE: Debug messages that are used to identify problems and which are + * turned on by default. + * - FINER: More detailed debug messages to investigate problems in more + * detail. Not turned on by default. Increase log file limit when using + * FINER. + * - FINEST: Most detailed debug messages. Not used. + */ +public class LoggingConfiguration { + + public LoggingConfiguration() { + + /* Remove default console handler. */ + for (Handler h : Logger.getLogger("").getHandlers()) { + Logger.getLogger("").removeHandler(h); + } + + /* Disable logging of internal Sun classes. */ + Logger.getLogger("sun").setLevel(Level.OFF); + + /* Set minimum log level we care about from INFO to FINER. */ + Logger.getLogger("").setLevel(Level.FINER); + + /* Create log handler that writes messages on WARNING or higher to the + * console. */ + final SimpleDateFormat dateTimeFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Formatter cf = new Formatter() { + public String format(LogRecord record) { + return dateTimeFormat.format(new Date(record.getMillis())) + " " + + record.getMessage() + "\n"; + } + }; + Handler ch = new ConsoleHandler(); + ch.setFormatter(cf); + ch.setLevel(Level.WARNING); + Logger.getLogger("").addHandler(ch); + + /* Initialize own logger for this class. */ + Logger logger = Logger.getLogger( + LoggingConfiguration.class.getName()); + + /* Create log handler that writes all messages on FINE or higher to a + * local file. */ + Formatter ff = new Formatter() { + public String format(LogRecord record) { + return dateTimeFormat.format(new Date(record.getMillis())) + " " + + record.getLevel() + " " + record.getSourceClassName() + " " + + record.getSourceMethodName() + " " + record.getMessage() + + (record.getThrown() != null ? " " + record.getThrown() : "") + + "\n"; + } + }; + try { + FileHandler fh = new FileHandler("log", 5000000, 5, true); + fh.setFormatter(ff); + fh.setLevel(Level.FINE); + Logger.getLogger("").addHandler(fh); + } catch (SecurityException e) { + logger.log(Level.WARNING, "No permission to create log file. " + + "Logging to file is disabled.", e); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not write to log file. Logging to " + + "file is disabled.", e); + } + } +} + diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java new file mode 100644 index 0000000..b95a133 --- /dev/null +++ b/src/org/torproject/ernie/cron/Main.java @@ -0,0 +1,67 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.util.*; +import java.util.logging.*; + +/** + * Coordinate downloading and parsing of descriptors and extraction of + * statistically relevant data for later processing with R. + */ +public class Main { + public static void main(String[] args) { + + /* Initialize logging configuration. */ + new LoggingConfiguration(); + + Logger logger = Logger.getLogger(Main.class.getName()); + logger.info("Starting ERNIE."); + + // Initialize configuration + Configuration config = new Configuration(); + + // Use lock file to avoid overlapping runs + LockFile lf = new LockFile(); + if (!lf.acquireLock()) { + logger.severe("Warning: ERNIE is already running or has not exited " + + "cleanly! Exiting!"); + System.exit(1); + } + + // Define stats directory for temporary files + File statsDirectory = new File("stats"); + + // Prepare consensus health checker + ConsensusHealthChecker chc = config.getWriteConsensusHealth() ? + new ConsensusHealthChecker() : null; + + // Prepare relay descriptor parser (only if we are writing the + // consensus-health page to disk) + RelayDescriptorParser rdp = config.getWriteConsensusHealth() ? + new RelayDescriptorParser(chc) : null; + + // Import relay descriptors + if (rdp != null) { + if (config.getImportDirectoryArchives()) { + new ArchiveReader(rdp, + new File(config.getDirectoryArchivesDirectory()), + statsDirectory, + config.getKeepDirectoryArchiveImportHistory()); + } + } + + // Write consensus health website + if (chc != null) { + chc.writeStatusWebsite(); + chc = null; + } + + // Remove lock file + lf.releaseLock(); + + logger.info("Terminating ERNIE."); + } +} + diff --git a/src/org/torproject/ernie/cron/RelayDescriptorParser.java b/src/org/torproject/ernie/cron/RelayDescriptorParser.java new file mode 100644 index 0000000..24b512b --- /dev/null +++ b/src/org/torproject/ernie/cron/RelayDescriptorParser.java @@ -0,0 +1,91 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.text.*; +import java.util.*; +import java.util.logging.*; +import org.apache.commons.codec.digest.*; +import org.apache.commons.codec.binary.*; + +/** + * Parses relay descriptors including network status consensuses and + * votes, server and extra-info descriptors, and passes the results to the + * stats handlers, to the archive writer, or to the relay descriptor + * downloader. + */ +public class RelayDescriptorParser { + + private ConsensusHealthChecker chc; + + /** + * Logger for this class. + */ + private Logger logger; + + private SimpleDateFormat dateTimeFormat; + + /** + * Initializes this class. + */ + public RelayDescriptorParser(ConsensusHealthChecker chc) { + this.chc = chc; + + /* Initialize logger. */ + this.logger = Logger.getLogger(RelayDescriptorParser.class.getName()); + + this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + public void parse(byte[] data) { + try { + /* Convert descriptor to ASCII for parsing. This means we'll lose + * the non-ASCII chars, but we don't care about them for parsing + * anyway. */ + BufferedReader br = new BufferedReader(new StringReader(new String( + data, "US-ASCII"))); + String line = br.readLine(); + if (line == null) { + this.logger.fine("We were given an empty descriptor for " + + "parsing. Ignoring."); + return; + } + SimpleDateFormat parseFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + if (line.equals("network-status-version 3")) { + // TODO when parsing the current consensus, check the fresh-until + // time to see when we switch from hourly to half-hourly + // consensuses + boolean isConsensus = true; + String validAfterTime = null; + String dirSource = null; + while ((line = br.readLine()) != null) { + if (line.equals("vote-status vote")) { + isConsensus = false; + } else if (line.startsWith("valid-after ")) { + validAfterTime = line.substring("valid-after ".length()); + } else if (line.startsWith("dir-source ")) { + dirSource = line.split(" ")[2]; + break; + } + } + if (isConsensus) { + if (this.chc != null) { + this.chc.processConsensus(validAfterTime, data); + } + } else { + if (this.chc != null) { + this.chc.processVote(validAfterTime, dirSource, data); + } + } + } + } catch (IOException e) { + this.logger.log(Level.WARNING, "Could not parse descriptor. " + + "Skipping.", e); + } + } +} +