[or-cvs] [ernie/master 3/3] Add useful logs about the last execution to website.
karsten at torproject.org
karsten at torproject.org
Wed Apr 14 10:24:50 UTC 2010
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed, 14 Apr 2010 12:22:53 +0200
Subject: Add useful logs about the last execution to website.
Commit: ac61e609853e63669c84fe31bff9c667a5cf455d
---
src/ArchiveReader.java | 6 ++
src/ArchiveWriter.java | 123 +++++++++++++++++++---------------
src/CachedRelayDescriptorReader.java | 11 +++
src/Main.java | 9 ++-
src/RelayDescriptorDownloader.java | 79 ++++++++++++++++++++++
5 files changed, 169 insertions(+), 59 deletions(-)
diff --git a/src/ArchiveReader.java b/src/ArchiveReader.java
index 92a07aa..ef68cc1 100644
--- a/src/ArchiveReader.java
+++ b/src/ArchiveReader.java
@@ -10,6 +10,7 @@ import org.apache.commons.compress.compressors.bzip2.*;
public class ArchiveReader {
public ArchiveReader(RelayDescriptorParser rdp, String archivesDir,
boolean keepImportHistory) {
+ int parsedFiles = 0, ignoredFiles = 0;
Logger logger = Logger.getLogger(ArchiveReader.class.getName());
SortedSet<String> archivesImportHistory = new TreeSet<String>();
File archivesImportHistoryFile =
@@ -46,6 +47,7 @@ public class ArchiveReader {
BufferedInputStream bis = null;
if (keepImportHistory &&
archivesImportHistory.contains(pop.getName())) {
+ ignoredFiles++;
continue;
} else if (pop.getName().endsWith(".tar.bz2")) {
logger.warning("Cannot parse compressed tarball "
@@ -72,6 +74,7 @@ public class ArchiveReader {
bis.close();
byte[] allData = baos.toByteArray();
rdp.parse(allData);
+ parsedFiles++;
} catch (IOException e) {
problems.add(pop);
if (problems.size() > 3) {
@@ -111,6 +114,9 @@ public class ArchiveReader {
+ "history file.");
}
}
+ logger.info("Finished importing relay descriptors from local "
+ + "directory:\nParsed " + parsedFiles + " and ignored "
+ + ignoredFiles + " files.");
}
}
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
index de72da1..051431f 100644
--- a/src/ArchiveWriter.java
+++ b/src/ArchiveWriter.java
@@ -8,12 +8,15 @@ import org.apache.commons.codec.binary.*;
public class ArchiveWriter {
private Logger logger;
private String outputDirectory;
+ private int storedConsensuses = 0, storedVotes = 0,
+ storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
+
public ArchiveWriter(String outputDirectory) {
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
this.outputDirectory = outputDirectory;
}
- private void store(byte[] data, String filename) {
+ private boolean store(byte[] data, String filename) {
try {
File file = new File(filename);
if (!file.exists()) {
@@ -23,11 +26,13 @@ public class ArchiveWriter {
new FileOutputStream(file));
bos.write(data, 0, data.length);
bos.close();
+ return true;
}
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not store relay descriptor "
+ filename, e);
}
+ return false;
}
public void storeConsensus(byte[] data, long validAfter) {
@@ -36,7 +41,9 @@ public class ArchiveWriter {
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String filename = outputDirectory + "/consensus/"
+ printFormat.format(new Date(validAfter)) + "-consensus";
- this.store(data, filename);
+ if (this.store(data, filename)) {
+ this.storedConsensuses++;
+ }
}
public void storeVote(byte[] data, long validAfter,
@@ -47,7 +54,9 @@ public class ArchiveWriter {
String filename = outputDirectory + "/vote/"
+ printFormat.format(new Date(validAfter)) + "-vote-"
+ fingerprint + "-" + digest;
- this.store(data, filename);
+ if (this.store(data, filename)) {
+ this.storedVotes++;
+ }
}
public void storeServerDescriptor(byte[] data, String digest,
@@ -58,7 +67,9 @@ public class ArchiveWriter {
+ printFormat.format(new Date(published))
+ digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ digest;
- this.store(data, filename);
+ if (this.store(data, filename)) {
+ this.storedServerDescriptors++;
+ }
}
public void storeExtraInfoDescriptor(byte[] data,
@@ -70,27 +81,9 @@ public class ArchiveWriter {
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
+ extraInfoDigest;
- this.store(data, filename);
- }
-
- private SortedSet<String> getFileNames(File dir) {
- SortedSet<String> files = new TreeSet<String>();
- Stack<File> leftToParse = new Stack<File>();
- leftToParse.add(dir);
- while (!leftToParse.isEmpty()) {
- File pop = leftToParse.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- leftToParse.add(f);
- }
- } else if (pop.length() > 0) {
- String absPath = pop.getAbsolutePath().replaceAll(":", "-");
- String relPath = absPath.substring(absPath.indexOf(
- outputDirectory + "/"));
- files.add(relPath);
- }
+ if (this.store(data, filename)) {
+ this.storedExtraInfoDescriptors++;
}
- return files;
}
/**
@@ -98,15 +91,16 @@ public class ArchiveWriter {
* on level INFO.
*/
public void dumpStats() {
+ StringBuilder sb = new StringBuilder("Finished writing relay "
+ + "descriptors to disk:\nIn this execution, we stored "
+ + this.storedConsensuses + " consensuses, " + this.storedVotes
+ + " votes, " + this.storedServerDescriptors
+ + " server descriptors, and " + this.storedExtraInfoDescriptors
+ + " extra-info descriptors to disk.\n");
+ sb.append("Statistics on the completeness of written relay "
+ + "descriptors of the past 12 consensuses (Consensus/Vote, "
+ + "valid-after, votes, server descriptors, extra-infos):");
try {
- SortedSet<String> votes = getFileNames(
- new File(outputDirectory + "/vote"));
- SortedSet<String> serverDescs = getFileNames(
- new File(outputDirectory + "/server-descriptor"));
- SortedSet<String> extraInfos = getFileNames(
- new File(outputDirectory + "/extra-info"));
- SortedSet<String> consensuses = getFileNames(
- new File(outputDirectory + "/consensus"));
SimpleDateFormat validAfterFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -116,17 +110,37 @@ public class ArchiveWriter {
SimpleDateFormat descriptorFormat =
new SimpleDateFormat("yyyy/MM/");
descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- StringBuilder sb = new StringBuilder();
- sb.append(" valid-after votes "
- + "server descriptors extra-infos\n");
- SortedSet<String> lastConsensuses = new TreeSet<String>();
- for (int i = 0; !consensuses.isEmpty() && i < 12; i++) {
- String last = consensuses.last();
- lastConsensuses.add(last);
- consensuses.remove(last);
+
+ SortedSet<File> consensuses = new TreeSet<File>();
+ Stack<File> leftToParse = new Stack<File>();
+ leftToParse.add(new File(outputDirectory + "/consensus"));
+ while (!leftToParse.isEmpty()) {
+ File pop = leftToParse.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ leftToParse.add(f);
+ }
+ } else if (pop.length() > 0) {
+ consensuses.add(pop);
+ }
+ while (consensuses.size() > 12) {
+ consensuses.remove(consensuses.first());
+ }
+ }
+ leftToParse.add(new File(outputDirectory + "/vote"));
+ SortedSet<File> votes = new TreeSet<File>();
+ while (!leftToParse.isEmpty()) {
+ File pop = leftToParse.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ leftToParse.add(f);
+ }
+ } else if (pop.length() > 0) {
+ votes.add(pop);
+ }
}
- for (String f : lastConsensuses) {
- BufferedReader br = new BufferedReader(new FileReader(new File(f)));
+ for (File f : consensuses) {
+ BufferedReader br = new BufferedReader(new FileReader(f));
String line = null, validAfterTime = null, votePrefix = null;
int allVotes = 0, foundVotes = 0,
allServerDescs = 0, foundServerDescs = 0,
@@ -143,9 +157,9 @@ public class ArchiveWriter {
allVotes++;
String pattern = votePrefix + line.split(" ")[2];
String votefilename = null;
- for (String v : votes) {
- if (v.startsWith(pattern)) {
- votefilename = v;
+ for (File v : votes) {
+ if (v.getName().startsWith(pattern)) {
+ votefilename = v.getName();
break;
}
}
@@ -169,7 +183,7 @@ public class ArchiveWriter {
+ descriptorFormat.format(new Date(published))
+ digest.substring(0, 1) + "/"
+ digest.substring(1, 2) + "/" + digest;
- if (serverDescs.contains(filename)) {
+ if (new File(filename).exists()) {
BufferedReader sbr = new BufferedReader(new FileReader(
new File(filename)));
String line2 = null;
@@ -186,7 +200,7 @@ public class ArchiveWriter {
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
+ extraInfoDigest;
- if (extraInfos.contains(filename2)) {
+ if (new File(filename2).exists()) {
voteFoundExtraInfos++;
}
}
@@ -197,8 +211,8 @@ public class ArchiveWriter {
}
}
vbr.close();
- sb.append(String.format("V %s "
- + " %d/%d (%5.1f%%) %d/%d (%5.1f%%)%n",
+ sb.append(String.format("%nV %s "
+ + " %d/%d (%5.1f%%) %d/%d (%5.1f%%)",
validAfterTime,
voteFoundServerDescs, voteAllServerDescs,
100.0D * (double) voteFoundServerDescs /
@@ -217,7 +231,7 @@ public class ArchiveWriter {
+ descriptorFormat.format(new Date(published))
+ digest.substring(0, 1) + "/"
+ digest.substring(1, 2) + "/" + digest;
- if (serverDescs.contains(filename)) {
+ if (new File (filename).exists()) {
BufferedReader sbr = new BufferedReader(new FileReader(
new File(filename)));
String line2 = null;
@@ -233,7 +247,7 @@ public class ArchiveWriter {
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
+ extraInfoDigest;
- if (extraInfos.contains(filename2)) {
+ if (new File (filename2).exists()) {
foundExtraInfos++;
}
}
@@ -243,8 +257,8 @@ public class ArchiveWriter {
}
}
}
- sb.append(String.format("C %s %d/%d (%5.1f%%) %d/%d (%5.1f%%) "
- + "%d/%d (%5.1f%%)%n",
+ sb.append(String.format("%nC %s %d/%d (%5.1f%%) %d/%d (%5.1f%%) "
+ + "%d/%d (%5.1f%%)",
validAfterTime, foundVotes, allVotes,
100.0D * (double) foundVotes / (double) allVotes,
foundServerDescs, allServerDescs,
@@ -252,8 +266,7 @@ public class ArchiveWriter {
foundExtraInfos, allExtraInfos,
100.0D * (double) foundExtraInfos / (double) allExtraInfos));
}
- this.logger.info("Statistics on relay descriptors from the last 12 "
- + "known consensuses:\n" + sb.toString());
+ this.logger.info(sb.toString());
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
e);
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
index 64a1966..cb5ca28 100644
--- a/src/CachedRelayDescriptorReader.java
+++ b/src/CachedRelayDescriptorReader.java
@@ -10,6 +10,8 @@ import java.util.logging.*;
public class CachedRelayDescriptorReader {
public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
List<String> inputDirectories) {
+ StringBuilder dumpStats = new StringBuilder("Finished importing "
+ + "relay descriptors from local Tor data directories:");
Logger logger = Logger.getLogger(
CachedRelayDescriptorReader.class.getName());
for (String inputDirectory : inputDirectories) {
@@ -42,6 +44,8 @@ public class CachedRelayDescriptorReader {
String line = null;
while ((line = br.readLine()) != null) {
if (line.startsWith("valid-after ")) {
+ dumpStats.append("\n" + f.getName() + ": " + line.substring(
+ "valid-after ".length()));
SimpleDateFormat dateTimeFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -52,6 +56,7 @@ public class CachedRelayDescriptorReader {
+ cachedDescDir.getAbsolutePath() + " are stale. "
+ "The valid-after line in cached-consensus is '"
+ line + "'.");
+ dumpStats.append(" (stale!)");
}
break;
}
@@ -72,6 +77,7 @@ public class CachedRelayDescriptorReader {
"router " : "extra-info ";
String sigToken = "\nrouter-signature\n";
String endToken = "\n-----END SIGNATURE-----\n";
+ int parsedNum = 0;
while (end < ascii.length()) {
start = ascii.indexOf(startToken, end);
if (start < 0) {
@@ -91,8 +97,12 @@ public class CachedRelayDescriptorReader {
System.arraycopy(allData, start, descBytes, 0, end - start);
if (rdp != null) {
rdp.parse(descBytes);
+ parsedNum++;
}
}
+ dumpStats.append("\n" + f.getName() + ": " + parsedNum
+ + (f.getName().startsWith("cached-descriptors") ?
+ "server" : "extra-info") + " descriptors");
logger.fine("Finished reading "
+ cachedDescDir.getAbsolutePath() + " directory.");
}
@@ -105,6 +115,7 @@ public class CachedRelayDescriptorReader {
}
}
}
+ logger.info(dumpStats.toString());
}
}
diff --git a/src/Main.java b/src/Main.java
index d1e4c6a..a147cc9 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -92,10 +92,15 @@ public class Main {
if (rdd != null) {
rdd.downloadMissingDescriptors();
rdd.writeFile();
+ rdd = null;
}
}
// Write output to disk that only depends on relay descriptors
+ if (aw != null) {
+ aw.dumpStats();
+ aw = null;
+ }
if (dsfh != null) {
dsfh.writeFile();
dsfh = null;
@@ -104,10 +109,6 @@ public class Main {
sdsfh.writeFiles();
sdsfh = null;
}
- if (aw != null) {
- aw.dumpStats();
- aw = null;
- }
// Import/download GeoIP databases
GeoIPDatabaseManager gd = new GeoIPDatabaseManager(
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index d7654c5..53ce5c1 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -99,6 +99,13 @@ public class RelayDescriptorDownloader {
*/
private Logger logger;
+ private StringBuilder dumpStats;
+ private int newMissingConsensuses = 0, newMissingVotes = 0,
+ newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0,
+ triedConsensuses = 0, triedVotes = 0, triedServerDescriptors = 0,
+ triedExtraInfoDescriptors = 0, downloadedConsensuses = 0,
+ downloadedVotes = 0, downloadedServerDescriptors = 0,
+ downloadedExtraInfoDescriptors = 0;
/**
* Initializes this class, including reading in missing descriptors from
* <code>stats/missing-relay-descriptors</code>.
@@ -144,6 +151,8 @@ public class RelayDescriptorDownloader {
* directory servers. */
this.missingDescriptorsFile = new File(
"stats/missing-relay-descriptors");
+ int missingConsensuses = 0, missingVotes = 0,
+ missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
if (this.missingDescriptorsFile.exists()) {
try {
this.logger.fine("Reading file "
@@ -160,6 +169,15 @@ public class RelayDescriptorDownloader {
((line.startsWith("server,") ||
line.startsWith("extra,")) &&
this.descriptorCutOff.compareTo(published) <= 0)) {
+ if (line.startsWith("consensus,")) {
+ missingConsensuses++;
+ } else if (line.startsWith("vote,")) {
+ missingVotes++;
+ } else if (line.startsWith("server,")) {
+ missingServerDescriptors++;
+ } else if (line.startsWith("extra,")) {
+ missingExtraInfoDescriptors++;
+ }
int separateAt = line.lastIndexOf(",");
this.missingDescriptors.put(line.substring(0,
separateAt), line.substring(separateAt + 1));
@@ -179,6 +197,13 @@ public class RelayDescriptorDownloader {
+ "! This means that we might forget to dowload relay "
+ "descriptors we are missing.", e);
}
+
+ dumpStats.append("Finished downloading relay descriptors from the "
+ + "directory authorities:\nAt the beginning of this execution, "
+ + "we were missing " + missingConsensuses + " consensuses, "
+ + missingVotes + " votes, " + missingServerDescriptors
+ + " server descriptors, and " + missingExtraInfoDescriptors
+ + " extra-info descriptors.");
}
}
@@ -203,6 +228,7 @@ public class RelayDescriptorDownloader {
String voteKey = "vote," + validAfter + "," + dirSource;
if (!this.missingDescriptors.containsKey(voteKey)) {
this.missingDescriptors.put(voteKey, "NA");
+ this.newMissingVotes++;
}
}
}
@@ -222,6 +248,7 @@ public class RelayDescriptorDownloader {
if (!this.missingDescriptors.containsKey(
serverDescriptorKey)) {
this.missingDescriptors.put(serverDescriptorKey, "NA");
+ this.newMissingServerDescriptors++;
}
}
}
@@ -254,6 +281,7 @@ public class RelayDescriptorDownloader {
if (!this.missingDescriptors.containsKey(
serverDescriptorKey)) {
this.missingDescriptors.put(serverDescriptorKey, "NA");
+ this.newMissingServerDescriptors++;
}
}
}
@@ -286,6 +314,7 @@ public class RelayDescriptorDownloader {
+ relayIdentity + "," + extraInfoDigest;
if (!this.missingDescriptors.containsKey(extraInfoKey)) {
this.missingDescriptors.put(extraInfoKey, "NA");
+ this.newMissingExtraInfoDescriptors++;
}
}
}
@@ -350,10 +379,12 @@ public class RelayDescriptorDownloader {
this.downloadCurrentConsensus &&
this.currentValidAfter.equals(parts[1])) {
urls.add("/tor/status-vote/current/consensus");
+ this.triedConsensuses++;
} else if (parts[0].equals("vote") &&
this.downloadCurrentVotes &&
this.currentValidAfter.equals(parts[1])) {
urls.add("/tor/status-vote/current/" + parts[2]);
+ this.triedVotes++;
} else if (parts[0].equals("server") &&
(this.downloadAllServerDescriptors ||
(this.downloadDescriptorsForRelays != null &&
@@ -361,6 +392,7 @@ public class RelayDescriptorDownloader {
toUpperCase()))) &&
this.descriptorCutOff.compareTo(parts[1]) <= 0) {
urls.add("/tor/server/d/" + parts[3]);
+ this.triedServerDescriptors++;
} else if (parts[0].equals("extra") &&
(this.downloadAllExtraInfos ||
(this.downloadDescriptorsForRelays != null &&
@@ -368,6 +400,7 @@ public class RelayDescriptorDownloader {
toUpperCase()))) &&
this.descriptorCutOff.compareTo(parts[1]) <= 0) {
urls.add("/tor/extra/d/" + parts[3]);
+ this.triedExtraInfoDescriptors++;
}
}
}
@@ -413,6 +446,15 @@ public class RelayDescriptorDownloader {
in.close();
byte[] allData = baos.toByteArray();
rdp.parse(allData);
+ if (url.endsWith("consensus")) {
+ this.downloadedConsensuses++;
+ } else if (url.contains("status-vote")) {
+ this.downloadedVotes++;
+ } else if (url.contains("server")) {
+ this.downloadedServerDescriptors++;
+ } else if (url.contains("extra")) {
+ this.downloadedExtraInfoDescriptors++;
+ }
} else {
retryUrls.add(url);
}
@@ -440,6 +482,8 @@ public class RelayDescriptorDownloader {
}
public void writeFile() {
+ int missingConsensuses = 0, missingVotes = 0,
+ missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
try {
this.logger.fine("Writing file "
+ this.missingDescriptorsFile.getAbsolutePath() + "...");
@@ -448,6 +492,16 @@ public class RelayDescriptorDownloader {
this.missingDescriptorsFile));
for (Map.Entry<String, String> e :
this.missingDescriptors.entrySet()) {
+ String key = e.getKey();
+ if (key.startsWith("consensus,")) {
+ missingConsensuses++;
+ } else if (key.startsWith("vote,")) {
+ missingVotes++;
+ } else if (key.startsWith("server,")) {
+ missingServerDescriptors++;
+ } else if (key.startsWith("extra,")) {
+ missingExtraInfoDescriptors++;
+ }
bw.write(e.getKey() + "," + e.getValue() + "\n");
}
bw.close();
@@ -457,5 +511,30 @@ public class RelayDescriptorDownloader {
this.logger.log(Level.WARNING, "Failed writing "
+ this.missingDescriptorsFile.getAbsolutePath() + "!", e);
}
+
+ dumpStats.append("During this execution, we added "
+ + this.newMissingConsensuses + " consensuses, "
+ + this.newMissingVotes + " votes, "
+ + this.newMissingServerDescriptors + " server descriptors, and "
+ + this.newMissingExtraInfoDescriptors + " extra-info descriptors "
+ + "to the missing list.\n");
+ dumpStats.append("We attempted to download " + this.triedConsensuses
+ + " consensuses, " + this.triedVotes + " votes, "
+ + this.triedServerDescriptors + " server descriptors, and "
+ + this.triedExtraInfoDescriptors + " extra-info descriptors from "
+ + "the directory authorities.\n");
+ dumpStats.append("We successfully downloaded "
+ + this.downloadedConsensuses + " consensuses, "
+ + this.downloadedVotes + " votes, "
+ + this.downloadedServerDescriptors + " server descriptors, and "
+ + this.downloadedExtraInfoDescriptors + " extra-info descriptors "
+ + "from the directory authorities.\n");
+ dumpStats.append("At the end of this execution, "
+ + "we are missing " + missingConsensuses + " consensuses, "
+ + missingVotes + " votes, " + missingServerDescriptors
+ + " server descriptors, and " + missingExtraInfoDescriptors
+ + " extra-info descriptors, some of which we may try in the next "
+ + "execution.");
+ this.logger.info(dumpStats.toString());
}
}
--
1.6.5
More information about the tor-commits
mailing list