[tor-commits] [doctor/master] Include download statistics in the website report.
karsten at torproject.org
karsten at torproject.org
Thu Dec 8 11:56:09 UTC 2011
commit 812c1ab54f4eb99b2518d505274c5d79a4165d84
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Dec 8 12:55:38 2011 +0100
Include download statistics in the website report.
---
src/org/torproject/doctor/Download.java | 9 ++-
src/org/torproject/doctor/DownloadStatistics.java | 91 ++++++++++++++++++++
src/org/torproject/doctor/Downloader.java | 2 +-
src/org/torproject/doctor/Main.java | 7 ++
.../torproject/doctor/MetricsWebsiteReport.java | 62 +++++++++++++
src/org/torproject/doctor/Report.java | 4 +
src/org/torproject/doctor/StatusFileReport.java | 5 +
7 files changed, 178 insertions(+), 2 deletions(-)
diff --git a/src/org/torproject/doctor/Download.java b/src/org/torproject/doctor/Download.java
index 35c24f9..1d53cd5 100644
--- a/src/org/torproject/doctor/Download.java
+++ b/src/org/torproject/doctor/Download.java
@@ -22,6 +22,12 @@ public class Download {
return this.responseString;
}
+ /* Request start timestamp. */
+ private long requestStartMillis;
+ public long getRequestStartMillis() {
+ return this.requestStartMillis;
+ }
+
/* Fetch time in millis. */
private long fetchTime;
public long getFetchTime() {
@@ -29,9 +35,10 @@ public class Download {
}
public Download(String authority, String url, String responseString,
- long fetchTime) {
+ long requestStartMillis, long fetchTime) {
this.authority = authority;
this.responseString = responseString;
+ this.requestStartMillis = requestStartMillis;
this.fetchTime = fetchTime;
}
}
diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java
new file mode 100644
index 0000000..9eeffde
--- /dev/null
+++ b/src/org/torproject/doctor/DownloadStatistics.java
@@ -0,0 +1,91 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.doctor;
+
+import java.io.*;
+import java.util.*;
+
+public class DownloadStatistics {
+ public void memorizeFetchTimes(List<Download> downloadedConsensuses) {
+ try {
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.statisticsFile, true));
+ for (Download downloadedConsensus : downloadedConsensuses) {
+ String authority = downloadedConsensus.getAuthority();
+ long requestStartMillis =
+ downloadedConsensus.getRequestStartMillis();
+ long fetchTimeMillis = downloadedConsensus.getFetchTime();
+ String line = authority + ","
+ + String.valueOf(requestStartMillis) + ","
+ + String.valueOf(fetchTimeMillis);
+ bw.write(line + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ System.err.println("Could not write "
+ + this.statisticsFile.getAbsolutePath() + ". Ignoring.");
+ }
+ }
+ private SortedMap<String, List<Long>> downloadData =
+ new TreeMap<String, List<Long>>();
+ private int maxDownloadsPerAuthority = 0;
+ private File statisticsFile = new File("download-stats.csv");
+ public void prepareStatistics() {
+ if (this.statisticsFile.exists()) {
+ long cutOffMillis = System.currentTimeMillis()
+ - (7L * 24L * 60L * 60L * 1000L);
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.statisticsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ long requestStartMillis = Long.parseLong(parts[1]);
+ if (requestStartMillis < cutOffMillis) {
+ continue;
+ }
+ String authority = parts[0];
+ if (!this.downloadData.containsKey(authority)) {
+ this.downloadData.put(authority, new ArrayList<Long>());
+ }
+ long fetchTimeMillis = Long.parseLong(parts[2]);
+ this.downloadData.get(authority).add(fetchTimeMillis);
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("Could not read "
+ + this.statisticsFile.getAbsolutePath() + ". Ignoring.");
+ }
+ for (Map.Entry<String, List<Long>> e :
+ this.downloadData.entrySet()) {
+ Collections.sort(e.getValue());
+ int downloads = e.getValue().size();
+ if (downloads > this.maxDownloadsPerAuthority) {
+ this.maxDownloadsPerAuthority = downloads;
+ }
+ }
+ }
+ }
+ public SortedSet<String> getKnownAuthorities() {
+ return new TreeSet<String>(this.downloadData.keySet());
+ }
+ public String getPercentile(String authority, int percentile) {
+ if (percentile < 0 || percentile > 100 ||
+ !this.downloadData.containsKey(authority)) {
+ return "NA";
+ } else {
+ List<Long> fetchTimes = this.downloadData.get(authority);
+ int index = (percentile * (fetchTimes.size() - 1)) / 100;
+ return String.valueOf(fetchTimes.get(index));
+ }
+ }
+ public String getNAs(String authority) {
+ if (!this.downloadData.containsKey(authority)) {
+ return "NA";
+ } else {
+ return String.valueOf(this.maxDownloadsPerAuthority
+ - this.downloadData.get(authority).size());
+ }
+ }
+}
+
diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java
index c929a96..db44b8b 100755
--- a/src/org/torproject/doctor/Downloader.java
+++ b/src/org/torproject/doctor/Downloader.java
@@ -106,7 +106,7 @@ public class Downloader {
Download result = null;
if (this.response != null) {
result = new Download(this.nickname, this.url, this.response,
- this.requestEnd - this.requestStart);
+ this.requestStart, this.requestEnd - this.requestStart);
}
return result;
}
diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java
index ec160e9..0de6298 100755
--- a/src/org/torproject/doctor/Main.java
+++ b/src/org/torproject/doctor/Main.java
@@ -22,6 +22,12 @@ public class Main {
List<Download> downloadedConsensuses = downloader.getConsensuses();
List<Download> downloadedVotes = downloader.getVotes();
+ /* Write fetch times for requesting consensuses to disk and prepare
+ * statistics about fetch times in the last 7 days. */
+ DownloadStatistics fetchStatistics = new DownloadStatistics();
+ fetchStatistics.memorizeFetchTimes(downloadedConsensuses);
+ fetchStatistics.prepareStatistics();
+
/* Parse consensus and votes. */
Parser parser = new Parser();
SortedMap<String, Status> parsedDownloadedConsensuses = parser.parse(
@@ -37,6 +43,7 @@ public class Main {
for (Report report : reports) {
report.processWarnings(warnings);
report.processDownloadedConsensuses(parsedDownloadedConsensuses);
+ report.includeFetchStatistics(fetchStatistics);
report.writeReport();
}
diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java
index b03f652..9ae897a 100755
--- a/src/org/torproject/doctor/MetricsWebsiteReport.java
+++ b/src/org/torproject/doctor/MetricsWebsiteReport.java
@@ -51,6 +51,14 @@ public class MetricsWebsiteReport implements Report {
}
}
+ /* Store the DownloadStatistics reference to request download statistics
+ * when writing the report. */
+ private DownloadStatistics statistics;
+ public void includeFetchStatistics(
+ DownloadStatistics statistics) {
+ this.statistics = statistics;
+ }
+
/* Writer to write all HTML output to. */
private BufferedWriter bw;
@@ -71,6 +79,7 @@ public class MetricsWebsiteReport implements Report {
writeAuthorityKeys();
writeBandwidthScannerStatus();
writeAuthorityVersions();
+ writeDownloadStatistics();
writeRelayFlagsTable();
writeRelayFlagsSummary();
writePageFooter();
@@ -569,6 +578,59 @@ public class MetricsWebsiteReport implements Report {
}
}
+
+ /* Write some download statistics. */
+ private void writeDownloadStatistics() throws IOException {
+ SortedSet<String> knownAuthorities =
+ this.statistics.getKnownAuthorities();
+ if (knownAuthorities.isEmpty()) {
+ return;
+ }
+ this.bw.write(" <br>\n"
+ + " <a name=\"downloadstats\">\n"
+ + " <h3><a href=\"#downloadstats\" class=\"anchor\">"
+ + "Consensus download statistics</a></h3>\n"
+ + " <br>\n"
+ + " <p>The following table contains statistics on "
+ + "consensus download times in milliseconds over the last 7 "
+ + "days:</p>\n"
+ + " <table border=\"0\" cellpadding=\"4\" "
+ + "cellspacing=\"0\" summary=\"\">\n"
+ + " <colgroup>\n"
+ + " <col width=\"160\">\n"
+ + " <col width=\"100\">\n"
+ + " <col width=\"100\">\n"
+ + " <col width=\"100\">\n"
+ + " <col width=\"100\">\n"
+ + " <col width=\"100\">\n"
+ + " <col width=\"100\">\n"
+ + " </colgroup>\n"
+ + " <tr><td><b>Authority</b></td>"
+ + "<td><b>Minimum</b></td>"
+ + "<td><b>1st Quartile</b></td>"
+ + "<td><b>Median</b></td>"
+ + "<td><b>3rd Quartile</b></td>"
+ + "<td><b>Maximum</b></td>"
+ + "<td><b>Timeouts</b></td></tr>\n");
+ for (String authority : knownAuthorities) {
+ this.bw.write(" <tr>\n"
+ + " <td>" + authority + "</td>\n"
+ + " <td>"
+ + this.statistics.getPercentile(authority, 0) + "</td>"
+ + " <td>"
+ + this.statistics.getPercentile(authority, 25) + "</td>"
+ + " <td>"
+ + this.statistics.getPercentile(authority, 50) + "</td>"
+ + " <td>"
+ + this.statistics.getPercentile(authority, 75) + "</td>"
+ + " <td>"
+ + this.statistics.getPercentile(authority, 100) + "</td>"
+ + " <td>"
+ + this.statistics.getNAs(authority) + "</td></tr>\n");
+ }
+ this.bw.write(" </table>\n");
+ }
+
/* Write the (huge) table containing relay flags contained in votes and
* the consensus for each relay. */
private void writeRelayFlagsTable() throws IOException {
diff --git a/src/org/torproject/doctor/Report.java b/src/org/torproject/doctor/Report.java
index 04e2bc1..2e508bc 100755
--- a/src/org/torproject/doctor/Report.java
+++ b/src/org/torproject/doctor/Report.java
@@ -17,6 +17,10 @@ public interface Report {
public abstract void processWarnings(
SortedMap<Warning, String> warnings);
+ /* Include download statistics. */
+ public abstract void includeFetchStatistics(
+ DownloadStatistics statistics);
+
/* Finish writing report. */
public abstract void writeReport();
}
diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java
index 9583132..1fe3b87 100755
--- a/src/org/torproject/doctor/StatusFileReport.java
+++ b/src/org/torproject/doctor/StatusFileReport.java
@@ -33,6 +33,11 @@ public class StatusFileReport implements Report {
this.warnings = warnings;
}
+ /* Ignore download statistics for this report. */
+ public void includeFetchStatistics(DownloadStatistics statistics) {
+ /* Do nothing. */
+ }
+
/* Check consensuses and votes for irregularities and write output to
* stdout. */
public void writeReport() {
More information about the tor-commits
mailing list