commit 812c1ab54f4eb99b2518d505274c5d79a4165d84 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Dec 8 12:55:38 2011 +0100

Include download statistics in the website report. --- src/org/torproject/doctor/Download.java | 9 ++- src/org/torproject/doctor/DownloadStatistics.java | 91 ++++++++++++++++++++ src/org/torproject/doctor/Downloader.java | 2 +- src/org/torproject/doctor/Main.java | 7 ++ .../torproject/doctor/MetricsWebsiteReport.java | 62 +++++++++++++ src/org/torproject/doctor/Report.java | 4 + src/org/torproject/doctor/StatusFileReport.java | 5 + 7 files changed, 178 insertions(+), 2 deletions(-)

diff --git a/src/org/torproject/doctor/Download.java b/src/org/torproject/doctor/Download.java index 35c24f9..1d53cd5 100644 --- a/src/org/torproject/doctor/Download.java +++ b/src/org/torproject/doctor/Download.java @@ -22,6 +22,12 @@ public class Download { return this.responseString; }

+ /* Request start timestamp. */ + private long requestStartMillis; + public long getRequestStartMillis() { + return this.requestStartMillis; + } + /* Fetch time in millis. */ private long fetchTime; public long getFetchTime() { @@ -29,9 +35,10 @@ public class Download { }

public Download(String authority, String url, String responseString, - long fetchTime) { + long requestStartMillis, long fetchTime) { this.authority = authority; this.responseString = responseString; + this.requestStartMillis = requestStartMillis; this.fetchTime = fetchTime; } } diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java new file mode 100644 index 0000000..9eeffde --- /dev/null +++ b/src/org/torproject/doctor/DownloadStatistics.java @@ -0,0 +1,91 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.doctor; + +import java.io.*; +import java.util.*; + +public class DownloadStatistics { + public void memorizeFetchTimes(List<Download> downloadedConsensuses) { + try { + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.statisticsFile, true)); + for (Download downloadedConsensus : downloadedConsensuses) { + String authority = downloadedConsensus.getAuthority(); + long requestStartMillis = + downloadedConsensus.getRequestStartMillis(); + long fetchTimeMillis = downloadedConsensus.getFetchTime(); + String line = authority + "," + + String.valueOf(requestStartMillis) + "," + + String.valueOf(fetchTimeMillis); + bw.write(line + "\n"); + } + bw.close(); + } catch (IOException e) { + System.err.println("Could not write " + + this.statisticsFile.getAbsolutePath() + ". Ignoring."); + } + } + private SortedMap<String, List<Long>> downloadData = + new TreeMap<String, List<Long>>(); + private int maxDownloadsPerAuthority = 0; + private File statisticsFile = new File("download-stats.csv"); + public void prepareStatistics() { + if (this.statisticsFile.exists()) { + long cutOffMillis = System.currentTimeMillis() + - (7L * 24L * 60L * 60L * 1000L); + try { + BufferedReader br = new BufferedReader(new FileReader( + this.statisticsFile)); + String line; + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + long requestStartMillis = Long.parseLong(parts[1]); + if (requestStartMillis < cutOffMillis) { + continue; + } + String authority = parts[0]; + if (!this.downloadData.containsKey(authority)) { + this.downloadData.put(authority, new ArrayList<Long>()); + } + long fetchTimeMillis = Long.parseLong(parts[2]); + this.downloadData.get(authority).add(fetchTimeMillis); + } + br.close(); + } catch (IOException e) { + System.err.println("Could not read " + + this.statisticsFile.getAbsolutePath() + ". Ignoring."); + } + for (Map.Entry<String, List<Long>> e : + this.downloadData.entrySet()) { + Collections.sort(e.getValue()); + int downloads = e.getValue().size(); + if (downloads > this.maxDownloadsPerAuthority) { + this.maxDownloadsPerAuthority = downloads; + } + } + } + } + public SortedSet<String> getKnownAuthorities() { + return new TreeSet<String>(this.downloadData.keySet()); + } + public String getPercentile(String authority, int percentile) { + if (percentile < 0 || percentile > 100 || + !this.downloadData.containsKey(authority)) { + return "NA"; + } else { + List<Long> fetchTimes = this.downloadData.get(authority); + int index = (percentile * (fetchTimes.size() - 1)) / 100; + return String.valueOf(fetchTimes.get(index)); + } + } + public String getNAs(String authority) { + if (!this.downloadData.containsKey(authority)) { + return "NA"; + } else { + return String.valueOf(this.maxDownloadsPerAuthority + - this.downloadData.get(authority).size()); + } + } +} + diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java index c929a96..db44b8b 100755 --- a/src/org/torproject/doctor/Downloader.java +++ b/src/org/torproject/doctor/Downloader.java @@ -106,7 +106,7 @@ public class Downloader { Download result = null; if (this.response != null) { result = new Download(this.nickname, this.url, this.response, - this.requestEnd - this.requestStart); + this.requestStart, this.requestEnd - this.requestStart); } return result; } diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java index ec160e9..0de6298 100755 --- a/src/org/torproject/doctor/Main.java +++ b/src/org/torproject/doctor/Main.java @@ -22,6 +22,12 @@ public class Main { List<Download> downloadedConsensuses = downloader.getConsensuses(); List<Download> downloadedVotes = downloader.getVotes();

+ /* Write fetch times for requesting consensuses to disk and prepare + * statistics about fetch times in the last 7 days. */ + DownloadStatistics fetchStatistics = new DownloadStatistics(); + fetchStatistics.memorizeFetchTimes(downloadedConsensuses); + fetchStatistics.prepareStatistics(); + /* Parse consensus and votes. */ Parser parser = new Parser(); SortedMap<String, Status> parsedDownloadedConsensuses = parser.parse( @@ -37,6 +43,7 @@ public class Main { for (Report report : reports) { report.processWarnings(warnings); report.processDownloadedConsensuses(parsedDownloadedConsensuses); + report.includeFetchStatistics(fetchStatistics); report.writeReport(); }

diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java index b03f652..9ae897a 100755 --- a/src/org/torproject/doctor/MetricsWebsiteReport.java +++ b/src/org/torproject/doctor/MetricsWebsiteReport.java @@ -51,6 +51,14 @@ public class MetricsWebsiteReport implements Report { } }

+ /* Store the DownloadStatistics reference to request download statistics + * when writing the report. */ + private DownloadStatistics statistics; + public void includeFetchStatistics( + DownloadStatistics statistics) { + this.statistics = statistics; + } + /* Writer to write all HTML output to. */ private BufferedWriter bw;

@@ -71,6 +79,7 @@ public class MetricsWebsiteReport implements Report { writeAuthorityKeys(); writeBandwidthScannerStatus(); writeAuthorityVersions(); + writeDownloadStatistics(); writeRelayFlagsTable(); writeRelayFlagsSummary(); writePageFooter(); @@ -569,6 +578,59 @@ public class MetricsWebsiteReport implements Report { } }

+ + /* Write some download statistics. */ + private void writeDownloadStatistics() throws IOException { + SortedSet<String> knownAuthorities = + this.statistics.getKnownAuthorities(); + if (knownAuthorities.isEmpty()) { + return; + } + this.bw.write(" <br>\n" + + " <a name="downloadstats">\n" + + " <h3><a href="#downloadstats" class="anchor">" + + "Consensus download statistics</a></h3>\n" + + " <br>\n" + + " <p>The following table contains statistics on " + + "consensus download times in milliseconds over the last 7 " + + "days:</p>\n" + + " <table border="0" cellpadding="4" " + + "cellspacing="0" summary="">\n" + + " <colgroup>\n" + + " <col width="160">\n" + + " <col width="100">\n" + + " <col width="100">\n" + + " <col width="100">\n" + + " <col width="100">\n" + + " <col width="100">\n" + + " <col width="100">\n" + + " </colgroup>\n" + + " <tr><td><b>Authority</b></td>" + + "<td><b>Minimum</b></td>" + + "<td><b>1st Quartile</b></td>" + + "<td><b>Median</b></td>" + + "<td><b>3rd Quartile</b></td>" + + "<td><b>Maximum</b></td>" + + "<td><b>Timeouts</b></td></tr>\n"); + for (String authority : knownAuthorities) { + this.bw.write(" <tr>\n" + + " <td>" + authority + "</td>\n" + + " <td>" + + this.statistics.getPercentile(authority, 0) + "</td>" + + " <td>" + + this.statistics.getPercentile(authority, 25) + "</td>" + + " <td>" + + this.statistics.getPercentile(authority, 50) + "</td>" + + " <td>" + + this.statistics.getPercentile(authority, 75) + "</td>" + + " <td>" + + this.statistics.getPercentile(authority, 100) + "</td>" + + " <td>" + + this.statistics.getNAs(authority) + "</td></tr>\n"); + } + this.bw.write(" </table>\n"); + } + /* Write the (huge) table containing relay flags contained in votes and * the consensus for each relay. */ private void writeRelayFlagsTable() throws IOException { diff --git a/src/org/torproject/doctor/Report.java b/src/org/torproject/doctor/Report.java index 04e2bc1..2e508bc 100755 --- a/src/org/torproject/doctor/Report.java +++ b/src/org/torproject/doctor/Report.java @@ -17,6 +17,10 @@ public interface Report { public abstract void processWarnings( SortedMap<Warning, String> warnings);

+ /* Include download statistics. */ + public abstract void includeFetchStatistics( + DownloadStatistics statistics); + /* Finish writing report. */ public abstract void writeReport(); } diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java index 9583132..1fe3b87 100755 --- a/src/org/torproject/doctor/StatusFileReport.java +++ b/src/org/torproject/doctor/StatusFileReport.java @@ -33,6 +33,11 @@ public class StatusFileReport implements Report { this.warnings = warnings; }

+ /* Ignore download statistics for this report. */ + public void includeFetchStatistics(DownloadStatistics statistics) { + /* Do nothing. */ + } + /* Check consensuses and votes for irregularities and write output to * stdout. */ public void writeReport() {