[tor-commits] [metrics-tasks/master] Add code for bw scanner failure graph.
karsten at torproject.org
karsten at torproject.org
Thu Mar 17 12:33:30 UTC 2011
commit c7c45eedad124208fc696c3136ba87e3740914af
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Mar 17 13:32:41 2011 +0100
Add code for bw scanner failure graph.
---
task-2772/.gitignore | 6 ++
task-2772/Eval.java | 125 ++++++++++++++++++++++++++++++++++++++++
task-2772/README | 30 ++++++++++
task-2772/torperf-bwscanners.R | 32 ++++++++++
4 files changed, 193 insertions(+), 0 deletions(-)
diff --git a/task-2772/.gitignore b/task-2772/.gitignore
new file mode 100644
index 0000000..820c53c
--- /dev/null
+++ b/task-2772/.gitignore
@@ -0,0 +1,6 @@
+consensus-params
+votes-measured
+*.class
+*.csv
+*.pdf
+
diff --git a/task-2772/Eval.java b/task-2772/Eval.java
new file mode 100755
index 0000000..c4344fb
--- /dev/null
+++ b/task-2772/Eval.java
@@ -0,0 +1,125 @@
+import java.io.*;
+import java.text.*;
+import java.util.*;
+public class Eval {
+ public static void main(String[] args) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ "consensus-params"));
+ String line = null;
+ SortedMap<String, String> consensusParams =
+ new TreeMap<String, String>();
+ SortedSet<String> params = new TreeSet<String>(Arrays.asList((
+ "CircPriorityHalflifeMsec,CircuitPriorityHalflifeMsec,"
+ + "CircuitPriorityHalflife,bwconnburst,bwconnrate,"
+ + "circwindow,cbtquantile,refuseunknownexits,cbtnummodes").
+ split(",")));
+ while ((line = br.readLine()) != null) {
+ String date = line.substring(23, 33);
+ StringBuilder sb = new StringBuilder();
+ for (String param : params) {
+ if (line.contains(param + "=")) {
+ sb.append("," + line.substring(line.indexOf(param + "=")).
+ split(" ")[0].split("=")[1]);
+ } else {
+ sb.append(",NA");
+ }
+ }
+ consensusParams.put(date, sb.toString());
+ String[] parts = line.split(" ");
+ for (int i = 1; i < parts.length; i++) {
+ if (!params.contains(parts[i].split("=")[0])) {
+ System.out.println("Unknown consensus param '"
+ + parts[i].split("=")[0] + "' in " + line);
+ }
+ }
+ }
+ br.close();
+ br = new BufferedReader(new FileReader("votes-measured"));
+ SortedMap<String, String> votesMeasured =
+ new TreeMap<String, String>();
+ long bwscannerStart = 0L, bwscannerEnd = 0L;
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ "bwscanner-failures.csv"));
+ bw.write("start,end\n");
+ while ((line = br.readLine()) != null) {
+ String date = line.substring(5, 15);
+ String votes = line.substring(0, 5).trim();
+ votesMeasured.put(date, votes);
+ String dateTime = line.substring(5, 15) + " "
+ + line.substring(16, 18) + ":" + line.substring(19, 21) + ":"
+ + line.substring(22, 24);
+ long dateTimeSeconds = dateTimeFormat.parse(dateTime).getTime();
+ if (Integer.parseInt(votes) >= 3) {
+ if (bwscannerStart == 0L) {
+ bwscannerStart = bwscannerEnd = dateTimeSeconds;
+ } else if (bwscannerEnd + 12L * 60L * 60L * 1000L >=
+ dateTimeSeconds) {
+ bwscannerEnd = dateTimeSeconds;
+ } else {
+ //bw.write(dateFormat.format(bwscannerStart) + ","
+ // + dateFormat.format(bwscannerEnd) + "\n");
+ bw.write(dateFormat.format(bwscannerEnd) + ","
+ + dateFormat.format(dateTimeSeconds
+ + 24L * 60L * 60L * 1000L) + "\n");
+ bwscannerStart = bwscannerEnd = dateTimeSeconds;
+ }
+ }
+ }
+ /*if (bwscannerStart > 0L) {
+ bw.write(dateFormat.format(bwscannerStart) + ","
+ + dateFormat.format(bwscannerEnd) + "\n");
+ }*/
+ bw.close();
+ br.close();
+ br = new BufferedReader(new FileReader("torperf.csv"));
+ br.readLine();
+ bw = new BufferedWriter(new FileWriter("torperf-stats.csv"));
+ bw.write("date,source,md");
+ for (String param : params) {
+ bw.write("," + param);
+ }
+ bw.write(",votesMeasured\n");
+ long lastDateSeconds = 0L;
+ String lastSource = null;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ String date = parts[1];
+ long dateSeconds = dateFormat.parse(date).getTime();
+ String source = parts[0];
+ while (source.equals(lastSource) &&
+ lastDateSeconds + 24L * 60L * 60L * 1000L < dateSeconds) {
+ lastDateSeconds += 24L * 60L * 60L * 1000L;
+ bw.write(dateFormat.format(lastDateSeconds) + "," + source
+ + ",NA");
+ for (String param : params) {
+ bw.write(",NA");
+ }
+ bw.write(",NA\n");
+ }
+ lastDateSeconds = dateSeconds;
+ lastSource = source;
+ String md = parts[3];
+ bw.write(date + "," + source + "," + md);
+ if (consensusParams.containsKey(date)) {
+ bw.write(consensusParams.get(date));
+ } else {
+ for (String param : params) {
+ bw.write(",NA");
+ }
+ }
+ if (votesMeasured.containsKey(date)) {
+ bw.write("," + votesMeasured.get(date) + "\n");
+ } else {
+ bw.write(",NA\n");
+ }
+ }
+ bw.close();
+ br.close();
+ }
+}
+
diff --git a/task-2772/README b/task-2772/README
new file mode 100644
index 0000000..2c8b94e
--- /dev/null
+++ b/task-2772/README
@@ -0,0 +1,30 @@
+Graphing torperf results and bandwidth authority failures
+=========================================================
+
+ - Download the consensus and vote tarballs from the metrics website and
+ extract them locally. You'll want the tarballs since October 2009.
+
+ - Run grep to extract the relevant pieces and save Java the bulk of the
+ parsing work.
+
+ $ grep -Rm1 "Measured" votes-20* | cut -c 18-36 | sort | uniq -c
+ > votes-measured
+
+ $ grep -Rm1 "^params " consensuses-20* > consensus-params
+
+ - Download the Torperf statistics from the metrics website here:
+
+ https://metrics.torproject.org/csv/torperf.csv
+
+ - Compile and run Eval.java
+
+ $ javac Eval.java && java Eval
+
+ - You should find two files bwscanner-failures.csv and torperf-stats.csv
+ in this directory.
+
+ - Run the R code to draw the graph of Torperf results and bwscanner
+ outages. The result will be a new file torperf-bwscanners.pdf.
+
+ $ R --slave -f torperf-bwscanners.R
+
diff --git a/task-2772/torperf-bwscanners.R b/task-2772/torperf-bwscanners.R
new file mode 100644
index 0000000..f2dee38
--- /dev/null
+++ b/task-2772/torperf-bwscanners.R
@@ -0,0 +1,32 @@
+options(warn = -1)
+suppressPackageStartupMessages(library("ggplot2"))
+
+a <- read.csv("torperf-stats.csv", stringsAsFactors = FALSE,
+ header = TRUE)
+a <- a[a$source == "all-1mb",]
+ymax <- max(a$md, na.rm = TRUE) / 1e3
+
+b <- read.csv("bwscanner-failures.csv", stringsAsFactors = FALSE,
+ header = TRUE)
+b <- b[b$end >= min(a$date, na.rm = TRUE),]
+b[1,1] <- max(b[1,1], min(a$date, na.rm = TRUE))
+
+ggplot(a, aes(x = as.Date(date), y = md/1e3)) +
+
+geom_line(size = 0.5) +
+
+scale_y_continuous(name = paste("Download time of 1 MB file over Tor",
+ "in seconds\n")) +
+
+scale_x_date(name = "") +
+
+geom_rect(aes(NULL, NULL, xmin = as.Date(start), xmax = as.Date(end),
+ ymin = -Inf, ymax = Inf, fill = TRUE), data = b) +
+
+scale_fill_manual(name = "", breaks = TRUE,
+ labels = "Bandwidth scanners failing", values = alpha("red2", 0.4)) +
+
+opts(legend.position = "bottom")
+
+ggsave(filename = "torperf-bwscanners.pdf", width = 8, height = 6)
+
More information about the tor-commits
mailing list