commit c7c45eedad124208fc696c3136ba87e3740914af Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Mar 17 13:32:41 2011 +0100

Add code for bw scanner failure graph. --- task-2772/.gitignore | 6 ++ task-2772/Eval.java | 125 ++++++++++++++++++++++++++++++++++++++++ task-2772/README | 30 ++++++++++ task-2772/torperf-bwscanners.R | 32 ++++++++++ 4 files changed, 193 insertions(+), 0 deletions(-)

diff --git a/task-2772/.gitignore b/task-2772/.gitignore new file mode 100644 index 0000000..820c53c --- /dev/null +++ b/task-2772/.gitignore @@ -0,0 +1,6 @@ +consensus-params +votes-measured +*.class +*.csv +*.pdf + diff --git a/task-2772/Eval.java b/task-2772/Eval.java new file mode 100755 index 0000000..c4344fb --- /dev/null +++ b/task-2772/Eval.java @@ -0,0 +1,125 @@ +import java.io.*; +import java.text.*; +import java.util.*; +public class Eval { + public static void main(String[] args) throws Exception { + BufferedReader br = new BufferedReader(new FileReader( + "consensus-params")); + String line = null; + SortedMap<String, String> consensusParams = + new TreeMap<String, String>(); + SortedSet<String> params = new TreeSet<String>(Arrays.asList(( + "CircPriorityHalflifeMsec,CircuitPriorityHalflifeMsec," + + "CircuitPriorityHalflife,bwconnburst,bwconnrate," + + "circwindow,cbtquantile,refuseunknownexits,cbtnummodes"). + split(","))); + while ((line = br.readLine()) != null) { + String date = line.substring(23, 33); + StringBuilder sb = new StringBuilder(); + for (String param : params) { + if (line.contains(param + "=")) { + sb.append("," + line.substring(line.indexOf(param + "=")). + split(" ")[0].split("=")[1]); + } else { + sb.append(",NA"); + } + } + consensusParams.put(date, sb.toString()); + String[] parts = line.split(" "); + for (int i = 1; i < parts.length; i++) { + if (!params.contains(parts[i].split("=")[0])) { + System.out.println("Unknown consensus param '" + + parts[i].split("=")[0] + "' in " + line); + } + } + } + br.close(); + br = new BufferedReader(new FileReader("votes-measured")); + SortedMap<String, String> votesMeasured = + new TreeMap<String, String>(); + long bwscannerStart = 0L, bwscannerEnd = 0L; + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + BufferedWriter bw = new BufferedWriter(new FileWriter( + "bwscanner-failures.csv")); + bw.write("start,end\n"); + while ((line = br.readLine()) != null) { + String date = line.substring(5, 15); + String votes = line.substring(0, 5).trim(); + votesMeasured.put(date, votes); + String dateTime = line.substring(5, 15) + " " + + line.substring(16, 18) + ":" + line.substring(19, 21) + ":" + + line.substring(22, 24); + long dateTimeSeconds = dateTimeFormat.parse(dateTime).getTime(); + if (Integer.parseInt(votes) >= 3) { + if (bwscannerStart == 0L) { + bwscannerStart = bwscannerEnd = dateTimeSeconds; + } else if (bwscannerEnd + 12L * 60L * 60L * 1000L >= + dateTimeSeconds) { + bwscannerEnd = dateTimeSeconds; + } else { + //bw.write(dateFormat.format(bwscannerStart) + "," + // + dateFormat.format(bwscannerEnd) + "\n"); + bw.write(dateFormat.format(bwscannerEnd) + "," + + dateFormat.format(dateTimeSeconds + + 24L * 60L * 60L * 1000L) + "\n"); + bwscannerStart = bwscannerEnd = dateTimeSeconds; + } + } + } + /*if (bwscannerStart > 0L) { + bw.write(dateFormat.format(bwscannerStart) + "," + + dateFormat.format(bwscannerEnd) + "\n"); + }*/ + bw.close(); + br.close(); + br = new BufferedReader(new FileReader("torperf.csv")); + br.readLine(); + bw = new BufferedWriter(new FileWriter("torperf-stats.csv")); + bw.write("date,source,md"); + for (String param : params) { + bw.write("," + param); + } + bw.write(",votesMeasured\n"); + long lastDateSeconds = 0L; + String lastSource = null; + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + String date = parts[1]; + long dateSeconds = dateFormat.parse(date).getTime(); + String source = parts[0]; + while (source.equals(lastSource) && + lastDateSeconds + 24L * 60L * 60L * 1000L < dateSeconds) { + lastDateSeconds += 24L * 60L * 60L * 1000L; + bw.write(dateFormat.format(lastDateSeconds) + "," + source + + ",NA"); + for (String param : params) { + bw.write(",NA"); + } + bw.write(",NA\n"); + } + lastDateSeconds = dateSeconds; + lastSource = source; + String md = parts[3]; + bw.write(date + "," + source + "," + md); + if (consensusParams.containsKey(date)) { + bw.write(consensusParams.get(date)); + } else { + for (String param : params) { + bw.write(",NA"); + } + } + if (votesMeasured.containsKey(date)) { + bw.write("," + votesMeasured.get(date) + "\n"); + } else { + bw.write(",NA\n"); + } + } + bw.close(); + br.close(); + } +} + diff --git a/task-2772/README b/task-2772/README new file mode 100644 index 0000000..2c8b94e --- /dev/null +++ b/task-2772/README @@ -0,0 +1,30 @@ +Graphing torperf results and bandwidth authority failures +========================================================= + + - Download the consensus and vote tarballs from the metrics website and + extract them locally. You'll want the tarballs since October 2009. + + - Run grep to extract the relevant pieces and save Java the bulk of the + parsing work. + + $ grep -Rm1 "Measured" votes-20* | cut -c 18-36 | sort | uniq -c + > votes-measured + + $ grep -Rm1 "^params " consensuses-20* > consensus-params + + - Download the Torperf statistics from the metrics website here: + + https://metrics.torproject.org/csv/torperf.csv + + - Compile and run Eval.java + + $ javac Eval.java && java Eval + + - You should find two files bwscanner-failures.csv and torperf-stats.csv + in this directory. + + - Run the R code to draw the graph of Torperf results and bwscanner + outages. The result will be a new file torperf-bwscanners.pdf. + + $ R --slave -f torperf-bwscanners.R + diff --git a/task-2772/torperf-bwscanners.R b/task-2772/torperf-bwscanners.R new file mode 100644 index 0000000..f2dee38 --- /dev/null +++ b/task-2772/torperf-bwscanners.R @@ -0,0 +1,32 @@ +options(warn = -1) +suppressPackageStartupMessages(library("ggplot2")) + +a <- read.csv("torperf-stats.csv", stringsAsFactors = FALSE, + header = TRUE) +a <- a[a$source == "all-1mb",] +ymax <- max(a$md, na.rm = TRUE) / 1e3 + +b <- read.csv("bwscanner-failures.csv", stringsAsFactors = FALSE, + header = TRUE) +b <- b[b$end >= min(a$date, na.rm = TRUE),] +b[1,1] <- max(b[1,1], min(a$date, na.rm = TRUE)) + +ggplot(a, aes(x = as.Date(date), y = md/1e3)) + + +geom_line(size = 0.5) + + +scale_y_continuous(name = paste("Download time of 1 MB file over Tor", + "in seconds\n")) + + +scale_x_date(name = "") + + +geom_rect(aes(NULL, NULL, xmin = as.Date(start), xmax = as.Date(end), + ymin = -Inf, ymax = Inf, fill = TRUE), data = b) + + +scale_fill_manual(name = "", breaks = TRUE, + labels = "Bandwidth scanners failing", values = alpha("red2", 0.4)) + + +opts(legend.position = "bottom") + +ggsave(filename = "torperf-bwscanners.pdf", width = 8, height = 6) +