[metrics-web/master] Refactor even more of the graphing code. - tor-commits

1 Mar 2018

commit 5c8e727b61f516ddb30d2400d6f3c97462443801
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Thu Mar 1 12:27:07 2018 +0100
Refactor even more of the graphing code.
---
 src/main/R/rserver/graphs.R | 428 +++++++++++++++++---------------------------
 1 file changed, 167 insertions(+), 261 deletions(-)

diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 8847967..93ff919 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -329,28 +329,20 @@ stats_dir = "/srv/metrics.torproject.org/metrics/shared/stats/"
 rdata_dir = "/srv/metrics.torproject.org/metrics/shared/RData/"
prepare_networksize <- function(start, end) {
-  s <- read.csv(paste(stats_dir, "servers.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  s <- s[s$date >= start & s$date <= end & s$flag == '' &
-         s$country == '' & s$version == '' & s$platform == '' &
-         s$ec2bridge == '', ]
-  s <- data.frame(date = as.Date(s$date, "%Y-%m-%d"), relays = s$relays,
-                  bridges = s$bridges)
-  s
+  read.csv(paste(stats_dir, "servers.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), flag == '',
+      country == '', version == '', platform == '', ec2bridge == '') %>%
+    select(date, relays, bridges)
 }
plot_networksize <- function(start, end, path) {
-  s <- prepare_networksize(start, end)
-  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
-      to = as.Date(end, "%Y-%m-%d"), by="1 day")
-  missing <- setdiff(dates, as.Date(s$date, origin = "1970-01-01"))
-  if (length(missing) > 0)
-    s <- rbind(s,
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        relays = NA, bridges = NA))
-  networksize <- gather(s, variable, value, -date)
-  ggplot(networksize, aes(x = as.Date(date, "%Y-%m-%d"), y = value,
-    colour = variable)) + geom_line() +
+  prepare_networksize(start, end) %>%
+    gather(variable, value, -date) %>%
+    complete(date = full_seq(date, period = 1),
+      variable = c("relays", "bridges")) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
+    geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
     scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
@@ -367,14 +359,11 @@ write_networksize <- function(start, end, path) {
 }
prepare_versions <- function(start, end) {
-  s <- read.csv(paste(stats_dir, "servers.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  s <- s[s$date >= start & s$date <= end & s$flag == '' &
-         s$country == '' & s$version != '' & s$platform == '' &
-         s$ec2bridge == '', ]
-  s <- data.frame(date = as.Date(s$date, "%Y-%m-%d"), version = s$version,
-                  relays = s$relays)
-  s
+  read.csv(paste(stats_dir, "servers.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), flag == '',
+      country == '', version != '', platform == '', ec2bridge == '') %>%
+    select(date, version, relays)
 }
plot_versions <- function(start, end, path) {
@@ -389,8 +378,7 @@ plot_versions <- function(start, end, path) {
     stringsAsFactors = FALSE)
   versions <- s[s$version %in% known_versions, ]
   visible_versions <- sort(unique(versions$version))
-  ggplot(versions, aes(x = as.Date(date, "%Y-%m-%d"), y = relays,
-      colour = version)) +
+  ggplot(versions, aes(x = date, y = relays, colour = version)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -410,21 +398,18 @@ write_versions <- function(start, end, path) {
 }
prepare_platforms <- function(start, end) {
-  s <- read.csv(paste(stats_dir, "servers.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  s <- s[s$date >= start & s$date <= end & s$flag == '' &
-         s$country == '' & s$version == '' & s$platform != '' &
-         s$ec2bridge == '', ]
-  platforms <- data.frame(date = as.Date(s$date, "%Y-%m-%d"),
-      variable = ifelse(s$platform == "Darwin", "macOS", s$platform),
-      value = s$relays)
-  platforms
+  read.csv(paste(stats_dir, "servers.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), flag == '',
+      country == '', version == '', platform != '', ec2bridge == '') %>%
+    select(date, platform, relays) %>%
+    mutate(platform = ifelse(platform == "Darwin", "macOS",
+      as.character(platform)))
 }
plot_platforms <- function(start, end, path) {
-  platforms <- prepare_platforms(start, end)
-  ggplot(platforms, aes(x = as.Date(date, "%Y-%m-%d"), y = value,
-      colour = variable)) +
+  prepare_platforms(start, end) %>%
+    ggplot(aes(x = date, y = relays, colour = platform)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -439,33 +424,31 @@ plot_platforms <- function(start, end, path) {
write_platforms <- function(start, end, path) {
   prepare_platforms(start, end) %>%
-    spread(variable, value) %>%
+    spread(platform, relays) %>%
     write.csv(path, quote = FALSE, row.names = FALSE)
 }
prepare_bandwidth <- function(start, end) {
-  b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  b <- b[b$date >= start & b$date <= end & b$isexit == '' &
-         b$isguard == '', ]
-  b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
-                  bwadv = b$advbw * 8 / 1e9,
-                  bwhist = (b$bwread + b$bwwrite) * 8 / 2e9)
-  b
+  read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), isexit == '',
+      isguard == '') %>%
+    mutate(advbw = advbw * 8 / 1e9) %>%
+    mutate(bwhist = (bwread + bwwrite) * 8 / 2e9) %>%
+    select(date, advbw, bwhist)
 }
plot_bandwidth <- function(start, end, path) {
-  b <- prepare_bandwidth(start, end)
-  bandwidth <- gather(b, variable, value, -date)
-  ggplot(bandwidth, aes(x = as.Date(date, "%Y-%m-%d"),
-      y = value, colour = variable)) +
+  prepare_bandwidth(start, end) %>%
+    gather(variable, value, -date) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
     scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
       limits = c(0, NA)) +
     scale_colour_hue(name = "", h.start = 90,
-        breaks = c("bwadv", "bwhist"),
+        breaks = c("advbw", "bwhist"),
         labels = c("Advertised bandwidth", "Bandwidth history")) +
     ggtitle("Total relay bandwidth") +
     labs(caption = copyright_notice) +
@@ -479,37 +462,22 @@ write_bandwidth <- function(start, end, path) {
 }
prepare_bwhist_flags <- function(start, end) {
-  b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  b <- b[b$date >= start & b$date <= end & b$isexit != '' &
-         b$isguard != '', ]
-  bw <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
-                  isexit = b$isexit == 't', isguard = b$isguard == 't',
-                  read = b$bwread, written = b$bwwrite)
-  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
-      to = as.Date(end, "%Y-%m-%d"), by = "1 day")
-  missing <- setdiff(dates, as.Date(bw$date, origin = "1970-01-01"))
-  if (length(missing) > 0)
-    bw <- rbind(bw,
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        isexit = FALSE, isguard = FALSE, read = NA, written = NA),
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        isexit = FALSE, isguard = TRUE, read = NA, written = NA),
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        isexit = TRUE, isguard = FALSE, read = NA, written = NA),
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        isexit = TRUE, isguard = TRUE, read = NA, written = NA))
-  bw <- data.frame(date = bw$date, variable = ifelse(bw$isexit,
-        ifelse(bw$isguard, "guard_and_exit", "exit_only"),
-        ifelse(bw$isguard, "guard_only", "middle_only")),
-        value = (bw$read + bw$written) * 8 / 2e9)
-  bw
+  read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), isexit != '',
+      isguard != '') %>%
+    mutate(variable = ifelse(isexit == 't',
+        ifelse(isguard == 't', "guard_and_exit", "exit_only"),
+        ifelse(isguard == 't', "guard_only", "middle_only")),
+      value = (bwread + bwwrite) * 8 / 2e9) %>%
+    select(date, variable, value)
 }
plot_bwhist_flags <- function(start, end, path) {
-  bw <- prepare_bwhist_flags(start, end)
-  ggplot(bw, aes(x = as.Date(date, "%Y-%m-%d"), y = value,
-      colour = variable)) +
+  prepare_bwhist_flags(start, end) %>%
+    complete(date = full_seq(date, period = 1),
+      variable = unique(variable)) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -532,21 +500,19 @@ write_bwhist_flags <- function(start, end, path) {
 }
prepare_dirbytes <- function(start, end, path) {
-  b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  b <- b[b$date >= start & b$date <= end & b$isexit == '' &
-         b$isguard == '', ]
-  b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
-                  dirread = b$dirread * 8 / 1e9,
-                  dirwrite = b$dirwrite * 8 / 1e9)
-  b
+  read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), isexit == '',
+      isguard == '') %>%
+    mutate(dirread = dirread * 8 / 1e9,
+      dirwrite = dirwrite * 8 / 1e9) %>%
+    select(date, dirread, dirwrite)
 }
plot_dirbytes <- function(start, end, path) {
-  b <- prepare_dirbytes(start, end)
-  dir <- gather(b, variable, value, -date)
-  ggplot(dir, aes(x = as.Date(date, "%Y-%m-%d"), y = value,
-      colour = variable)) +
+  prepare_dirbytes(start, end) %>%
+    gather(variable, value, -date) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -567,33 +533,20 @@ write_dirbytes <- function(start, end, path) {
 }
prepare_relayflags <- function(start, end, flags) {
-  s <- read.csv(paste(stats_dir, "servers.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  s <- s[s$date >= start & s$date <= end & s$country == '' &
-         s$version == '' & s$platform == '' & s$ec2bridge == '', ]
-  s <- data.frame(date = as.Date(s$date, "%Y-%m-%d"),
-                  variable = ifelse(s$flag == '', 'Running', s$flag),
-                  value = s$relays)
-  networksize <- s[s$variable %in% flags, ]
-  networksize
+  read.csv(paste(stats_dir, "servers.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end), country == '',
+      version == '', platform == '', ec2bridge == '') %>%
+    mutate(flag = ifelse(flag == '', 'Running', as.character(flag))) %>%
+    filter(flag %in% flags) %>%
+    select(date, flag, relays)
 }
plot_relayflags <- function(start, end, flags, path) {
-  networksize <- prepare_relayflags(start, end, flags)
-  networksize <- rbind(data.frame(
-    date = as.Date(end) + 1,
-    variable = c("Running", "Exit", "Guard", "Fast", "Stable", "HSDir"),
-    value = NA), networksize)
-  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
-      to = as.Date(end, "%Y-%m-%d"), by="1 day")
-  missing <- setdiff(dates, networksize$date)
-  if (length(missing) > 0)
-    networksize <- rbind(data.frame(
-      date = as.Date(rep(missing, 6), origin = "1970-01-01"),
-      variable = c("Running", "Exit", "Guard", "Fast", "Stable", "HSDir"),
-      value = rep(NA, length(missing) * 6)), networksize)
-  ggplot(networksize, aes(x = as.Date(date, "%Y-%m-%d"), y = value,
-    colour = as.factor(variable))) + geom_line() +
+  prepare_relayflags(start, end, flags) %>%
+    complete(date = full_seq(date, period = 1), flag = unique(flag)) %>%
+    ggplot(aes(x = date, y = relays, colour = as.factor(flag))) +
+    geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
     scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) +
@@ -616,32 +569,24 @@ plot_torperf <- function(start, end, source, server, filesize, path) {
   filesizeVal <- ifelse(filesize == '50kb', 50 * 1024,
           ifelse(filesize == '1mb', 1024 * 1024, 5 * 1024 * 1024))
   t <- read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
-    stringsAsFactors = FALSE)
+    colClasses = c("date" = "Date"))
   known_sources <- c("all", unique(t[t$source != "", "source"]))
   colours <- data.frame(source = known_sources,
       colour = brewer.pal(length(known_sources), "Paired"),
       stringsAsFactors = FALSE)
-  t <- t[t$date >= start & t$date <= end & t$filesize == filesizeVal &
-         t$source == ifelse(source == 'all', '', source) &
-         t$server == server, ]
-  torperf <- data.frame(date = as.Date(t$date, "%Y-%m-%d"),
-                        q1 = t$q1, md = t$md, q3 = t$q3)
-  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
-      to = as.Date(end, "%Y-%m-%d"), by="1 day")
-  missing <- setdiff(dates, torperf$date)
-  if (length(missing) > 0)
-    torperf <- rbind(torperf,
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        q1 = NA, md = NA, q3 = NA))
   colour <- colours[colours$source == source, "colour"]
   filesizes <- data.frame(filesizes = c("5mb", "1mb", "50kb"),
       label = c("5 MiB", "1 MiB", "50 KiB"), stringsAsFactors = FALSE)
   filesizeStr <- filesizes[filesizes$filesize == filesize, "label"]
-  ggplot(torperf, aes(x = as.Date(date, "%Y-%m-%d"), y = md/1e3,
-      fill = "line")) +
+  t[t$date >= as.Date(start) & t$date <= as.Date(end) &
+         t$filesize == filesizeVal &
+         t$source == ifelse(source == 'all', '', source) &
+         t$server == server, ] %>%
+    transmute(date, q1 = q1 / 1e3, md = md / 1e3, q3 = q3 / 1e3) %>%
+    complete(date = full_seq(date, period = 1)) %>%
+    ggplot(aes(x = date, y = md, fill = "line")) +
     geom_line(colour = colour, size = 0.75) +
-    geom_ribbon(data = torperf, aes(x = date, ymin = q1/1e3,
-      ymax = q3/1e3, fill = "ribbon")) +
+    geom_ribbon(aes(x = date, ymin = q1, ymax = q3, fill = "ribbon")) +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
     scale_y_continuous(name = "", labels = unit_format(unit = "s"),
@@ -671,8 +616,7 @@ write_torperf <- function(start, end, source_, server_, filesize_, path) {
         ifelse(filesize_ == '1mb', 1024 * 1024, 5 * 1024 * 1024)),
       source == ifelse(source_ == 'all', '', source_),
       server == server_) %>%
-    select(date, q1, md, q3) %>%
-    mutate(q1 = q1 / 1e3, md = md / 1e3, q3 = q3 / 1e3) %>%
+    transmute(date, q1 = q1 / 1e3, md = md / 1e3, q3 = q3 / 1e3) %>%
     write.csv(path, quote = FALSE, row.names = FALSE)
 }
@@ -680,31 +624,21 @@ prepare_torperf_failures <- function(start, end, source, server, filesize) {
   filesizeVal <- ifelse(filesize == '50kb', 50 * 1024,
           ifelse(filesize == '1mb', 1024 * 1024, 5 * 1024 * 1024))
   t <- read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  t <- t[t$date >= start & t$date <= end & t$filesize == filesizeVal &
+    colClasses = c("date" = "Date"))
+  t[t$date >= start & t$date <= end & t$filesize == filesizeVal &
          t$source == ifelse(source == 'all', '', source) &
-         t$server == server & t$requests > 0, ]
-  torperf <- data.frame(date = as.Date(t$date, "%Y-%m-%d"),
-                        timeouts = t$timeouts / t$requests,
-                        failures = t$failures / t$requests)
-  torperf
+         t$server == server & t$requests > 0, ] %>%
+  transmute(date, timeouts = timeouts / requests,
+    failures = failures / requests)
 }
plot_torperf_failures <- function(start, end, source, server, filesize, path) {
-  torperf <- prepare_torperf_failures(start, end, source, server, filesize)
-  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
-      to = as.Date(end, "%Y-%m-%d"), by="1 day")
-  missing <- setdiff(dates, torperf$date)
-  if (length(missing) > 0)
-    torperf <- rbind(torperf,
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        timeouts = NA, failures = NA))
-  torperf <- gather(torperf, variable, value, -date)
   filesizes <- data.frame(filesizes = c("5mb", "1mb", "50kb"),
       label = c("5 MiB", "1 MiB", "50 KiB"), stringsAsFactors = FALSE)
   filesizeStr <- filesizes[filesizes$filesize == filesize, "label"]
-  ggplot(torperf, aes(x = as.Date(date, "%Y-%m-%d"), y = value,
-    colour = variable)) +
+  prepare_torperf_failures(start, end, source, server, filesize) %>%
+    gather(variable, value, -date) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
     geom_point(size = 2) +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -726,21 +660,17 @@ write_torperf_failures <- function(start, end, source, server, filesize, path) {
 }
prepare_connbidirect <- function(start, end) {
-  c <- read.csv(paste(stats_dir, "connbidirect2.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  c <- c[c$date >= start & c$date <= end, ]
-  c <- data.frame(date = as.Date(c$date),
-                  direction = factor(c$direction,
-                              levels = c("both", "write", "read")),
-                  quantile = paste("X", c$quantile, sep = ""),
-                  fraction = c$fraction / 100)
-  c <- spread(c, quantile, fraction)
-  c
+  read.csv(paste(stats_dir, "connbidirect2.csv", sep = ""),
+    colClasses = c("date" = "Date", "direction" = "factor")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end)) %>%
+    mutate(quantile = paste("X", quantile, sep = ""),
+      fraction = fraction / 100) %>%
+    spread(quantile, fraction)
 }
plot_connbidirect <- function(start, end, path) {
-  c <- prepare_connbidirect(start, end)
-  ggplot(c, aes(x = date, y = X0.5, colour = direction)) +
+  prepare_connbidirect(start, end) %>%
+    ggplot(aes(x = date, y = X0.5, colour = direction)) +
     geom_line(size = 0.75) +
     geom_ribbon(aes(x = date, ymin = X0.25, ymax = X0.75,
                 fill = direction), alpha = 0.5, show.legend = FALSE) +
@@ -772,10 +702,10 @@ write_connbidirect <- function(start, end, path) {
prepare_bandwidth_flags <- function(start, end) {
   b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""),
-    stringsAsFactors = FALSE)
+    colClasses = c("date" = "Date"))
   b <- b[b$date >= start & b$date <= end & b$isexit != '' &
          b$isguard != '', ]
-  b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
+  b <- data.frame(date = b$date,
                   isexit = b$isexit == 't', isguard = b$isguard == 't',
                   advbw = b$advbw * 8 / 1e9,
                   bwhist = (b$bwread + b$bwwrite) * 8 / 2e9)
@@ -789,21 +719,6 @@ prepare_bandwidth_flags <- function(start, end) {
                  na.rm = TRUE, na.action = NULL)
   b <- gather(b, type, value, -c(date, flag))
   bandwidth <- b[b$value > 0, ]
-  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
-      to = as.Date(end, "%Y-%m-%d"), by = "1 day")
-  missing <- setdiff(dates, as.Date(bandwidth$date,
-    origin = "1970-01-01"))
-  if (length(missing) > 0) {
-    bandwidth <- rbind(bandwidth,
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        type = "advbw", flag = "exit", value = NA),
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        type = "bwhist", flag = "exit", value = NA),
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        type = "advbw", flag = "guard", value = NA),
-        data.frame(date = as.Date(missing, origin = "1970-01-01"),
-        type = "bwhist", flag = "guard", value = NA))
-  }
   bandwidth <- data.frame(date = bandwidth$date,
     variable = as.factor(paste(bandwidth$flag, "_", bandwidth$type,
     sep = "")), value = bandwidth$value)
@@ -813,9 +728,10 @@ prepare_bandwidth_flags <- function(start, end) {
 }
plot_bandwidth_flags <- function(start, end, path) {
-  bandwidth <- prepare_bandwidth_flags(start, end)
-  ggplot(bandwidth, aes(x = as.Date(date, "%Y-%m-%d"),
-      y = value, colour = variable)) +
+  prepare_bandwidth_flags(start, end) %>%
+    complete(date = full_seq(date, period = 1),
+      variable = unique(variable)) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -1107,29 +1023,27 @@ write_userstats_bridge_combined <- function(start, end, country, path) {
 }
prepare_advbwdist_perc <- function(start, end, p) {
-  t <- read.csv(paste(stats_dir,
-                "advbwdist.csv", sep = ""), stringsAsFactors = FALSE)
-  t <- t[t$date >= start & t$date <= end &
-         t$percentile %in% as.numeric(p), ]
-  t <- data.frame(date = t$date, percentile = as.factor(t$percentile),
-                  variable = ifelse(t$isexit != "t", "all", "exits"),
-                  advbw = t$advbw * 8 / 1e9)
-  t
+  read.csv(paste(stats_dir, "advbwdist.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end),
+      percentile %in% as.numeric(p)) %>%
+    transmute(date, percentile = as.factor(percentile),
+      variable = ifelse(isexit != "t", "all", "exits"),
+      advbw = advbw * 8 / 1e9)
 }
plot_advbwdist_perc <- function(start, end, p, path) {
-  t <- prepare_advbwdist_perc(start, end, p)
-  t$variable <- ifelse(t$variable == "all", "All relays",
-                       "Exits only")
-  ggplot(t, aes(x = as.Date(date), y = advbw, colour = percentile)) +
+  prepare_advbwdist_perc(start, end, p) %>%
+    mutate(variable = ifelse(variable == "all", "All relays",
+      "Exits only")) %>%
+    ggplot(aes(x = date, y = advbw, colour = percentile)) +
     facet_grid(variable ~ .) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
     scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
       limits = c(0, NA)) +
-    scale_colour_hue(name = "Percentile",
-        breaks = rev(levels(t$percentile))) +
+    scale_colour_hue(name = "Percentile") +
     ggtitle("Advertised bandwidth distribution") +
     labs(caption = copyright_notice)
   ggsave(filename = path, width = 8, height = 5, dpi = 150)
@@ -1143,27 +1057,27 @@ write_advbwdist_perc <- function(start, end, p, path) {
 }
prepare_advbwdist_relay <- function(start, end, n) {
-  t <- read.csv(paste(stats_dir, "advbwdist.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  t <- t[t$date >= start & t$date <= end & t$relay %in% as.numeric(n), ]
-  t <- data.frame(date = t$date, relay = as.factor(t$relay),
-                  variable = ifelse(t$isexit != "t", "all", "exits"),
-                  advbw = t$advbw * 8 / 1e9)
-  t
+  read.csv(paste(stats_dir, "advbwdist.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end),
+      relay %in% as.numeric(n)) %>%
+    transmute(date, relay = as.factor(relay),
+      variable = ifelse(isexit != "t", "all", "exits"),
+      advbw = advbw * 8 / 1e9)
 }
plot_advbwdist_relay <- function(start, end, n, path) {
-  t <- prepare_advbwdist_relay(start, end, n)
-  t$variable <- ifelse(t$variable == "all", "All relays",
-                       "Exits only")
-  ggplot(t, aes(x = as.Date(date), y = advbw, colour = relay)) +
+  prepare_advbwdist_relay(start, end, n) %>%
+    mutate(variable = ifelse(variable == "all", "All relays",
+      "Exits only")) %>%
+    ggplot(aes(x = date, y = advbw, colour = relay)) +
     facet_grid(variable ~ .) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
     scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
       limits = c(0, NA)) +
-    scale_colour_hue(name = "n", breaks = levels(t$relay)) +
+    scale_colour_hue(name = "n") +
     ggtitle("Advertised bandwidth of n-th fastest relays") +
     labs(caption = copyright_notice)
   ggsave(filename = path, width = 8, height = 5, dpi = 150)
@@ -1177,17 +1091,16 @@ write_advbwdist_relay <- function(start, end, n, path) {
 }
prepare_hidserv_dir_onions_seen <- function(start, end) {
-  h <- read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  h <- h[h$date >= start & h$date <= end & h$type == "dir-onions-seen", ]
-  h <- data.frame(date = as.Date(h$date, "%Y-%m-%d"),
-                  onions = ifelse(h$frac >= 0.01, h$wiqm, NA))
-  h
+  read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end),
+      type == "dir-onions-seen") %>%
+    transmute(date = date, onions = ifelse(frac >= 0.01, wiqm, NA))
 }
plot_hidserv_dir_onions_seen <- function(start, end, path) {
-  h <- prepare_hidserv_dir_onions_seen(start, end)
-  ggplot(h, aes(x = as.Date(date, origin = "1970-01-01"), y = onions)) +
+  prepare_hidserv_dir_onions_seen(start, end) %>%
+    ggplot(aes(x = date, y = onions)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -1203,19 +1116,17 @@ write_hidserv_dir_onions_seen <- function(start, end, path) {
 }
prepare_hidserv_rend_relayed_cells <- function(start, end) {
-  h <- read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  h <- h[h$date >= start & h$date <= end &
-         h$type == "rend-relayed-cells", ]
-  h <- data.frame(date = as.Date(h$date, "%Y-%m-%d"),
-                  relayed = ifelse(h$frac >= 0.01,
-                                   h$wiqm * 8 * 512 / (86400 * 1e9), NA))
-  h
+  read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end),
+      type == "rend-relayed-cells") %>%
+    transmute(date,
+      relayed = ifelse(frac >= 0.01, wiqm * 8 * 512 / (86400 * 1e9), NA))
 }
plot_hidserv_rend_relayed_cells <- function(start, end, path) {
-  h <- prepare_hidserv_rend_relayed_cells(start, end)
-  ggplot(h, aes(x = as.Date(date, origin = "1970-01-01"), y = relayed)) +
+  prepare_hidserv_rend_relayed_cells(start, end) %>%
+    ggplot(aes(x = date, y = relayed)) +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
       labels = custom_labels, minor_breaks = custom_minor_breaks) +
@@ -1232,18 +1143,15 @@ write_hidserv_rend_relayed_cells <- function(start, end, path) {
 }
prepare_hidserv_frac_reporting <- function(start, end) {
-  h <- read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  h <- h[h$date >= start & h$date <= end, ]
-  h <- data.frame(date = as.Date(h$date, "%Y-%m-%d"),
-                  frac = h$frac, type = h$type)
-  h
+  read.csv(paste(stats_dir, "hidserv.csv", sep = ""),
+    colClasses = c("date" = "Date")) %>%
+    filter(date >= as.Date(start), date <= as.Date(end)) %>%
+    select(date, frac, type)
 }
plot_hidserv_frac_reporting <- function(start, end, path) {
-  h <- prepare_hidserv_frac_reporting(start, end)
-  ggplot(h, aes(x = as.Date(date, origin = "1970-01-01"), y = frac,
-      colour = type)) +
+  prepare_hidserv_frac_reporting(start, end) %>%
+    ggplot(aes(x = date, y = frac, colour = type)) +
     geom_line() +
     geom_hline(yintercept = 0.01, linetype = 2) +
     scale_x_date(name = "", breaks = custom_breaks,
@@ -1269,10 +1177,9 @@ write_hidserv_frac_reporting <- function(start, end, path) {
prepare_webstats_tb <- function(start, end) {
   load(paste(rdata_dir, "webstats-tb.RData", sep = ""))
-  d <- data
-  d <- d[d$log_date >= start & d$log_date <= end, ]
-  d$request_type <- factor(d$request_type)
-  d
+  data %>%
+    filter(log_date >= as.Date(start), log_date <= as.Date(end)) %>%
+    mutate(request_type = factor(request_type))
 }
plot_webstats_tb <- function(start, end, path) {
@@ -1306,17 +1213,17 @@ write_webstats_tb <- function(start, end, path) {
 }
prepare_webstats_tb_platform <- function(start, end) {
-  d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""),
-    stringsAsFactors = FALSE)
-  d <- d[d$log_date >= start & d$log_date <= end & d$request_type == 'tbid', ]
-  d <- aggregate(list(count = d$count), by = list(log_date = as.Date(d$log_date),
-    platform = d$platform), FUN = sum)
-  d
+  read.csv(paste(stats_dir, "webstats.csv", sep = ""),
+    colClasses = c("log_date" = "Date")) %>%
+    filter(log_date >= as.Date(start), log_date <= as.Date(end),
+      request_type == 'tbid') %>%
+    group_by(log_date, platform) %>%
+    summarize(count = sum(count))
 }
plot_webstats_tb_platform <- function(start, end, path) {
-  d <- prepare_webstats_tb_platform(start, end)
-  ggplot(d, aes(x = log_date, y = count, colour = platform)) +
+  prepare_webstats_tb_platform(start, end) %>%
+    ggplot(aes(x = log_date, y = count, colour = platform)) +
     geom_point() +
     geom_line() +
     scale_x_date(name = "", breaks = custom_breaks,
@@ -1342,13 +1249,13 @@ write_webstats_tb_platform <- function(start, end, path) {
plot_webstats_tb_locale <- function(start, end, path) {
   d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""),
-    stringsAsFactors = FALSE)
+    colClasses = c("log_date" = "Date"))
   d <- d[d$log_date >= start & d$log_date <= end & d$request_type == 'tbid', ]
   e <- d
   e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum)
   e <- e[order(e$count, decreasing = TRUE), ]
   e <- e[1:5, ]
-  d <- aggregate(list(count = d$count), by = list(log_date = as.Date(d$log_date),
+  d <- aggregate(list(count = d$count), by = list(log_date = d$log_date,
     locale = ifelse(d$locale %in% e$locale, d$locale, '(other)')), FUN = sum)
   ggplot(d, aes(x = log_date, y = count, colour = locale)) +
     geom_point() +
@@ -1373,13 +1280,13 @@ plot_webstats_tb_locale <- function(start, end, path) {
 # breaks and labels. Left as future work.
 write_webstats_tb_locale <- function(start, end, path) {
   d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""),
-    stringsAsFactors = FALSE)
+    colClasses = c("log_date" = "Date"))
   d <- d[d$log_date >= start & d$log_date <= end & d$request_type == 'tbid', ]
   e <- d
   e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum)
   e <- e[order(e$count, decreasing = TRUE), ]
   e <- e[1:5, ]
-  d <- aggregate(list(count = d$count), by = list(log_date = as.Date(d$log_date),
+  d <- aggregate(list(count = d$count), by = list(log_date = d$log_date,
     locale = ifelse(d$locale %in% e$locale, d$locale, 'other')), FUN = sum)
   d %>%
     mutate(locale = tolower(locale)) %>%
@@ -1390,10 +1297,9 @@ write_webstats_tb_locale <- function(start, end, path) {
prepare_webstats_tm <- function(start, end) {
   load(paste(rdata_dir, "webstats-tm.RData", sep = ""))
-  d <- data
-  d <- d[d$log_date >= start & d$log_date <= end, ]
-  d$request_type <- factor(d$request_type)
-  d
+  data %>%
+    filter(log_date >= as.Date(start), log_date <= as.Date(end)) %>%
+    mutate(request_type = factor(request_type))
 }
plot_webstats_tm <- function(start, end, path) {

    