commit 167e72b5a06cec3753f7f952fb3e3247bae943a5 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Fri May 11 11:54:45 2018 +0200 Make all parameters in write_* functions optional. We now permit parameters in write_* functions to be omitted. The effect is that we're not filtering if a parameter is missing, thus producing a CSV file with more rows. At the same time we're adding columns for data that was previously pre-determined by parameter values. For example, if a user specified a given country in a parameter, we didn't have to include a country column containing only that country. Now we need to put that column back. Implements #25383. --- src/main/R/rserver/graphs.R | 369 ++++++++++++--------- .../torproject/metrics/web/RObjectGenerator.java | 4 + 2 files changed, 222 insertions(+), 151 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index ebb8c80..a9b7fc7 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -351,8 +351,13 @@ robust_call <- function(wrappee, filename) { prepare_networksize <- function(start, end) { read.csv(paste(stats_dir, "servers.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), flag == "", - country == "", version == "", platform == "", ec2bridge == "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(flag == "") %>% + filter(country == "") %>% + filter(version == "") %>% + filter(platform == "") %>% + filter(ec2bridge == "") %>% select(date, relays, bridges) } @@ -373,16 +378,21 @@ plot_networksize <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_networksize <- function(start, end, path) { +write_networksize <- function(start = NULL, end = NULL, path) { prepare_networksize(start, end) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_versions <- function(start, end) { read.csv(paste(stats_dir, "servers.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), flag == "", - country == "", version != "", platform == "", ec2bridge == "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(flag == "") %>% + filter(country == "") %>% + filter(version != "") %>% + filter(platform == "") %>% + filter(ec2bridge == "") %>% select(date, version, relays) } @@ -411,17 +421,22 @@ plot_versions <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_versions <- function(start, end, path) { +write_versions <- function(start = NULL, end = NULL, path) { prepare_versions(start, end) %>% spread(key = "version", value = "relays", fill = 0) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_platforms <- function(start, end) { read.csv(paste(stats_dir, "servers.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), flag == "", - country == "", version == "", platform != "", ec2bridge == "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(flag == "") %>% + filter(country == "") %>% + filter(version == "") %>% + filter(platform != "") %>% + filter(ec2bridge == "") %>% select(date, platform, relays) %>% mutate(platform = ifelse(platform == "Darwin", "macOS", as.character(platform))) @@ -442,17 +457,19 @@ plot_platforms <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_platforms <- function(start, end, path) { +write_platforms <- function(start = NULL, end = NULL, path) { prepare_platforms(start, end) %>% spread(platform, relays) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_bandwidth <- function(start, end) { read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), isexit != "", - isguard != "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(isexit != "") %>% + filter(isguard != "") %>% group_by(date) %>% summarize(advbw = sum(advbw) * 8 / 1e9, bwhist = sum(bwread + bwwrite) * 8 / 2e9) %>% @@ -477,16 +494,18 @@ plot_bandwidth <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_bandwidth <- function(start, end, path) { +write_bandwidth <- function(start = NULL, end = NULL, path) { prepare_bandwidth(start, end) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_bwhist_flags <- function(start, end) { read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), isexit != "", - isguard != "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(isexit != "") %>% + filter(isguard != "") %>% mutate(variable = ifelse(isexit == "t", ifelse(isguard == "t", "guard_and_exit", "exit_only"), ifelse(isguard == "t", "guard_only", "middle_only")), @@ -514,17 +533,19 @@ plot_bwhist_flags <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_bwhist_flags <- function(start, end, path) { +write_bwhist_flags <- function(start = NULL, end = NULL, path) { prepare_bwhist_flags(start, end) %>% spread(variable, value) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_dirbytes <- function(start, end, path) { read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), isexit == "", - isguard == "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(isexit == "") %>% + filter(isguard == "") %>% mutate(dirread = dirread * 8 / 1e9, dirwrite = dirwrite * 8 / 1e9) %>% select(date, dirread, dirwrite) @@ -548,18 +569,22 @@ plot_dirbytes <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_dirbytes <- function(start, end, path) { +write_dirbytes <- function(start = NULL, end = NULL, path) { prepare_dirbytes(start, end) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_relayflags <- function(start, end, flags) { read.csv(paste(stats_dir, "servers.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), country == "", - version == "", platform == "", ec2bridge == "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(country == "") %>% + filter(version == "") %>% + filter(platform == "") %>% + filter(ec2bridge == "") %>% mutate(flag = ifelse(flag == "", "Running", as.character(flag))) %>% - filter(flag %in% flags) %>% + filter(if (!is.null(flags)) flag %in% flags else TRUE) %>% select(date, flag, relays) } @@ -579,11 +604,11 @@ plot_relayflags <- function(start, end, flags, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_relayflags <- function(start, end, flags, path) { +write_relayflags <- function(start = NULL, end = NULL, flags = NULL, path) { prepare_relayflags(start, end, flags) %>% mutate(flag = tolower(flag)) %>% spread(flag, relays) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } plot_torperf <- function(start, end, source, server, filesize, path) { @@ -629,28 +654,39 @@ plot_torperf <- function(start, end, source, server, filesize, path) { # harder than for other functions, because plot_torperf uses different # colours based on which sources exist, unrelated to which source is # plotted. Left as future work. -write_torperf <- function(start, end, source, server, filesize, path) { +write_torperf <- function(start = NULL, end = NULL, source = NULL, + server = NULL, filesize = NULL, path) { read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), - filesize == ifelse(!!filesize == "50kb", 50 * 1024, - ifelse(!!filesize == "1mb", 1024 * 1024, 5 * 1024 * 1024)), - source == ifelse(!!source == "all", "", !!source), - server == !!server) %>% - transmute(date, q1 = q1 / 1e3, md = md / 1e3, q3 = q3 / 1e3) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(!!source)) + source == ifelse(!!source == "all", "", !!source) else TRUE) %>% + filter(if (!is.null(!!server)) server == !!server else TRUE) %>% + filter(if (!is.null(!!filesize)) + filesize == ifelse(!!filesize == "50kb", 50 * 1024, + ifelse(!!filesize == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else + TRUE) %>% + transmute(date, filesize, source, server, q1 = q1 / 1e3, md = md / 1e3, + q3 = q3 / 1e3) %>% + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_torperf_failures <- function(start, end, source, server, filesize) { - filesize_val <- ifelse(filesize == "50kb", 50 * 1024, - ifelse(filesize == "1mb", 1024 * 1024, 5 * 1024 * 1024)) - t <- read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""), - colClasses = c("date" = "Date")) - t[t$date >= start & t$date <= end & t$filesize == filesize_val & - t$source == ifelse(source == "all", "", source) & - t$server == server & t$requests > 0, ] %>% - transmute(date, timeouts = timeouts / requests, - failures = failures / requests) + read.csv(paste(stats_dir, "torperf-1.1.csv", sep = ""), + colClasses = c("date" = "Date")) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(!!filesize)) + filesize == ifelse(!!filesize == "50kb", 50 * 1024, + ifelse(!!filesize == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else + TRUE) %>% + filter(if (!is.null(!!source)) + source == ifelse(!!source == "all", "", !!source) else TRUE) %>% + filter(if (!is.null(!!server)) server == !!server else TRUE) %>% + filter(requests > 0) %>% + transmute(date, filesize, source, server, timeouts = timeouts / requests, + failures = failures / requests) } plot_torperf_failures <- function(start, end, source, server, filesize, path) { @@ -675,15 +711,17 @@ plot_torperf_failures <- function(start, end, source, server, filesize, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_torperf_failures <- function(start, end, source, server, filesize, path) { +write_torperf_failures <- function(start = NULL, end = NULL, source = NULL, + server = NULL, filesize = NULL, path) { prepare_torperf_failures(start, end, source, server, filesize) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_connbidirect <- function(start, end) { read.csv(paste(stats_dir, "connbidirect2.csv", sep = ""), colClasses = c("date" = "Date", "direction" = "factor")) %>% - filter(date >= as.Date(start), date <= as.Date(end)) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% mutate(quantile = paste("X", quantile, sep = ""), fraction = fraction / 100) %>% spread(quantile, fraction) @@ -712,20 +750,23 @@ plot_connbidirect <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_connbidirect <- function(start, end, path) { +write_connbidirect <- function(start = NULL, end = NULL, path) { prepare_connbidirect(start, end) %>% rename(q1 = X0.25, md = X0.5, q3 = X0.75) %>% gather(variable, value, -(date:direction)) %>% unite(temp, direction, variable) %>% spread(temp, value) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_bandwidth_flags <- function(start, end) { b <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) - b <- b[b$date >= start & b$date <= end & b$isexit != "" & - b$isguard != "", ] + b <- b %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(isexit != "") %>% + filter(isguard != "") b <- data.frame(date = b$date, isexit = b$isexit == "t", isguard = b$isguard == "t", advbw = b$advbw * 8 / 1e9, @@ -770,10 +811,10 @@ plot_bandwidth_flags <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_bandwidth_flags <- function(start, end, path) { +write_bandwidth_flags <- function(start = NULL, end = NULL, path) { prepare_bandwidth_flags(start, end) %>% spread(variable, value) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } plot_userstats <- function(start, end, node, variable, value, events, @@ -921,48 +962,48 @@ plot_userstats_bridge_version <- function(start, end, version, path) { plot_userstats(start, end, "bridge", "version", version, "off", path) } -write_userstats_relay_country <- function(start, end, country, events, - path) { +write_userstats_relay_country <- function(start = NULL, end = NULL, + country = NULL, events = NULL, path) { load(paste(rdata_dir, "clients-relay.RData", sep = "")) u <- data %>% - filter(date >= as.Date(start), date <= as.Date(end), - country == ifelse(!!country == "all", "", !!country), transport == "", - version == "") - if (country != "all" && events == "on") { - u <- u %>% - mutate(downturns = clients < u$lower, upturns = clients > upper) %>% - select(date, clients, downturns, upturns, lower, upper) - } else if (country != "all" && events != "off") { - u <- u %>% - mutate(downturns = clients < u$lower, upturns = clients > upper) %>% - select(date, clients, downturns, upturns) - } else { - u <- u %>% - select(date, clients) - } - u %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(!!country)) + country == ifelse(!!country == "all", "", !!country) else TRUE) %>% + filter(transport == "") %>% + filter(version == "") %>% + mutate(downturns = clients < lower, upturns = clients > upper) %>% + select(date, country, clients, downturns, upturns, lower, upper) %>% rename(users = clients) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } -write_userstats_bridge_country <- function(start, end, country, path) { +write_userstats_bridge_country <- function(start = NULL, end = NULL, + country = NULL, path) { load(paste(rdata_dir, "clients-bridge.RData", sep = "")) data %>% - filter(date >= as.Date(start), date <= as.Date(end), - country == ifelse(!!country == "all", "", !!country), transport == "", - version == "") %>% - select(date, clients) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(!!country)) + country == ifelse(!!country == "all", "", !!country) else TRUE) %>% + filter(transport == "") %>% + filter(version == "") %>% + select(date, country, clients) %>% rename(users = clients) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } -write_userstats_bridge_transport <- function(start, end, transports, path) { +write_userstats_bridge_transport <- function(start = NULL, end = NULL, + transports = NULL, path) { load(paste(rdata_dir, "clients-bridge.RData", sep = "")) u <- data %>% - filter(date >= as.Date(start), date <= as.Date(end), - country == "", version == "", transport != "") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(country == "") %>% + filter(version == "") %>% + filter(transport != "") %>% select(date, transport, clients) - if ("!<OR>" %in% transports) { + if (is.null(transports) || "!<OR>" %in% transports) { n <- u %>% filter(transport != "<OR>") %>% group_by(date) %>% @@ -971,7 +1012,7 @@ write_userstats_bridge_transport <- function(start, end, transports, path) { clients = n$clients)) } u %>% - filter(transport %in% transports) %>% + filter(if (!is.null(transports)) transport %in% transports else TRUE) %>% mutate(transport = ifelse(transport == "<OR>", "default_or_protocol", ifelse(transport == "!<OR>", "any_pt", ifelse(transport == "<??>", "unknown_pluggable_transports", @@ -979,38 +1020,41 @@ write_userstats_bridge_transport <- function(start, end, transports, path) { group_by(date, transport) %>% select(date, transport, clients) %>% spread(transport, clients) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } -write_userstats_bridge_version <- function(start, end, version, path) { +write_userstats_bridge_version <- function(start = NULL, end = NULL, + version = NULL, path) { load(paste(rdata_dir, "clients-bridge.RData", sep = "")) data %>% - filter(date >= as.Date(start), date <= as.Date(end), - country == "", transport == "", version == !!version) %>% - select(date, clients) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(country == "") %>% + filter(transport == "") %>% + filter(if (!is.null(!!version)) version == !!version else TRUE) %>% + select(date, version, clients) %>% rename(users = clients) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_userstats_bridge_combined <- function(start, end, country) { - top <- 3 - country <- ifelse(country == "all", NA, country) load(paste(rdata_dir, "userstats-bridge-combined.RData", sep = "")) - u <- data - u <- u[u$date >= start & u$date <= end - & (is.na(country) | u$country == country), ] - a <- aggregate(list(mid = (u$high + u$low) / 2), - by = list(transport = u$transport), FUN = sum) - a <- a[order(a$mid, decreasing = TRUE)[1:top], ] - u <- u[u$transport %in% a$transport, ] - u + data %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(!!country)) country == !!country else TRUE) } plot_userstats_bridge_combined <- function(start, end, country, path) { if (country == "all") { plot_userstats_bridge_country(start, end, country, path) } else { + top <- 3 u <- prepare_userstats_bridge_combined(start, end, country) + a <- aggregate(list(mid = (u$high + u$low) / 2), + by = list(transport = u$transport), FUN = sum) + a <- a[order(a$mid, decreasing = TRUE)[1:top], ] + u <- u[u$transport %in% a$transport, ] title <- paste("Bridge users by transport from ", countryname(country), sep = "") ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high, @@ -1028,26 +1072,29 @@ plot_userstats_bridge_combined <- function(start, end, country, path) { } } -write_userstats_bridge_combined <- function(start, end, country, path) { - if (country == "all") { +write_userstats_bridge_combined <- function(start = NULL, end = NULL, + country = NULL, path) { + if (!is.null(country) && country == "all") { write_userstats_bridge_country(start, end, country, path) } else { prepare_userstats_bridge_combined(start, end, country) %>% - select(date, transport, low, high) %>% - mutate(transport = ifelse(transport == "<OR>", - "default_or_protocol", transport)) %>% + select(date, country, transport, low, high) %>% + mutate(transport = ifelse(transport == "<OR>", "default_or_protocol", + ifelse(transport == "<??>", "unknown_transport", transport))) %>% gather(variable, value, -(date:transport)) %>% unite(temp, transport, variable) %>% spread(temp, value) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } } prepare_advbwdist_perc <- function(start, end, p) { read.csv(paste(stats_dir, "advbwdist.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), - percentile %in% as.numeric(p)) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(p)) percentile %in% as.numeric(p) else + percentile != "") %>% transmute(date, percentile = as.factor(percentile), variable = ifelse(isexit != "t", "all", "exits"), advbw = advbw * 8 / 1e9) @@ -1070,18 +1117,20 @@ plot_advbwdist_perc <- function(start, end, p, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_advbwdist_perc <- function(start, end, p, path) { +write_advbwdist_perc <- function(start = NULL, end = NULL, p = NULL, path) { prepare_advbwdist_perc(start, end, p) %>% unite(temp, variable, percentile) %>% spread(temp, advbw) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_advbwdist_relay <- function(start, end, n) { read.csv(paste(stats_dir, "advbwdist.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), - relay %in% as.numeric(n)) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(if (!is.null(n)) relay %in% as.numeric(n) else + relay != "") %>% transmute(date, relay = as.factor(relay), variable = ifelse(isexit != "t", "all", "exits"), advbw = advbw * 8 / 1e9) @@ -1104,18 +1153,19 @@ plot_advbwdist_relay <- function(start, end, n, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_advbwdist_relay <- function(start, end, n, path) { +write_advbwdist_relay <- function(start = NULL, end = NULL, n = NULL, path) { prepare_advbwdist_relay(start, end, n) %>% unite(temp, variable, relay) %>% spread(temp, advbw) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_hidserv_dir_onions_seen <- function(start, end) { read.csv(paste(stats_dir, "hidserv.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), - type == "dir-onions-seen") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(type == "dir-onions-seen") %>% transmute(date = date, onions = ifelse(frac >= 0.01, wiqm, NA)) } @@ -1131,16 +1181,17 @@ plot_hidserv_dir_onions_seen <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_hidserv_dir_onions_seen <- function(start, end, path) { +write_hidserv_dir_onions_seen <- function(start = NULL, end = NULL, path) { prepare_hidserv_dir_onions_seen(start, end) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_hidserv_rend_relayed_cells <- function(start, end) { read.csv(paste(stats_dir, "hidserv.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end), - type == "rend-relayed-cells") %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% + filter(type == "rend-relayed-cells") %>% transmute(date, relayed = ifelse(frac >= 0.01, wiqm * 8 * 512 / (86400 * 1e9), NA)) } @@ -1158,15 +1209,16 @@ plot_hidserv_rend_relayed_cells <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_hidserv_rend_relayed_cells <- function(start, end, path) { +write_hidserv_rend_relayed_cells <- function(start = NULL, end = NULL, path) { prepare_hidserv_rend_relayed_cells(start, end) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_hidserv_frac_reporting <- function(start, end) { read.csv(paste(stats_dir, "hidserv.csv", sep = ""), colClasses = c("date" = "Date")) %>% - filter(date >= as.Date(start), date <= as.Date(end)) %>% + filter(if (!is.null(start)) date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) date <= as.Date(end) else TRUE) %>% select(date, frac, type) } @@ -1189,17 +1241,18 @@ plot_hidserv_frac_reporting <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_hidserv_frac_reporting <- function(start, end, path) { +write_hidserv_frac_reporting <- function(start = NULL, end = NULL, path) { prepare_hidserv_frac_reporting(start, end) %>% mutate(type = ifelse(type == "dir-onions-seen", "onions", "relayed")) %>% spread(type, frac) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_webstats_tb <- function(start, end) { load(paste(rdata_dir, "webstats-tb.RData", sep = "")) data %>% - filter(log_date >= as.Date(start), log_date <= as.Date(end)) %>% + filter(if (!is.null(start)) log_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) log_date <= as.Date(end) else TRUE) %>% mutate(request_type = factor(request_type)) } @@ -1224,20 +1277,21 @@ plot_webstats_tb <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_webstats_tb <- function(start, end, path) { +write_webstats_tb <- function(start = NULL, end = NULL, path) { prepare_webstats_tb(start, end) %>% rename(date = log_date) %>% spread(request_type, count) %>% rename(initial_downloads = tbid, signature_downloads = tbsd, update_pings = tbup, update_requests = tbur) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_webstats_tb_platform <- function(start, end) { read.csv(paste(stats_dir, "webstats.csv", sep = ""), colClasses = c("log_date" = "Date")) %>% - filter(log_date >= as.Date(start), log_date <= as.Date(end), - request_type == "tbid") %>% + filter(if (!is.null(start)) log_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) log_date <= as.Date(end) else TRUE) %>% + filter(request_type == "tbid") %>% group_by(log_date, platform) %>% summarize(count = sum(count)) } @@ -1260,12 +1314,12 @@ plot_webstats_tb_platform <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_webstats_tb_platform <- function(start, end, path) { +write_webstats_tb_platform <- function(start = NULL, end = NULL, path) { prepare_webstats_tb_platform(start, end) %>% rename(date = log_date) %>% spread(platform, count) %>% rename(linux = l, macos = m, windows = w) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } plot_webstats_tb_locale <- function(start, end, path) { @@ -1299,10 +1353,13 @@ plot_webstats_tb_locale <- function(start, end, path) { # turned out to be a bit harder than for other functions, because # plot_webstats_tb_locale needs the preliminary data frame e for its # breaks and labels. Left as future work. -write_webstats_tb_locale <- function(start, end, path) { +write_webstats_tb_locale <- function(start = NULL, end = NULL, path) { d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""), colClasses = c("log_date" = "Date", "locale" = "character")) - d <- d[d$log_date >= start & d$log_date <= end & d$request_type == "tbid", ] + d <- d %>% + filter(if (!is.null(start)) log_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) log_date <= as.Date(end) else TRUE) %>% + filter(request_type == "tbid") e <- d e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum) e <- e[order(e$count, decreasing = TRUE), ] @@ -1313,13 +1370,14 @@ write_webstats_tb_locale <- function(start, end, path) { mutate(locale = tolower(locale)) %>% rename(date = log_date) %>% spread(locale, count) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_webstats_tm <- function(start, end) { load(paste(rdata_dir, "webstats-tm.RData", sep = "")) data %>% - filter(log_date >= as.Date(start), log_date <= as.Date(end)) %>% + filter(if (!is.null(start)) log_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) log_date <= as.Date(end) else TRUE) %>% mutate(request_type = factor(request_type)) } @@ -1342,19 +1400,22 @@ plot_webstats_tm <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_webstats_tm <- function(start, end, path) { +write_webstats_tm <- function(start = NULL, end = NULL, path) { prepare_webstats_tm(start, end) %>% rename(date = log_date) %>% spread(request_type, count) %>% rename(initial_downloads = tmid, update_pings = tmup) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_relays_ipv6 <- function(start, end) { read.csv(paste(stats_dir, "ipv6servers.csv", sep = ""), colClasses = c("valid_after_date" = "Date")) %>% - filter(valid_after_date >= as.Date(start), - valid_after_date <= as.Date(end), server == "relay") %>% + filter(if (!is.null(start)) + valid_after_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) + valid_after_date <= as.Date(end) else TRUE) %>% + filter(server == "relay") %>% group_by(valid_after_date) %>% summarize(total = sum(server_count_sum_avg), announced = sum(server_count_sum_avg[announced_ipv6 == "t"]), @@ -1382,18 +1443,21 @@ plot_relays_ipv6 <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_relays_ipv6 <- function(start, end, path) { +write_relays_ipv6 <- function(start = NULL, end = NULL, path) { prepare_relays_ipv6(start, end) %>% rename(date = valid_after_date) %>% spread(category, count) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_bridges_ipv6 <- function(start, end) { read.csv(paste(stats_dir, "ipv6servers.csv", sep = ""), colClasses = c("valid_after_date" = "Date")) %>% - filter(valid_after_date >= as.Date(start), - valid_after_date <= as.Date(end), server == "bridge") %>% + filter(if (!is.null(start)) + valid_after_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) + valid_after_date <= as.Date(end) else TRUE) %>% + filter(server == "bridge") %>% group_by(valid_after_date) %>% summarize(total = sum(server_count_sum_avg), announced = sum(server_count_sum_avg[announced_ipv6 == "t"])) %>% @@ -1417,18 +1481,21 @@ plot_bridges_ipv6 <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_bridges_ipv6 <- function(start, end, path) { +write_bridges_ipv6 <- function(start = NULL, end = NULL, path) { prepare_bridges_ipv6(start, end) %>% rename(date = valid_after_date) %>% spread(category, count) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } prepare_advbw_ipv6 <- function(start, end) { read.csv(paste(stats_dir, "ipv6servers.csv", sep = ""), colClasses = c("valid_after_date" = "Date")) %>% - filter(valid_after_date >= as.Date(start), - valid_after_date <= as.Date(end), server == "relay") %>% + filter(if (!is.null(start)) + valid_after_date >= as.Date(start) else TRUE) %>% + filter(if (!is.null(end)) + valid_after_date <= as.Date(end) else TRUE) %>% + filter(server == "relay") %>% group_by(valid_after_date) %>% summarize(total = sum(advertised_bandwidth_bytes_sum_avg), total_guard = sum(advertised_bandwidth_bytes_sum_avg[guard_relay != "f"]), @@ -1465,10 +1532,10 @@ plot_advbw_ipv6 <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } -write_advbw_ipv6 <- function(start, end, path) { +write_advbw_ipv6 <- function(start = NULL, end = NULL, path) { prepare_advbw_ipv6(start, end) %>% rename(date = valid_after_date) %>% spread(category, advbw) %>% - write.csv(path, quote = FALSE, row.names = FALSE) + write.csv(path, quote = FALSE, row.names = FALSE, na = "") } diff --git a/src/main/java/org/torproject/metrics/web/RObjectGenerator.java b/src/main/java/org/torproject/metrics/web/RObjectGenerator.java index aea6db7..00fcc81 100644 --- a/src/main/java/org/torproject/metrics/web/RObjectGenerator.java +++ b/src/main/java/org/torproject/metrics/web/RObjectGenerator.java @@ -126,6 +126,10 @@ public class RObjectGenerator implements ServletContextListener { queryBuilder.append("robust_call(as.call(list("); if ("csv".equalsIgnoreCase(fileType)) { queryBuilder.append("write_"); + /* When we checked parameters above we also put in defaults for missing + * parameters. This is okay for graphs, but we want to support CSV files + * with empty parameters. Using the parameters we got here. */ + checkedParameters = parameterMap; } else { queryBuilder.append("plot_"); }