[tor-commits] [metrics-web/release] Use readr to speed up drawing graphs.

9 Nov 2019

commit 2c44721c9ab903183558b92d7a4e17674fcb79be
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Mon Dec 17 21:03:16 2018 +0100
Use readr to speed up drawing graphs.
Over two years ago, in commit 1f90b72 from October 2016, we made our
    user graphs faster by avoiding to read the large .csv file on demand.
    Instead we read it once as part of the daily update, saved it to disk
    as .RData file using R's save() function, and loaded it back to memory
    using R's load() function when drawing a graph.
This approach worked okay. It just had two disadvantages:
1. We had to write a small amount of R code for each graph type,
        which is why we only did it for graphs with large .csv files.
     2. Running these small R script as part of the daily update made it
        harder to move away from Ant towards a Java-only execution model.
The new approach implemented in this commit uses read_csv() fromt the
    readr package which reads CSV files several times faster than
    read.csv().
Requires installing the readr package from CRAN, which is available on
    Debian in stretch-backports and later as r-cran-readr.
Implements #28799.
---
 build.xml                          |  14 ---
 src/main/R/clients/split-clients.R |  12 ---
 src/main/R/rserver/graphs.R        | 169 +++++++++++++++++++++++++++++--------
 src/main/R/rserver/rserve-init.R   |   1 +
 src/main/R/webstats/write-RData.R  |  16 ----
 5 files changed, 136 insertions(+), 76 deletions(-)

diff --git a/build.xml b/build.xml
index 89c8b31..250417e 100644
--- a/build.xml
+++ b/build.xml
@@ -362,8 +362,6 @@
     <property name="module.name" value="clients" />
     <property name="localmoddir" value="${modulebase}/${module.name}" />
-    <property name="rdatadir" value="${localmoddir}/RData" />
-    <mkdir dir="${rdatadir}" />
     <property name="statsdir"
               value="${localmoddir}/stats" />
     <mkdir dir="${statsdir}" />
@@ -410,10 +408,6 @@
<copy file="${localmoddir}/clients.csv" todir="${statsdir}" />
     <copy file="${localmoddir}/userstats-combined.csv" todir="${statsdir}" />
-
-    <antcall target="run-R" >
-      <param name="module.Rscript" value="split-clients.R" />
-    </antcall>
   </target>
<target name="servers" >
@@ -426,13 +420,7 @@
<target name="webstats" >
     <property name="module.name" value="webstats" />
-    <property name="rdatadir" value="${modulebase}/${module.name}/RData" />
-    <mkdir dir="${rdatadir}" />
-
     <antcall target="run-java" />
-    <antcall target="run-R" >
-      <param name="module.Rscript" value="write-RData.R" />
-    </antcall>
   </target>
<target name="totalcw" >
@@ -482,8 +470,6 @@
       <fileset dir="${modulebase}/totalcw/stats" includes="totalcw.csv" />
     </copy>
     <copy todir="${rdatadir}" >
-      <fileset dir="${modulebase}/clients/RData" includes="*.RData" />
-      <fileset dir="${modulebase}/webstats/RData" includes="*.RData" />
       <fileset dir="${resources}/web/images/" includes="no-data-available.*" />
     </copy>
   </target>
diff --git a/src/main/R/clients/split-clients.R b/src/main/R/clients/split-clients.R
deleted file mode 100644
index 9f80902..0000000
--- a/src/main/R/clients/split-clients.R
+++ /dev/null
@@ -1,12 +0,0 @@
-dir.create("RData", showWarnings = FALSE)
-
-c <- read.csv("clients.csv", stringsAsFactors = FALSE)
-data <- c[c$node == 'relay', !(names(c) %in% c("node"))]
-save(data, file = "RData/clients-relay.RData")
-data <- c[c$node == 'bridge', !(names(c) %in% c("node"))]
-save(data, file = "RData/clients-bridge.RData")
-
-u <- read.csv("userstats-combined.csv", stringsAsFactors = FALSE)
-data <- u[, !(names(u) %in% c("node", "version"))]
-save(data, file = "RData/userstats-bridge-combined.RData")
-
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index 7501a95..e541c30 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -348,6 +348,9 @@ robust_call <- function(wrappee, filename) {
        })
 }
+# Disable readr's automatic progress bar.
+options(readr.show_progress = FALSE)
+
 prepare_networksize <- function(start_p, end_p) {
   read.csv(paste(stats_dir, "networksize.csv", sep = ""),
     colClasses = c("date" = "Date")) %>%
@@ -863,8 +866,19 @@ write_bandwidth_flags <- function(start_p = NULL, end_p = NULL, path_p) {
plot_userstats <- function(start_p, end_p, node_p, variable_p, value_p,
     events_p, path_p) {
-  load(paste(rdata_dir, "clients-", node_p, ".RData", sep = ""))
-  c <- data
+  c <- read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
+      col_types = cols(
+        date = col_date(format = ""),
+        node = col_character(),
+        country = col_character(),
+        transport = col_character(),
+        version = col_character(),
+        lower = col_double(),
+        upper = col_double(),
+        clients = col_double(),
+        frac = col_skip()),
+      na = character()) %>%
+    filter(node == node_p)
   u <- c[c$date >= start_p & c$date <= end_p, c("date", "country", "transport",
       "version", "lower", "upper", "clients")]
   u <- rbind(u, data.frame(date = start_p,
@@ -1011,14 +1025,24 @@ plot_userstats_bridge_version <- function(start_p, end_p, version_p, path_p) {
write_userstats_relay_country <- function(start_p = NULL, end_p = NULL,
     country_p = NULL, events_p = NULL, path_p) {
-  load(paste(rdata_dir, "clients-relay.RData", sep = ""))
-  u <- data %>%
+  read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
+      col_types = cols(
+        date = col_date(format = ""),
+        node = col_character(),
+        country = col_character(),
+        transport = col_character(),
+        version = col_character(),
+        lower = col_double(),
+        upper = col_double(),
+        clients = col_double(),
+        frac = col_double())) %>%
+    filter(node == "relay") %>%
     filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
     filter(if (!is.null(country_p))
       country == ifelse(country_p == "all", "", country_p) else TRUE) %>%
-    filter(transport == "") %>%
-    filter(version == "") %>%
+    filter(is.na(transport)) %>%
+    filter(is.na(version)) %>%
     select(date, country, clients, lower, upper, frac) %>%
     rename(users = clients) %>%
     write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
@@ -1026,14 +1050,24 @@ write_userstats_relay_country <- function(start_p = NULL, end_p = NULL,
write_userstats_bridge_country <- function(start_p = NULL, end_p = NULL,
     country_p = NULL, path_p) {
-  load(paste(rdata_dir, "clients-bridge.RData", sep = ""))
-  data %>%
+  read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
+      col_types = cols(
+        date = col_date(format = ""),
+        node = col_character(),
+        country = col_character(),
+        transport = col_character(),
+        version = col_character(),
+        lower = col_double(),
+        upper = col_double(),
+        clients = col_double(),
+        frac = col_double())) %>%
+    filter(node == "bridge") %>%
     filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
     filter(if (!is.null(country_p))
       country == ifelse(country_p == "all", "", country_p) else TRUE) %>%
-    filter(transport == "") %>%
-    filter(version == "") %>%
+    filter(is.na(transport)) %>%
+    filter(is.na(version)) %>%
     select(date, country, clients, frac) %>%
     rename(users = clients) %>%
     write.csv(path_p, quote = FALSE, row.names = FALSE, na = "")
@@ -1041,13 +1075,23 @@ write_userstats_bridge_country <- function(start_p = NULL, end_p = NULL,
write_userstats_bridge_transport <- function(start_p = NULL, end_p = NULL,
     transport_p = NULL, path_p) {
-  load(paste(rdata_dir, "clients-bridge.RData", sep = ""))
-  u <- data %>%
+  u <- read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
+      col_types = cols(
+        date = col_date(format = ""),
+        node = col_character(),
+        country = col_character(),
+        transport = col_character(),
+        version = col_character(),
+        lower = col_double(),
+        upper = col_double(),
+        clients = col_double(),
+        frac = col_double())) %>%
+    filter(node == "bridge") %>%
     filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-    filter(country == "") %>%
-    filter(version == "") %>%
-    filter(transport != "") %>%
+    filter(is.na(country)) %>%
+    filter(is.na(version)) %>%
+    filter(!is.na(transport)) %>%
     select(date, transport, clients, frac)
   if (is.null(transport_p) || "!<OR>" %in% transport_p) {
     n <- u %>%
@@ -1068,12 +1112,22 @@ write_userstats_bridge_transport <- function(start_p = NULL, end_p = NULL,
write_userstats_bridge_version <- function(start_p = NULL, end_p = NULL,
     version_p = NULL, path_p) {
-  load(paste(rdata_dir, "clients-bridge.RData", sep = ""))
-  data %>%
+  read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
+      col_types = cols(
+        date = col_date(format = ""),
+        node = col_character(),
+        country = col_character(),
+        transport = col_character(),
+        version = col_character(),
+        lower = col_double(),
+        upper = col_double(),
+        clients = col_double(),
+        frac = col_double())) %>%
+    filter(node == "bridge") %>%
     filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-    filter(country == "") %>%
-    filter(transport == "") %>%
+    filter(is.na(country)) %>%
+    filter(is.na(transport)) %>%
     filter(if (!is.null(version_p)) version == version_p else TRUE) %>%
     select(date, version, clients, frac) %>%
     rename(users = clients) %>%
@@ -1081,8 +1135,16 @@ write_userstats_bridge_version <- function(start_p = NULL, end_p = NULL,
 }
prepare_userstats_bridge_combined <- function(start_p, end_p, country_p) {
-  load(paste(rdata_dir, "userstats-bridge-combined.RData", sep = ""))
-  data %>%
+  read_csv(file = paste(stats_dir, "userstats-combined.csv", sep = ""),
+      col_types = cols(
+        date = col_date(format = ""),
+        node = col_skip(),
+        country = col_character(),
+        transport = col_character(),
+        version = col_skip(),
+        frac = col_double(),
+        low = col_double(),
+        high = col_double())) %>%
     filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
     filter(if (!is.null(country_p)) country == country_p else TRUE)
@@ -1135,7 +1197,7 @@ prepare_advbwdist_perc <- function(start_p, end_p, p_p) {
     filter(if (!is.null(p_p)) percentile %in% as.numeric(p_p) else
       percentile != "") %>%
     transmute(date, percentile = as.factor(percentile),
-      variable = ifelse(isexit != "t", "all", "exits"),
+      variable = ifelse(is.na(isexit), "all", "exits"),
       advbw = advbw * 8 / 1e9)
 }
@@ -1258,11 +1320,20 @@ write_hidserv_rend_relayed_cells <- function(start_p = NULL, end_p = NULL,
 }
prepare_webstats_tb <- function(start_p, end_p) {
-  load(paste(rdata_dir, "webstats-tb.RData", sep = ""))
-  data %>%
+  read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+      col_types = cols(
+        log_date = col_date(format = ""),
+        request_type = col_factor(),
+        platform = col_skip(),
+        channel = col_skip(),
+        locale = col_skip(),
+        incremental = col_skip(),
+        count = col_double())) %>%
     filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
-    mutate(request_type = factor(request_type))
+    filter(request_type %in% c("tbid", "tbsd", "tbup", "tbur")) %>%
+    group_by(log_date, request_type) %>%
+    summarize(count = sum(count))
 }
plot_webstats_tb <- function(start_p, end_p, path_p) {
@@ -1296,8 +1367,15 @@ write_webstats_tb <- function(start_p = NULL, end_p = NULL, path_p) {
 }
prepare_webstats_tb_platform <- function(start_p, end_p) {
-  read.csv(paste(stats_dir, "webstats.csv", sep = ""),
-    colClasses = c("log_date" = "Date")) %>%
+  read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+      col_types = cols(
+        log_date = col_date(format = ""),
+        request_type = col_factor(),
+        platform = col_factor(),
+        channel = col_skip(),
+        locale = col_skip(),
+        incremental = col_skip(),
+        count = col_double())) %>%
     filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
     filter(request_type %in% c("tbid", "tbup")) %>%
@@ -1337,8 +1415,15 @@ write_webstats_tb_platform <- function(start_p = NULL, end_p = NULL, path_p) {
 }
plot_webstats_tb_locale <- function(start_p, end_p, path_p) {
-  d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""),
-    colClasses = c("log_date" = "Date", "locale" = "character"))
+  d <- read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+      col_types = cols(
+        log_date = col_date(format = ""),
+        request_type = col_factor(),
+        platform = col_skip(),
+        channel = col_skip(),
+        locale = col_factor(),
+        incremental = col_skip(),
+        count = col_double()))
   d <- d[d$log_date >= start_p & d$log_date <= end_p &
          d$request_type %in% c("tbid", "tbup"), ]
   levels(d$request_type) <- list(
@@ -1375,8 +1460,15 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) {
 # plot_webstats_tb_locale needs the preliminary data frame e for its
 # breaks and labels. Left as future work.
 write_webstats_tb_locale <- function(start_p = NULL, end_p = NULL, path_p) {
-  read.csv(paste(stats_dir, "webstats.csv", sep = ""),
-    colClasses = c("log_date" = "Date", "locale" = "character")) %>%
+  read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+      col_types = cols(
+        log_date = col_date(format = ""),
+        request_type = col_factor(),
+        platform = col_skip(),
+        channel = col_skip(),
+        locale = col_factor(),
+        incremental = col_skip(),
+        count = col_double())) %>%
     filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
     filter(request_type %in% c("tbid", "tbup")) %>%
@@ -1390,11 +1482,20 @@ write_webstats_tb_locale <- function(start_p = NULL, end_p = NULL, path_p) {
 }
prepare_webstats_tm <- function(start_p, end_p) {
-  load(paste(rdata_dir, "webstats-tm.RData", sep = ""))
-  data %>%
+  read_csv(file = paste(stats_dir, "webstats.csv", sep = ""),
+      col_types = cols(
+        log_date = col_date(format = ""),
+        request_type = col_factor(),
+        platform = col_skip(),
+        channel = col_skip(),
+        locale = col_skip(),
+        incremental = col_skip(),
+        count = col_double())) %>%
     filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>%
-    mutate(request_type = factor(request_type))
+    filter(request_type %in% c("tmid", "tmup")) %>%
+    group_by(log_date, request_type) %>%
+    summarize(count = sum(count))
 }
plot_webstats_tm <- function(start_p, end_p, path_p) {
diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index b9a1d3b..f160698 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -5,6 +5,7 @@ library("RColorBrewer")
 library("scales")
 library(dplyr)
 library(tidyr)
+library(readr)
source('graphs.R')
 source('tables.R')
diff --git a/src/main/R/webstats/write-RData.R b/src/main/R/webstats/write-RData.R
deleted file mode 100644
index 96cc840..0000000
--- a/src/main/R/webstats/write-RData.R
+++ /dev/null
@@ -1,16 +0,0 @@
-dir.create("RData", showWarnings = FALSE)
-
-d <- read.csv("stats/webstats.csv", stringsAsFactors = FALSE)
-d <- d[d$request_type %in% c('tbid', 'tbsd', 'tbup', 'tbur'), ]
-data <- aggregate(list(count = d$count),
-    by = list(log_date = as.Date(d$log_date), request_type = d$request_type),
-    FUN = sum)
-save(data, file = "RData/webstats-tb.RData")
-
-d <- read.csv("stats/webstats.csv", stringsAsFactors = FALSE)
-d <- d[d$request_type %in% c('tmid', 'tmup'), ]
-data <- aggregate(list(count = d$count),
-    by = list(log_date = as.Date(d$log_date), request_type = d$request_type),
-    FUN = sum)
-save(data, file = "RData/webstats-tm.RData")
-

    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

[tor-commits] [metrics-web/release] Use readr to speed up drawing graphs.