commit 0b246ad078e4b548041ea9ef49089c12c3625fdd Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Jun 25 19:53:28 2020 +0200

Drop filesize parameter from torperf failures graph.

Implements tpo/metrics/website#40001. --- src/main/R/rserver/rserve-init.R | 27 ++++----------- .../torproject/metrics/stats/onionperf/Main.java | 19 ++++------- src/main/resources/web/json/metrics.json | 3 +- src/main/resources/web/jsps/stats.jsp | 3 +- src/main/sql/onionperf/init-onionperf.sql | 38 ++-------------------- 5 files changed, 19 insertions(+), 71 deletions(-)

diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 3ad89ce..e475e95 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -611,36 +611,26 @@ plot_torperf <- function(start_p, end_p, server_p, filesize_p, path_p) { }

prepare_torperf_failures <- function(start_p = NULL, end_p = NULL, - server_p = NULL, filesize_p = NULL) { - read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""), + server_p = NULL) { + read_csv(file = paste(stats_dir, "onionperf-failures.csv", sep = ""), col_types = cols( date = col_date(format = ""), - filesize = col_double(), source = col_character(), server = col_character(), - q1 = col_skip(), - md = col_skip(), - q3 = col_skip(), timeouts = col_double(), failures = col_double(), requests = col_double())) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(filesize_p)) - filesize == ifelse(filesize_p == "50kb", 50 * 1024, - ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else - TRUE) %>% filter(if (!is.null(server_p)) server == server_p else TRUE) %>% filter(requests > 0) %>% - transmute(date, filesize, source, server, timeouts = timeouts / requests, + transmute(date, source, server, timeouts = timeouts / requests, failures = failures / requests) }

-plot_torperf_failures <- function(start_p, end_p, server_p, filesize_p, - path_p) { - prepare_torperf_failures(start_p, end_p, server_p, filesize_p) %>% - filter(source != "") %>% - gather(variable, value, -c(date, filesize, source, server)) %>% +plot_torperf_failures <- function(start_p, end_p, server_p, path_p) { + prepare_torperf_failures(start_p, end_p, server_p) %>% + gather(variable, value, -c(date, source, server)) %>% mutate(variable = factor(variable, levels = c("timeouts", "failures"), labels = c("Timeouts", "Failures"))) %>% ggplot(aes(x = date, y = value, colour = source)) + @@ -650,10 +640,7 @@ plot_torperf_failures <- function(start_p, end_p, server_p, filesize_p, scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) + scale_colour_hue(name = "Source") + facet_grid(variable ~ .) + - ggtitle(paste("Timeouts and failures of", - ifelse(filesize_p == "50kb", "50 KiB", - ifelse(filesize_p == "1mb", "1 MiB", "5 MiB")), - "requests to", server_p, "server")) + + ggtitle(paste("Timeouts and failures of requests to", server_p, "server")) + labs(caption = copyright_notice) + theme(legend.position = "top") ggsave(filename = path_p, width = 8, height = 5, dpi = 150) diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 02fa08b..5cbd518 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -49,8 +49,8 @@ public class Main { writeStatistics( new File(baseDir, "stats/onionperf-including-partials.csv").toPath(), queryOnionperfIncludingPartials(connection)); - writeStatistics(new File(baseDir, "stats/torperf-1.1.csv").toPath(), - queryOnionPerf(connection)); + writeStatistics(new File(baseDir, "stats/onionperf-failures.csv").toPath(), + queryOnionperfFailures(connection)); writeStatistics(new File(baseDir, "stats/buildtimes.csv").toPath(), queryBuildTimes(connection)); writeStatistics(new File(baseDir, "stats/latencies.csv").toPath(), @@ -281,26 +281,21 @@ public class Main { return statistics; }

- static List<String> queryOnionPerf(Connection connection) + static List<String> queryOnionperfFailures(Connection connection) throws SQLException { logger.info("Querying timeout and failure statistics from database."); List<String> statistics = new ArrayList<>(); - statistics - .add("date,filesize,source,server,q1,md,q3,timeouts,failures,requests"); + statistics.add("date,source,server,timeouts,failures,requests"); Statement st = connection.createStatement(); - String queryString = "SELECT date, filesize, source, server, q1, md, q3, " - + "timeouts, failures, requests FROM onionperf"; + String queryString = "SELECT date, source, server, timeouts, failures, " + + "requests FROM onionperf_failures"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { - statistics.add(String.format("%s,%d,%s,%s,%.0f,%.0f,%.0f,%d,%d,%d", + statistics.add(String.format("%s,%s,%s,%d,%d,%d", dateFormat.format(rs.getDate("date")), - rs.getInt("filesize"), getStringFromResultSet(rs, "source"), getStringFromResultSet(rs, "server"), - getDoubleFromResultSet(rs, "q1"), - getDoubleFromResultSet(rs, "md"), - getDoubleFromResultSet(rs, "q3"), rs.getInt("timeouts"), rs.getInt("failures"), rs.getInt("requests"))); diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index f2b36a5..4735359 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -291,8 +291,7 @@ "parameters": [ "start", "end", - "server", - "filesize" + "server" ] }, { diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index ad6c566..37e5bbb 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -57,6 +57,7 @@ https://metrics.torproject.org/identifier.csv <li><b>December 18, 2019:</b> Added <a href="#bridgedb-transport">BridgeDB requests by requested transport</a> and <a href="#bridgedb-distributor">BridgeDB requests by distributor</a> graphs.</li> <li><b>May 24, 2020:</b> Included version 3 onion service measurements in <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> <li><b>May 24, 2020:</b> Included partial downloads of larger file sizes in <a href="#torperf">Time to download files over Tor</a> graph.</li> +<li><b>June 25, 2020:</b> Removed filesize parameter from <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a> graphs.</li> </ul>

</div> @@ -576,14 +577,12 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> <li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> <li><b>server:</b> Either <b>"public"</b> for requests to a server on the public internet, or <b>"onion"</b> for requests to an onion server.</li> -<li><b>filesize:</b> Size of the downloaded file in bytes, with pre-defined possible values: <b>"50kb"</b>, <b>"1mb"</b>, or <b>"5mb"</b>.</li> </ul>

<h4>Columns</h4>

<ul> <li><b>date:</b> UTC date (YYYY-MM-DD) when download performance was measured.</li> -<li><b>filesize:</b> Size of the downloaded file in bytes.</li> <li><b>source:</b> Name of the OnionPerf or Torperf service performing measurements.</li> <li><b>server:</b> Either <b>"public"</b> if the request was made to a server on the public internet, or <b>"onion"</b> if the request was made to an onion server.</li> <li><b>timeouts:</b> Fraction of requests that timed out when attempting to download the static file over Tor.</li> diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index ebe9f77..7c8a859 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -110,43 +110,11 @@ SELECT date, FROM grouped ORDER BY date, filesize, source, server;

-CREATE OR REPLACE VIEW onionperf AS -SELECT date, - filesize, - source, - server, - CASE WHEN q IS NULL THEN NULL ELSE q[1] END AS q1, - CASE WHEN q IS NULL THEN NULL ELSE q[2] END AS md, - CASE WHEN q IS NULL THEN NULL ELSE q[3] END AS q3, - timeouts, - failures, - requests -FROM ( +CREATE OR REPLACE VIEW onionperf_failures AS SELECT DATE(start) AS date, - filesize, source, CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' ELSE 'public' END AS server, - CASE WHEN COUNT(*) > 0 THEN - PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) - ELSE NULL END AS q, - COUNT(CASE WHEN didtimeout OR datacomplete < 1 THEN 1 ELSE NULL END) - AS timeouts, - COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1 - AND readbytes < filesize THEN 1 ELSE NULL END) AS failures, - COUNT(*) AS requests -FROM measurements -WHERE DATE(start) < current_date - 1 -GROUP BY date, filesize, source, server -UNION -SELECT DATE(start) AS date, - filesize, - '' AS source, - CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' - ELSE 'public' END AS server, - CASE WHEN COUNT(*) > 0 THEN - PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) - ELSE NULL END AS q, COUNT(CASE WHEN didtimeout OR datacomplete < 1 THEN 1 ELSE NULL END) AS timeouts, COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1 @@ -154,8 +122,8 @@ SELECT DATE(start) AS date, COUNT(*) AS requests FROM measurements WHERE DATE(start) < current_date - 1 -GROUP BY date, filesize, 3, server) sub -ORDER BY date, filesize, source, server; +GROUP BY date, source, server +ORDER BY date, source, server;

CREATE OR REPLACE VIEW buildtimes_stats AS SELECT date,