commit af06d70a389b4f5748dbdeb87c75855fbabbc220 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Oct 11 12:26:47 2018 +0200

Extend Tor Browser graphs to include update pings.

Implements the second half of #27931 by making the actual code changes. --- src/main/R/rserver/graphs.R | 42 ++++++++++++++-------- src/main/resources/web/json/metrics.json | 8 ++--- .../resources/web/jsps/reproducible-metrics.jsp | 4 +-- src/main/resources/web/jsps/stats.jsp | 17 ++++----- 4 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index 60d905f..b021791 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -1331,14 +1331,17 @@ prepare_webstats_tb_platform <- function(start_p, end_p) { colClasses = c("log_date" = "Date")) %>% filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type == "tbid") %>% - group_by(log_date, platform) %>% + filter(request_type %in% c("tbid", "tbup")) %>% + group_by(log_date, platform, request_type) %>% summarize(count = sum(count)) }

plot_webstats_tb_platform <- function(start_p, end_p, path_p) { - prepare_webstats_tb_platform(start_p, end_p) %>% - ggplot(aes(x = log_date, y = count, colour = platform)) + + d <- prepare_webstats_tb_platform(start_p, end_p) + levels(d$request_type) <- list( + "Initial downloads" = "tbid", + "Update pings" = "tbup") + ggplot(d, aes(x = log_date, y = count, colour = platform)) + geom_point() + geom_line() + scale_x_date(name = "", breaks = custom_breaks, @@ -1347,9 +1350,11 @@ plot_webstats_tb_platform <- function(start_p, end_p, path_p) { scale_colour_hue(name = "Platform", breaks = c("w", "m", "l", "o", ""), labels = c("Windows", "macOS", "Linux", "Other", "Unknown")) + + facet_grid(request_type ~ ., scales = "free_y") + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA)) + - ggtitle("Tor Browser downloads by platform") + + strip.background = element_rect(fill = NA), + legend.position = "top") + + ggtitle("Tor Browser downloads and updates by platform") + labs(caption = copyright_notice) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } @@ -1357,8 +1362,8 @@ plot_webstats_tb_platform <- function(start_p, end_p, path_p) { write_webstats_tb_platform <- function(start_p = NULL, end_p = NULL, path_p) { prepare_webstats_tb_platform(start_p, end_p) %>% rename(date = log_date) %>% - spread(platform, count) %>% - rename(linux = l, macos = m, windows = w) %>% + spread(request_type, count, fill = 0) %>% + rename(initial_downloads = tbid, update_pings = tbup) %>% write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") }

@@ -1366,12 +1371,16 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) { d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""), colClasses = c("log_date" = "Date", "locale" = "character")) d <- d[d$log_date >= start_p & d$log_date <= end_p & - d$request_type == "tbid", ] + d$request_type %in% c("tbid", "tbup"), ] + levels(d$request_type) <- list( + "Initial downloads" = "tbid", + "Update pings" = "tbup") e <- d e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum) e <- e[order(e$count, decreasing = TRUE), ] e <- e[1:5, ] d <- aggregate(list(count = d$count), by = list(log_date = d$log_date, + request_type = d$request_type, locale = ifelse(d$locale %in% e$locale, d$locale, "(other)")), FUN = sum) ggplot(d, aes(x = log_date, y = count, colour = locale)) + geom_point() + @@ -1382,9 +1391,11 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) { scale_colour_hue(name = "Locale", breaks = c(e$locale, "(other)"), labels = c(e$locale, "Other")) + + facet_grid(request_type ~ ., scales = "free_y") + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA)) + - ggtitle("Tor Browser downloads by locale") + + strip.background = element_rect(fill = NA), + legend.position = "top") + + ggtitle("Tor Browser downloads and updates by locale") + labs(caption = copyright_notice) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } @@ -1399,10 +1410,13 @@ write_webstats_tb_locale <- function(start_p = NULL, end_p = NULL, path_p) { colClasses = c("log_date" = "Date", "locale" = "character")) %>% filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type == "tbid") %>% - group_by(log_date, locale) %>% - summarize(initial_downloads = sum(count)) %>% + filter(request_type %in% c("tbid", "tbup")) %>% rename(date = log_date) %>% + group_by(date, locale, request_type) %>% + summarize(count = sum(count)) %>% + mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>% + spread(request_type, count, fill = 0) %>% + rename(initial_downloads = tbid, update_pings = tbup) %>% write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") }

diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index ed74959..502b0e6 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -397,9 +397,9 @@ }, { "id": "webstats-tb-platform", - "title": "Tor Browser downloads by platform", + "title": "Tor Browser downloads and updates by platform", "type": "Graph", - "description": "<p>This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable, broken down by platform (Windows, macOS, Linux) of the requested executable. Note that this graph does <em>not</em> show the platform used to download Tor Browser but the platform that it was downloaded for.</p>", + "description": "<p>This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable and requests made by Tor Browser to check for an update, broken down by platform (Windows, macOS, Linux). Note that, at least for initial downloads, this graph does <em>not</em> show the platform used to download Tor Browser but the platform that it was downloaded for.</p>", "function": "webstats_tb_platform", "parameters": [ "start", @@ -408,9 +408,9 @@ }, { "id": "webstats-tb-locale", - "title": "Tor Browser downloads by locale", + "title": "Tor Browser downloads and updates by locale", "type": "Graph", - "description": "<p>This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable, broken down by requested locale.</p>", + "description": "<p>This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable and requests made by Tor Browser to check for an update, broken down by locale.</p>", "function": "webstats_tb_locale", "parameters": [ "start", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 4bed5cc..939b42e 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -809,8 +809,8 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h

<ul> <li>Tor Browser downloads and updates <a href="/webstats-tb.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Tor Browser downloads by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Tor Browser downloads by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Tor Browser downloads and updates by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Tor Browser downloads and updates by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Tor Messenger downloads and updates <a href="/webstats-tm.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> </ul>

diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index c6da19f..3482320 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -47,7 +47,7 @@ https://metrics.torproject.org/identifier.csv <li><b>July 31, 2018:</b> Announced pending changes to per-graph CSV files to become effective on August 15 and pre-aggregated CSV files to be removed by September 15.</li> <li><b>August 15, 2018:</b> Made the first batch of changes to per-graph CSV files.</li> <li><b>September 15, 2018:</b> Removed all pre-aggregated CSV files.</li> -<li><b>October 28, 2018:</b> Add and/or remove columns to <a href="#webstats-tb-platform">Tor Browser downloads by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads by locale</a> graphs.</li> +<li><b>October 28, 2018:</b> Added and/or removed columns to <a href="#webstats-tb-platform">Tor Browser downloads and updates by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads and updates by locale</a> graphs.</li> </ul>

</div> @@ -685,7 +685,7 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <li><b>update_requests:</b> Number of Tor Browser update requests: GET requests to all sites with resource strings <code>'%/torbrowser/%.mar'</code> and response code 302.</li> </ul>

-<h3>Tor Browser downloads by platform +<h3>Tor Browser downloads and updates by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/webstats-tb-platform.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> <a href="#webstats-tb-platform" name="webstats-tb-platform" class="anchor">#</a></h3> @@ -701,15 +701,12 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h

<ul> <li><b>date:</b> UTC date (YYYY-MM-DD) when requests to <code>torproject.org</code> web servers have been logged.</li> -<li><b>linux:</b> Number of Tor Browser initial downloads for Linux. <span class="red">This column is going to be removed after October 28, 2018.</span></li> -<li><b>macos:</b> Number of Tor Browser initial downloads for macOS. <span class="red">This column is going to be removed after October 28, 2018.</span></li> -<li><b>windows:</b> Number of Tor Browser initial downloads for Windows. <span class="red">This column is going to be removed after October 28, 2018.</span></li> -<li><b>platform:</b> Platform, like "Linux", "macOS", or "Windows". <span class="blue">This column is going to be added after October 28, 2018.</span></li> -<li><b>initial_downloads:</b> Number of Tor Browser initial downloads. <span class="blue">This column is going to be added after October 28, 2018.</span></li> -<li><b>update_pings:</b> Number of Tor Browser update pings. <span class="blue">This column is going to be added after October 28, 2018.</span></li> +<li><b>platform:</b> Platform, like "Linux", "macOS", or "Windows".</li> +<li><b>initial_downloads:</b> Number of Tor Browser initial downloads.</li> +<li><b>update_pings:</b> Number of Tor Browser update pings.</li> </ul>

-<h3>Tor Browser downloads by locale +<h3>Tor Browser downloads and updates by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/webstats-tb-locale.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> <a href="#webstats-tb-locale" name="webstats-tb-locale" class="anchor">#</a></h3> @@ -727,7 +724,7 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <li><b>date:</b> UTC date (YYYY-MM-DD) when requests to <code>torproject.org</code> web servers have been logged.</li> <li><b>locale:</b> Locale, like "en-US" for English (United States), "de" for German, etc., and "??" for unrecognized locales.</li> <li><b>initial_downloads:</b> Number of Tor Browser initial downloads.</li> -<li><b>update_pings:</b> Number of Tor Browser update pings. <span class="blue">This column is going to be added after October 28, 2018.</span></li> +<li><b>update_pings:</b> Number of Tor Browser update pings.</li> </ul>

<h3>Tor Messenger downloads and updates