November 2019 - tor-commits - lists.torproject.org

[metrics-web/release] Ignore v3 onion OnionPerf measurements.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit 7af8f108136b5bb37a0e88f794ce7154c4660ba9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 29 15:14:10 2019 +0100 Ignore v3 onion OnionPerf measurements. We're later going to include these measurements in graphs, but that requires more changes. For now, let's ignore these measurements rather than include them in the v2 onion results. They will be in the database, so we'll be able to make those graphs including past measurements. --- src/main/sql/onionperf/init-onionperf.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index e9af3b1..7c0176e 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS measurements ( used_by INTEGER, endpointlocal CHARACTER VARYING(64), endpointproxy CHARACTER VARYING(64), - endpointremote CHARACTER VARYING(64), + endpointremote CHARACTER VARYING(96), hostnamelocal CHARACTER VARYING(64), hostnameremote CHARACTER VARYING(64), sourceaddress CHARACTER VARYING(64), @@ -80,6 +80,7 @@ SELECT DATE(start) AS date, COUNT(*) AS requests FROM measurements WHERE DATE(start) < current_date - 1 +AND endpointremote NOT SIMILAR TO '_{56}.onion%' GROUP BY date, filesize, source, server UNION SELECT DATE(start) AS date, @@ -97,6 +98,7 @@ SELECT DATE(start) AS date, COUNT(*) AS requests FROM measurements WHERE DATE(start) < current_date - 1 +AND endpointremote NOT SIMILAR TO '_{56}.onion%' GROUP BY date, filesize, 3, server) sub ORDER BY date, filesize, source, server; @@ -145,6 +147,7 @@ FROM measurements WHERE DATE(start) < current_date - 1 AND datarequest > 0 AND dataresponse > 0 +AND endpointremote NOT SIMILAR TO '_{56}.onion%' GROUP BY date, source, server UNION SELECT DATE(start) AS date, @@ -157,6 +160,7 @@ FROM measurements WHERE DATE(start) < current_date - 1 AND datarequest > 0 AND dataresponse > 0 +AND endpointremote NOT SIMILAR TO '_{56}.onion%' GROUP BY date, 2, server) sub ORDER BY date, source, server;

1 0

[metrics-web/release] Leave gaps for missing data.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit d1cedb7f2d011f8896f169338db7f0403702ea64 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jan 11 10:48:47 2019 +0100 Leave gaps for missing data. --- src/main/R/rserver/graphs.R | 47 ++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index 18a9d3e..0d7a90c 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -405,7 +405,9 @@ plot_versions <- function(start_p, end_p, path_p) { stringsAsFactors = FALSE) versions <- s[s$version %in% known_versions, ] visible_versions <- sort(unique(versions$version)) - ggplot(versions, aes(x = date, y = relays, colour = version)) + + versions <- versions %>% + complete(date = full_seq(date, period = 1), nesting(version)) %>% + ggplot(aes(x = date, y = relays, colour = version)) + geom_line() + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) + @@ -433,6 +435,7 @@ prepare_platforms <- function(start_p = NULL, end_p = NULL) { plot_platforms <- function(start_p, end_p, path_p) { prepare_platforms(start_p, end_p) %>% gather(platform, relays, -date) %>% + complete(date = full_seq(date, period = 1), nesting(platform)) %>% ggplot(aes(x = date, y = relays, colour = platform)) + geom_line() + scale_x_date(name = "", breaks = custom_breaks, @@ -470,6 +473,7 @@ prepare_dirbytes <- function(start_p = NULL, end_p = NULL) { plot_dirbytes <- function(start_p, end_p, path_p) { prepare_dirbytes(start_p, end_p) %>% gather(variable, value, -date) %>% + complete(date = full_seq(date, period = 1), nesting(variable)) %>% ggplot(aes(x = date, y = value, colour = variable)) + geom_line() + scale_x_date(name = "", breaks = custom_breaks, @@ -695,10 +699,10 @@ prepare_connbidirect <- function(start_p = NULL, end_p = NULL) { plot_connbidirect <- function(start_p, end_p, path_p) { prepare_connbidirect(start_p, end_p) %>% - ggplot(aes(x = date, y = md, colour = direction)) + - geom_line(size = 0.75) + - geom_ribbon(aes(x = date, ymin = q1, ymax = q3, - fill = direction), alpha = 0.5, show.legend = FALSE) + + complete(date = full_seq(date, period = 1), nesting(direction)) %>% + ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = direction)) + + geom_ribbon(alpha = 0.5) + + geom_line(aes(colour = direction), size = 0.75) + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) + scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) + @@ -1013,11 +1017,12 @@ plot_userstats_bridge_combined <- function(start_p, end_p, country_p, path_p) { a <- aggregate(list(mid = (u$high + u$low) / 2), by = list(transport = u$transport), FUN = sum) a <- a[order(a$mid, decreasing = TRUE)[1:top], ] - u <- u[u$transport %in% a$transport, ] + u <- u[u$transport %in% a$transport, ] %>% + complete(date = full_seq(date, period = 1), nesting(country, transport)) title <- paste("Bridge users by transport from ", countryname(country_p), sep = "") ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high, - colour = transport, fill = transport)) + + fill = transport)) + geom_ribbon(alpha = 0.5, size = 0.5) + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) + @@ -1055,6 +1060,7 @@ plot_advbwdist_perc <- function(start_p, end_p, p_p, path_p) { gather(variable, advbw, -c(date, p)) %>% mutate(variable = ifelse(variable == "all", "All relays", "Exits only")) %>% + complete(date = full_seq(date, period = 1), nesting(p, variable)) %>% ggplot(aes(x = date, y = advbw, colour = p)) + facet_grid(variable ~ .) + geom_line() + @@ -1092,6 +1098,7 @@ plot_advbwdist_relay <- function(start_p, end_p, n_p, path_p) { gather(variable, advbw, -c(date, n)) %>% mutate(variable = ifelse(variable == "all", "All relays", "Exits only")) %>% + complete(date = full_seq(date, period = 1), nesting(n, variable)) %>% ggplot(aes(x = date, y = advbw, colour = n)) + facet_grid(variable ~ .) + geom_line() + @@ -1123,6 +1130,7 @@ prepare_hidserv_dir_onions_seen <- function(start_p = NULL, end_p = NULL) { plot_hidserv_dir_onions_seen <- function(start_p, end_p, path_p) { prepare_hidserv_dir_onions_seen(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% ggplot(aes(x = date, y = onions)) + geom_line() + scale_x_date(name = "", breaks = custom_breaks, @@ -1152,6 +1160,7 @@ prepare_hidserv_rend_relayed_cells <- function(start_p = NULL, end_p = NULL) { plot_hidserv_rend_relayed_cells <- function(start_p, end_p, path_p) { prepare_hidserv_rend_relayed_cells(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% ggplot(aes(x = date, y = relayed)) + geom_line() + scale_x_date(name = "", breaks = custom_breaks, @@ -1192,6 +1201,8 @@ plot_webstats_tb <- function(start_p, end_p, path_p) { "update_requests"), labels = c("Initial downloads", "Signature downloads", "Update pings", "Update requests"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), nesting(request_type)) %>% ggplot(aes(x = date, y = count)) + geom_point() + geom_line() + @@ -1231,6 +1242,9 @@ plot_webstats_tb_platform <- function(start_p, end_p, path_p) { mutate(request_type = factor(request_type, levels = c("initial_downloads", "update_pings"), labels = c("Initial downloads", "Update pings"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), + nesting(platform, request_type)) %>% ggplot(aes(x = date, y = count, colour = platform)) + geom_point() + geom_line() + @@ -1299,6 +1313,7 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) { theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), strip.background = element_rect(fill = NA), legend.position = "top") + + guides(col = guide_legend(nrow = 1)) + ggtitle("Tor Browser downloads and updates by locale") + labs(caption = copyright_notice) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) @@ -1320,7 +1335,7 @@ prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) { group_by(log_date, request_type) %>% summarize(count = sum(count)) %>% mutate(request_type = factor(request_type, levels = c("tmid", "tmup"))) %>% - spread(request_type, count, drop = FALSE) %>% + spread(request_type, count, drop = FALSE, fill = 0) %>% rename(date = log_date, initial_downloads = tmid, update_pings = tmup) } @@ -1330,6 +1345,8 @@ plot_webstats_tm <- function(start_p, end_p, path_p) { mutate(request_type = factor(request_type, levels = c("initial_downloads", "update_pings"), labels = c("Initial downloads", "Update pings"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), nesting(request_type)) %>% ggplot(aes(x = date, y = count)) + geom_point() + geom_line() + @@ -1366,15 +1383,12 @@ prepare_relays_ipv6 <- function(start_p = NULL, end_p = NULL) { announced = sum(server_count_sum_avg[announced_ipv6]), reachable = sum(server_count_sum_avg[reachable_ipv6_relay]), exiting = sum(server_count_sum_avg[exiting_ipv6_relay])) %>% - complete(valid_after_date = full_seq(valid_after_date, period = 1)) %>% - gather(total, announced, reachable, exiting, key = "category", - value = "count") %>% - rename(date = valid_after_date) %>% - spread(category, count) + rename(date = valid_after_date) } plot_relays_ipv6 <- function(start_p, end_p, path_p) { prepare_relays_ipv6(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% gather(category, count, -date) %>% ggplot(aes(x = date, y = count, colour = category)) + geom_line() + @@ -1411,12 +1425,12 @@ prepare_bridges_ipv6 <- function(start_p = NULL, end_p = NULL) { group_by(valid_after_date) %>% summarize(total = sum(server_count_sum_avg), announced = sum(server_count_sum_avg[announced_ipv6])) %>% - complete(valid_after_date = full_seq(valid_after_date, period = 1)) %>% rename(date = valid_after_date) } plot_bridges_ipv6 <- function(start_p, end_p, path_p) { prepare_bridges_ipv6(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% gather(category, count, -date) %>% ggplot(aes(x = date, y = count, colour = category)) + geom_line() + @@ -1461,12 +1475,12 @@ prepare_advbw_ipv6 <- function(start_p = NULL, end_p = NULL) { reachable_ipv6_relay & exit_relay]), exiting = sum(advertised_bandwidth_bytes_sum_avg[ exiting_ipv6_relay])) %>% - complete(valid_after_date = full_seq(valid_after_date, period = 1)) %>% rename(date = valid_after_date) } plot_advbw_ipv6 <- function(start_p, end_p, path_p) { prepare_advbw_ipv6(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% gather(category, advbw, -date) %>% ggplot(aes(x = date, y = advbw, colour = category)) + geom_line() + @@ -1506,9 +1520,10 @@ prepare_totalcw <- function(start_p = NULL, end_p = NULL) { plot_totalcw <- function(start_p, end_p, path_p) { prepare_totalcw(start_p, end_p) %>% - mutate(nickname = ifelse(nickname == "", "consensus", nickname)) %>% + mutate(nickname = ifelse(is.na(nickname), "consensus", nickname)) %>% mutate(nickname = factor(nickname, levels = c("consensus", unique(nickname[nickname != "consensus"])))) %>% + ungroup() %>% complete(date = full_seq(date, period = 1), nesting(nickname)) %>% ggplot(aes(x = date, y = totalcw, colour = nickname)) + geom_line(na.rm = TRUE) +

1 0

[metrics-web/release] Update news.json to version 308 of doc/MetricsTimeline.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit ad7ed81857aab28f85e7c0f5f63286fff7230af8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jan 11 11:42:42 2019 +0100 Update news.json to version 308 of doc/MetricsTimeline. --- src/main/resources/web/json/news.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/resources/web/json/news.json b/src/main/resources/web/json/news.json index 26bf78a..1a204e0 100644 --- a/src/main/resources/web/json/news.json +++ b/src/main/resources/web/json/news.json @@ -3468,6 +3468,19 @@ "target" : "https://media.ccc.de/v/35c3-9964-cat_mouse_evading_the_censors_in_2018#t=16…" } ] }, { + "start" : "2019-01-07", + "end" : "2018-01-08", + "places" : [ "ga" ], + "short_description" : "Internet shutdown in Gabon following a coup attempt.", + "description" : "Internet shutdown in Gabon following a coup attempt.", + "links" : [ { + "label" : "OONI report", + "target" : "https://ooni.torproject.org/post/gabon-internet-disruption/" + }, { + "label" : "NetBlocks report", + "target" : "https://netblocks.org/reports/evidence-of-gabon-full-internet-shutdown-coup…" + } ] +}, { "start" : "2016-02-24", "places" : [ "tm" ], "protocols" : [ "<OR>" ],

1 0

[metrics-web/release] Extend latency graph to contain high/low values.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit fd251d6c69efeb3d548f915cc40e001e89c1f41f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 29 16:10:40 2019 +0200 Extend latency graph to contain high/low values. This patch adds two new lines to the existing circuit round-trip latencies graph: lowest and highest measurements that are not outliers. Implements #29773. --- src/main/R/rserver/rserve-init.R | 15 ++++-- .../torproject/metrics/stats/onionperf/Main.java | 10 ++-- src/main/resources/web/json/metrics.json | 2 +- .../resources/web/jsps/reproducible-metrics.jsp | 3 +- src/main/resources/web/jsps/stats.jsp | 2 + src/main/sql/onionperf/init-onionperf.sql | 58 +++++++++++----------- 6 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 3d6dad7..88aa5b9 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -663,9 +663,11 @@ prepare_onionperf_latencies <- function(start_p = NULL, end_p = NULL, date = col_date(format = ""), source = col_character(), server = col_character(), + low = col_double(), q1 = col_double(), md = col_double(), - q3 = col_double())) %>% + q3 = col_double(), + high = col_double())) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% filter(if (!is.null(server_p)) server == server_p else TRUE) @@ -675,18 +677,23 @@ plot_onionperf_latencies <- function(start_p, end_p, server_p, path_p) { prepare_onionperf_latencies(start_p, end_p, server_p) %>% filter(source != "") %>% complete(date = full_seq(date, period = 1), nesting(source)) %>% - ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + + ggplot(aes(x = date, ymin = q1, ymax = q3, fill = source)) + geom_ribbon(alpha = 0.5) + - geom_line(aes(colour = source), size = 0.75) + + geom_line(aes(y = md, colour = source), size = 0.75) + + geom_line(aes(y = high, colour = source), size = 0.375) + + geom_line(aes(y = low, colour = source), size = 0.375) + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) + scale_y_continuous(name = "", labels = unit_format(unit = "ms"), limits = c(0, NA)) + scale_fill_hue(name = "Source") + scale_colour_hue(name = "Source") + + facet_grid(source ~ ., scales = "free", space = "free") + ggtitle(paste("Circuit round-trip latencies to", server_p, "server")) + labs(caption = copyright_notice) + - theme(legend.position = "top") + theme(legend.position = "none", + strip.text.y = element_text(angle = 0, hjust = 0), + strip.background = element_rect(fill = NA)) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index a970434..a75cd1b 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -298,22 +298,24 @@ public class Main { throws SQLException { log.info("Querying latency statistics from database."); List<String> statistics = new ArrayList<>(); - statistics.add("date,source,server,q1,md,q3"); + statistics.add("date,source,server,low,q1,md,q3,high"); Statement st = connection.createStatement(); - String queryString = "SELECT date, source, server, q1, md, q3 " + String queryString = "SELECT date, source, server, low, q1, md, q3, high " + "FROM latencies_stats"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { - statistics.add(String.format("%s,%s,%s,%d,%d,%d", + statistics.add(String.format("%s,%s,%s,%d,%d,%d,%d,%d", dateFormat.format(rs.getDate("date", calendar)), getStringFromResultSet(rs, "source"), rs.getString("server"), + rs.getInt("low"), rs.getInt("q1"), rs.getInt("md"), - rs.getInt("q3"))); + rs.getInt("q3"), + rs.getInt("high"))); } } return statistics; diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 5c99546..006de8b 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -300,7 +300,7 @@ "id": "onionperf-latencies", "title": "Circuit round-trip latencies", "type": "Graph", - "description": "This graph shows round-trip latencies of circuits used for downloading static files of different sizes over Tor, either from a server on the public internet or from a version 2 onion server. Round-trip latencies are measured as the time between sending the HTTP request and receiving the HTTP response header. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", + "description": "This graph shows round-trip latencies of circuits used for downloading static files of different sizes over Tor, either from a server on the public internet or from a version 2 onion server. Round-trip latencies are measured as the time between sending the HTTP request and receiving the HTTP response header. The graph shows the median of measurements as thick line, the range of measurements from first to third quartile as ribbon, and the highest and lowest non-outlier measurements as thin lines.", "function": "onionperf_latencies", "parameters": [ "start", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 3838972..a833b31 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -650,7 +650,8 @@ We then compute the 25th, 50th, and 75th percentile of download times by sorting We're also interested in circuit round-trip latencies for the <a href="/onionperf-latencies.html">Circuit round-trip latencies</a> graph. We measure circuit latency as the time between sending the HTTP request and receiving the HTTP response header. We calculate latencies as <code>DATARESPONSE - DATAREQUEST</code> for measurements with non-zero values for both timestamps. -We then compute 25th, 50th, and 75th percentiles in the same way as for download times above. +We then compute 25th, 50th, and 75th percentiles in the same way as for download times above. +We also compute the lowest latency within 1.5 IQR of the lower quartile and the highest latency within 1.5 IQR of the upper quartile. Ideally, all measurements would succeed. But it's also possible that some measurements did not complete within a pre-defined timeout or failed for some other reason. diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 6c1955a..443c292 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -563,9 +563,11 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>date: UTC date (YYYY-MM-DD) when download performance was measured.</li> <li>source: Name of the OnionPerf or Torperf service performing measurements.</li> <li>server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server.</li> +<li>low: Lowest latency within 1.5 IQR of lower quartile (lower whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>q1: First quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>md: Median of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>q3: Third quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> +<li>high: Highest latency within 1.5 IQR of upper quartile (upper whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> </ul> </div> diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index 7c0176e..536cd45 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -130,37 +130,37 @@ GROUP BY date, 2, position) sub ORDER BY date, source, position; CREATE OR REPLACE VIEW latencies_stats AS +WITH filtered_measurements AS ( + SELECT DATE(start) AS date, + source, + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' + ELSE 'public' END AS server, + dataresponse - datarequest AS latency + FROM measurements + WHERE DATE(start) < current_date - 1 + AND datarequest > 0 + AND dataresponse > 0 + AND endpointremote NOT SIMILAR TO '_{56}.onion%' +), quartiles AS ( + SELECT date, + source, + server, + PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) + WITHIN GROUP(ORDER BY latency) AS q + FROM filtered_measurements + GROUP BY date, source, server +) SELECT date, source, server, - TRUNC(q[1]) AS q1, - TRUNC(q[2]) AS md, - TRUNC(q[3]) AS q3 -FROM ( -SELECT DATE(start) AS date, - source, - CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' - ELSE 'public' END AS server, - PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) - WITHIN GROUP(ORDER BY dataresponse - datarequest) AS q -FROM measurements -WHERE DATE(start) < current_date - 1 -AND datarequest > 0 -AND dataresponse > 0 -AND endpointremote NOT SIMILAR TO '_{56}.onion%' -GROUP BY date, source, server -UNION -SELECT DATE(start) AS date, - '' AS source, - CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' - ELSE 'public' END AS server, - PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) - WITHIN GROUP(ORDER BY dataresponse - datarequest) AS q -FROM measurements -WHERE DATE(start) < current_date - 1 -AND datarequest > 0 -AND dataresponse > 0 -AND endpointremote NOT SIMILAR TO '_{56}.onion%' -GROUP BY date, 2, server) sub + MIN(CASE WHEN latency >= q[1] - ((q[3] - q[1]) * 1.5) + THEN latency ELSE NULL END) AS low, + TRUNC(AVG(q[1])) AS q1, + TRUNC(AVG(q[2])) AS md, + TRUNC(AVG(q[3])) AS q3, + MAX(CASE WHEN latency <= q[3] + ((q[3] - q[1]) * 1.5) + THEN latency ELSE NULL END) AS high +FROM filtered_measurements NATURAL JOIN quartiles +GROUP BY 1, 2, 3 ORDER BY date, source, server;

1 0

[metrics-web/release] Updates fallback directories
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit b6223ecf7a7242e8e4c47a0c5fb70f5ffa35c845 Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Mar 27 19:18:28 2019 +0000 Updates fallback directories --- src/main/resources/web/js/rs/fallback_dir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/js/rs/fallback_dir.js b/src/main/resources/web/js/rs/fallback_dir.js index 93f97c5..f1a000f 100644 --- a/src/main/resources/web/js/rs/fallback_dir.js +++ b/src/main/resources/web/js/rs/fallback_dir.js @@ -8,7 +8,7 @@ To update run: python3 scripts/fallback_dir.py > js/fallback_dir.js */ -var fallbackDirs = ["001524DD403D729F08F7E5D77813EF12756CFA8D", "0111BA9B604669E636FFD5B503F382A4B7AD6E80", "025B66CEBC070FCB0519D206CF0CF4965C20C96E", "0756B7CD4DFC8182BE23143FAC0642F515182CEB", "0B85617241252517E8ECF2CFC7F4C1A32DCD153F", "0BEA4A88D069753218EAAAD6D22EA87B9A1319D6", "0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F", "0D3EBA17E1C78F1E9900BABDB23861D46FCAF163", "0E8C0C8315B66DB5F703804B3889A1DD66C67CE0", "11DF0017A43AF1F08825CD5D973297F81AB00FF3", "12AD30E5D25AA67F519780E2111E611A455FDC89", "12FD624EE73CEF37137C90D38B2406A66F68FAA2", "136F9299A5009A4E0E96494E723BDB556FB0A26B", "16102E458460349EE45C0901DAA6C30094A9BBEA", "175921396C7C426309AB03775A9930B6F611F794", "185663B7C12777F052B2C2D23D7A239D8DA88A0F", "1938EBACBB1A7BFA888D9623C90061130E63BB3F", "1AE039EE0B11DB79E4B4B29CBA9F752864A0259E", "1C90D3AEADFF3BCD079810632C8B85637924A58E", "1DBAED235E3957DE1ABD25B4206BE71406FB61F8", "1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7", "1F6ABD086F40B890A33C93CC4606EE68B31C9556", "1FA8F638 298645BE58AC905276680889CB795A94", "20462CBA5DA4C2D963567D17D0B7249718114A68", "204DFD2A2C6A0DC1FA0EACB495218E0B661704FD", "230A8B2A8BA861210D9B4BA97745AEC217A94207", "231C2B9C8C31C295C472D031E06964834B745996", "2BA2C8E96B2590E1072AECE2BDB5C48921BF8510", "2CDCFED0142B28B002E89D305CBA2E26063FADE2", "2F0F32AB1E5B943CA7D062C03F18960C86E70D94", "30C19B81981F450C402306E2E7CFB6C3F79CB6B2", "328E54981C6DDD7D89B89E418724A4A7881E3192", "330CD3DB6AD266DC70CDB512B036957D03D9BC59", "33DA0CAB7C27812EFF2E22C9705630A54D101FEB", "3711E80B5B04494C971FB0459D4209AB7F2EA799", "379FB450010D17078B3766C2273303C358C3A442", "387B065A38E4DAA16D9D41C2964ECBC4B31D30FF", "39F096961ED2576975C866D450373A9913AFDC92", "3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B", "3C79699D4FBC37DE1A212D5033B56DAE079AC0EF", "3D7E274A87D9A89AF064C13D1EE4CA1F184F2600", "3E53D3979DB07EFD736661C934A1DED14127B684", "4061C553CA88021B8302F0814365070AAE617270", "40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD", "41C59606AFE1D1AA6EC6EF6719690B856F0B6 587", "439D0447772CB107B886F7782DBC201FA26B92D1", "4623A9EC53BFD83155929E56D6F7B55B5E718C24", "46791D156C9B6C255C2665D4D8393EC7DBAA7798", "484A10BA2B8D48A5F0216674C8DD50EF27BC32F3", "489D94333DF66D57FFE34D9D59CC2D97E2CB0053", "4CC9CC9195EC38645B699A33307058624F660CCF", "4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2", "50586E25BE067FD1F739998550EDDCB1A14CA5B2", "51E1CF613FD6F9F11FE24743C91D6F9981807D82", "52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9", "587E0A9552E4274B251F29B5B2673D38442EE4BF", "58ED9C9C35E433EE58764D62892B4FFD518A3CD0", "5E56738E7F97AA81DEEF59AF28494293DFBFCCDF", "5F4CD12099AF20FAF9ADFDCEC65316A376D0201C", "616081EC829593AF4232550DE6FFAA1D75B37A90", "68F175CCABE727AA2D2309BCD8789499CEE36ED7", "6A7551EEE18F78A9813096E82BF84F740D32B911", "6EF897645B79B6CB35E853B32506375014DE3621", "72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE", "7600680249A22080ECC6173FBBF64D6FCF330A61", "763C9556602BD6207771A7A3D958091D44C43228", "774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7", "775B0FAFDE71AADC23FFC8 782B7BEB1D5A92733E", "789EA6C9AE9ADDD8760903171CFA9AC5741B0C70", "78E2BE744A53631B4AAB781468E94C52AB73968B", "79E169B25E4C7CE99584F6ED06F379478F23E2B8", "7A32C9519D80CA458FC8B034A28F5F6815649A98", "7BB70F8585DFC27E75D692970C0EEB0F22983A63", "7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55", "7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E", "7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3", "80AAF8D5956A43C197104CEF2550CD42D165C6FB", "8456DFA94161CDD99E480C2A2992C366C6564410", "855BC2DABE24C861CD887DB9B2E950424B49FC34", "8567AD0A6369ED08527A8A8533A5162AC00F7678", "86C281AD135058238D7A337D546C902BE8505DDE", "88487BDD980BF6E72092EE690E8C51C0AA4A538C", "8C00FA7369A7A308F6A137600F0FA07990D9D451", "8D79F73DCD91FC4F5017422FAC70074D6DB8DD81", "9007C1D8E4F03D506A4A011B907A9E8D04E3C605", "91D23D8A539B83D2FB56AA67ECD4D75CC093AC55", "9285B22F7953D7874604EEE2B470609AD81C74E9", "92CFD9565B24646CAC2D172D3DB503D69E777B8A", "92ECC9E0E2AF81BB954719B189AC362E254AD4A5", "9772EFB535397C942C3AB8804FB35CFFAD012438", "998BF3E D7F70E33D1C307247B9626D9E7573C438", "9A0D54D3A6D2E0767596BF1515E6162A75B3293F", "9A68B85A02318F4E7E87F2828039FBD5D75B0142", "9B31F1F1C1554F9FFB3455911F82E818EF7C7883", "9EC5E097663862DF861A18C32B37C5F82284B27D", "9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A", "9F7D6E6420183C2B76D3CE99624EBC98A21A967E", "9FBEB75E8BC142565F12CBBE078D63310236A334", "A0F06C2FADF88D3A39AA3072B406F09D7095AC9E", "A10C4F666D27364036B562823E5830BC448E046A", "A2E6BB5C391CD46B38C55B4329C35304540771F1", "A478E421F83194C114F41E94F95999672AED51FE", "A4C98CEA3F34E05299417E9F885A642C88EF6029", "A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2", "AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C", "ACD889D86E02EDDAB1AFD81F598C0936238DC6D0", "ACDD9E85A05B127BA010466C13C8C47212E8A38F", "AD19490C7DBB26D3A68EFC824F67E69B0A96E601", "B0279A521375F3CB2AE210BDBFC645FDD2E1973A", "B0553175AADB0501E5A61FC61CEA3970BE130FF2", "B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1", "B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249", "B143D439B72D239A419F8DCE07B8A8EB1B48 6FA7", "B291D30517D23299AD7CEE3E60DFE60D0E3A4664", "B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20", "B5212DB685A2A0FCFBAE425738E478D12361710D", "B6904ADD4C0D10CDA7179E051962350A69A63243", "B83DC1558F0D34353BB992EF93AFEAFDB226A73E", "B86137AE9681701901C6720E55C16805B46BD8E3", "BC630CBBB518BE7E9F4E09712AB0269E9DC7D626", "BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E", "BCEF908195805E03E92CCFE669C48738E556B9C5", "BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6", "BF735F669481EE1CCC348F0731551C933D1E2278", "C2AAB088555850FC434E68943F551072042B85F1", "C37BC191AC389179674578C3E6944E925FE186C2", "C414F28FD2BEC1553024299B31D4E726BEB8E788", "C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2", "CE47F0356D86CF0A1A2008D97623216D560FB0A8", "CED527EAC230E7B56E5B363F839671829C3BA01B", "D30E9D4D639068611D6D96861C95C2099140B805", "D3E5EDDBE5159388704D6785BE51930AAFACEC6F", "D64366987CB39F61AD21DBCF8142FA0577B92811", "D760C5B436E42F93D77EF2D969157EEA14F9B39C", "D8B7A3A6542AA54D0946B9DC0257C53B6C376679", "DAA39FC00B196B353C2A2 71459C305C429AF09E4", "DD823AFB415380A802DCAEB9461AE637604107FB", "DD8BD7307017407FCC36F8D04A688F74A0774C02", "DED6892FF89DBD737BA689698A171B2392EB3E82", "E3DB2E354B883B59E8DC56B3E7A353DDFD457812", "E480D577F58E782A5BC4FA6F49A6650E9389302F", "E589316576A399C511A9781A73DA4545640B479D", "E781F4EC69671B3F1864AE2753E0890351506329", "E81EF60A73B3809F8964F73766B01BAA0A171E20", "EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F", "F4263275CF54A6836EE7BD527B1328836A6F06E1", "F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610", "F93D8F37E35C390BCAD9F9069E13085B745EC216", "FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B", "FE296180018833AF03A8EACD5894A614623D3F76"]; +var fallbackDirs = ["0111BA9B604669E636FFD5B503F382A4B7AD6E80", "01A9258A46E97FF8B2CAC7910577862C14F2C524", "0B85617241252517E8ECF2CFC7F4C1A32DCD153F", "0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6", "0C2C599AFCB26F5CFC2C7592435924C1D63D9484", "0E8C0C8315B66DB5F703804B3889A1DD66C67CE0", "113143469021882C3A4B82F084F8125B08EE471E", "11DF0017A43AF1F08825CD5D973297F81AB00FF3", "12AD30E5D25AA67F519780E2111E611A455FDC89", "136F9299A5009A4E0E96494E723BDB556FB0A26B", "14419131033443AE6E21DA82B0D307F7CAE42BDB", "14877C6384A9E793F422C8D1DDA447CACA4F7C4B", "1576BE143D8727745BB2BCDDF183291B3C3EFEFC", "15BE17C99FACE24470D40AF782D6A9C692AB36D6", "185F2A57B0C4620582602761097D17DB81654F70", "1938EBACBB1A7BFA888D9623C90061130E63BB3F", "1AE039EE0B11DB79E4B4B29CBA9F752864A0259E", "1C90D3AEADFF3BCD079810632C8B85637924A58E", "1F6ABD086F40B890A33C93CC4606EE68B31C9556", "20462CBA5DA4C2D963567D17D0B7249718114A68", "230A8B2A8BA861210D9B4BA97745AEC217A94207", "24E91955D969AEA1D80413C64FE106FAE7FD2EA9", "2BA2C8E9 6B2590E1072AECE2BDB5C48921BF8510", "2CDCFED0142B28B002E89D305CBA2E26063FADE2", "2F0F32AB1E5B943CA7D062C03F18960C86E70D94", "311A4533F7A2415F42346A6C8FA77E6FD279594C", "322C6E3A973BC10FC36DE3037AD27BC89F14723B", "330CD3DB6AD266DC70CDB512B036957D03D9BC59", "360CBA08D1E24F513162047BDB54A1015E531534", "361D33C96D0F161275EE67E2C91EE10B276E778B", "36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400", "375DCBB2DBD94E5263BC0C015F0C9E756669617E", "387B065A38E4DAA16D9D41C2964ECBC4B31D30FF", "39F096961ED2576975C866D450373A9913AFDC92", "3F092986E9B87D3FDA09B71FA3A602378285C77A", "4061C553CA88021B8302F0814365070AAE617270", "41A3C16269C7B63DB6EB741DBDDB4E1F586B1592", "41C59606AFE1D1AA6EC6EF6719690B856F0B6587", "45362E8ECD651CCAC521156FFBD2FF7F27FA8E88", "4623A9EC53BFD83155929E56D6F7B55B5E718C24", "4661DE96D3F8E923994B05218F23760C8D7935A4", "46791D156C9B6C255C2665D4D8393EC7DBAA7798", "47C42E2094EE482E7C9B586B10BABFB67557030B", "484A10BA2B8D48A5F0216674C8DD50EF27BC32F3", "4CC9CC9195EC38645B699A33307058624F660 CCF", "4EB55679FA91363B97372554F8DC7C63F4E5B101", "50586E25BE067FD1F739998550EDDCB1A14CA5B2", "51E1CF613FD6F9F11FE24743C91D6F9981807D82", "52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9", "547DA56F6B88B6C596B3E3086803CDA4F0EF8F21", "557ACEC850F54EEE65839F83CACE2B0825BE811E", "587E0A9552E4274B251F29B5B2673D38442EE4BF", "58ED9C9C35E433EE58764D62892B4FFD518A3CD0", "5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B", "5E56738E7F97AA81DEEF59AF28494293DFBFCCDF", "609E598FB6A00BCF7872906B602B705B64541C50", "68F175CCABE727AA2D2309BCD8789499CEE36ED7", "6EF897645B79B6CB35E853B32506375014DE3621", "7088D485934E8A403B81531F8C90BDC75FA43C98", "70C55A114C0EF3DC5784A4FAEE64388434A3398F", "71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36", "72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE", "7600680249A22080ECC6173FBBF64D6FCF330A61", "775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E", "7BB70F8585DFC27E75D692970C0EEB0F22983A63", "7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E", "80AAF8D5956A43C197104CEF2550CD42D165C6FB", "8101421BEFCCF4C271D548 3C5AABCAAD245BBB9D", "81AFA888F8F8F4A024AB58ECA0ADDEBB93FF01DA", "81B75D534F91BFB7C57AB67DA10BCEF622582AE8", "823AA81E277F366505545522CEDC2F529CE4DC3F", "844AE9CAD04325E955E2BE1521563B79FE7094B7", "8456DFA94161CDD99E480C2A2992C366C6564410", "86C281AD135058238D7A337D546C902BE8505DDE", "8844D87E9B038BE3270938F05AF797E1D3C74C0F", "8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F", "8C00FA7369A7A308F6A137600F0FA07990D9D451", "8FA37B93397015B2BC5A525C908485260BE9F422", "91D23D8A539B83D2FB56AA67ECD4D75CC093AC55", "9231DF741915AA1630031A93026D88726877E93A", "92412EA1B9AA887D462B51D816777002F4D58907", "924B24AFA7F075D059E8EEB284CC400B33D3D036", "92CFD9565B24646CAC2D172D3DB503D69E777B8A", "94C4B7B8C50C86A92B6A20107539EE2678CF9A28", "9772EFB535397C942C3AB8804FB35CFFAD012438", "99E246DB480B313A3012BC3363093CC26CD209C7", "9A0D54D3A6D2E0767596BF1515E6162A75B3293F", "9A68B85A02318F4E7E87F2828039FBD5D75B0142", "9B31F1F1C1554F9FFB3455911F82E818EF7C7883", "9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022", "9EAD5B2 D3DBD96DBC80DCE423B0C345E920A758D", "9F7D6E6420183C2B76D3CE99624EBC98A21A967E", "A0F06C2FADF88D3A39AA3072B406F09D7095AC9E", "A10C4F666D27364036B562823E5830BC448E046A", "A2A6616723B511D8E068BB71705191763191F6B2", "A2E6BB5C391CD46B38C55B4329C35304540771F1", "A478E421F83194C114F41E94F95999672AED51FE", "A4C98CEA3F34E05299417E9F885A642C88EF6029", "A53C46F5B157DD83366D45A8E99A244934A14C46", "AA0D167E03E298F9A8CD50F448B81FBD7FA80D56", "AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C", "ACD889D86E02EDDAB1AFD81F598C0936238DC6D0", "ACDD9E85A05B127BA010466C13C8C47212E8A38F", "AD19490C7DBB26D3A68EFC824F67E69B0A96E601", "ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D", "AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0", "B0553175AADB0501E5A61FC61CEA3970BE130FF2", "B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1", "B143D439B72D239A419F8DCE07B8A8EB1B486FA7", "B291D30517D23299AD7CEE3E60DFE60D0E3A4664", "B44FBE5366AD98B46D829754FA4AC599BAE41A6A", "B5212DB685A2A0FCFBAE425738E478D12361710D", "B6904ADD4C0D10CDA7179E051962350A69A6 3243", "B771AA877687F88E6F1CA5354756DF6C8A7B6B24", "B83DC1558F0D34353BB992EF93AFEAFDB226A73E", "B84F248233FEA90CAD439F292556A3139F6E1B82", "B86137AE9681701901C6720E55C16805B46BD8E3", "BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6", "BF0FB582E37F738CD33C3651125F2772705BB8E8", "BF735F669481EE1CCC348F0731551C933D1E2278", "C0192FF43E777250084175F4E59AC1BA2290CE38", "C08DE49658E5B3CFC6F2A952B453C4B608C9A16A", "C2AAB088555850FC434E68943F551072042B85F1", "C37BC191AC389179674578C3E6944E925FE186C2", "C414F28FD2BEC1553024299B31D4E726BEB8E788", "C4AEA05CF380BAD2230F193E083B8869B4A29937", "C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9", "C90CA3B7FE01A146B8268D56977DC4A2C024B9EA", "CBD0D1BD110EC52963082D839AC6A89D0AE243E7", "CE47F0356D86CF0A1A2008D97623216D560FB0A8", "D30E9D4D639068611D6D96861C95C2099140B805", "D405FCCF06ADEDF898DF2F29C9348DCB623031BA", "D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A", "D6BA940D3255AB40DC5EE5B0B285FA143E1F9865", "D71B1CA1C9DC7E8CA64158E106AD770A21160FEE", "D8B7A3A6542AA54D0946B 9DC0257C53B6C376679", "DAA39FC00B196B353C2A271459C305C429AF09E4", "DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E", "DD823AFB415380A802DCAEB9461AE637604107FB", "DD8BD7307017407FCC36F8D04A688F74A0774C02", "DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0", "DED6892FF89DBD737BA689698A171B2392EB3E82", "E51620B90DCB310138ED89EDEDD0A5C361AAE24E", "E81EF60A73B3809F8964F73766B01BAA0A171E20", "EB80A8D52F07238B576C42CEAB98ADD084EE075E", "ED2338CAC2711B3E331392E1ED2831219B794024", "EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F", "F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC", "F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8", "F4263275CF54A6836EE7BD527B1328836A6F06E1", "F6A358DD367B3282D6EF5824C9D45E1A19C7E815", "F741E5124CB12700DA946B78C9B2DD175D6CD2A1", "F8D27B163B9247B232A2EEE68DD8B698695C28DE", "F9246DEF2B653807236DA134F2AEAB103D58ABFE", "F93D8F37E35C390BCAD9F9069E13085B745EC216", "FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B", "FE296180018833AF03A8EACD5894A614623D3F76"]; function IsFallbackDir(fingerprint) { return $.inArray(fingerprint, fallbackDirs) > -1;

1 0

[metrics-web/release] Adapt "About Tor" links to redesigned Tor website.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit 31c4f88ebba9ad327594d4db47fa88fd7b5f6bb7 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon May 13 17:22:04 2019 +0200 Adapt "About Tor" links to redesigned Tor website. Reported by nusenu, changes suggested by irl. Fixes #30415. --- build.xml | 2 +- src/main/resources/web/properties/ExoneraTor.properties | 7 +++---- src/main/resources/web/properties/ExoneraTor_de.properties | 7 +++---- src/main/resources/web/properties/ExoneraTor_fr.properties | 7 +++---- src/main/resources/web/properties/ExoneraTor_ro.properties | 7 +++---- src/main/resources/web/properties/ExoneraTor_sv.properties | 7 +++---- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/build.xml b/build.xml index 254a71f..0c9fa0e 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ <property name="implementation-title" value="metrics-web" /> <property name="release.version" value="1.2.0-dev" /> <property name="metricslibversion" value="2.4.0" /> - <property name="exoneratorversion" value="4.0.0" /> + <property name="exoneratorversion" value="4.1.0" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" value="metrics-web-${release.version}.war"/> diff --git a/src/main/resources/web/properties/ExoneraTor.properties b/src/main/resources/web/properties/ExoneraTor.properties index 936754a..05bb0cb 100644 --- a/src/main/resources/web/properties/ExoneraTor.properties +++ b/src/main/resources/web/properties/ExoneraTor.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Yes technicaldetails.exit.no=No permanentlink.heading=Permanent link footer.abouttor.heading=About Tor -footer.abouttor.body.text=Tor is an international software project to anonymize Internet traffic by %s.  Therefore, if you see traffic from a Tor relay, this traffic usually originates from someone using Tor, rather than from the relay operator.  The Tor Project and Tor relay operators have no records of the traffic that passes over the network and therefore cannot provide any information about its origin.  Be sure to %s, and don't hesitate to %s for more information. -footer.abouttor.body.link1=encrypting packets and sending them through a series of hops before they reach their destination -footer.abouttor.body.link2=learn more about Tor -footer.abouttor.body.link3=contact The Tor Project, Inc. +footer.abouttor.body.text=Tor is an international software project to anonymize Internet traffic by encrypting packets and sending them through a series of hops before they reach their destination.  Therefore, if you see traffic from a Tor relay, this traffic usually originates from someone using Tor, rather than from the relay operator.  The Tor Project and Tor relay operators have no records of the traffic that passes over the network and therefore cannot provide any information about its origin.  Be sure to %s, and don't hesitate to %s for more information. +footer.abouttor.body.link1=learn more about Tor +footer.abouttor.body.link2=contact The Tor Project, Inc. footer.aboutexonerator.heading=About ExoneraTor footer.aboutexonerator.body=The ExoneraTor service maintains a database of IP addresses that have been part of the Tor network.  It answers the question whether there was a Tor relay running on a given IP address on a given date.  ExoneraTor may store more than one IP address per relay if relays use a different IP address for exiting to the Internet than for registering in the Tor network, and it stores whether a relay permitted transit of Tor traffic to the open Internet at that time. footer.language.name=English diff --git a/src/main/resources/web/properties/ExoneraTor_de.properties b/src/main/resources/web/properties/ExoneraTor_de.properties index fbe8464..13b150b 100644 --- a/src/main/resources/web/properties/ExoneraTor_de.properties +++ b/src/main/resources/web/properties/ExoneraTor_de.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Ja technicaldetails.exit.no=Nein permanentlink.heading=Dauerhafter Link footer.abouttor.heading=\u00dcber Tor -footer.abouttor.body.text=Tor ist ein internationales Softwareprojekt um Internetverbindungen zu anonymisieren indem %s. Wenn Sie eine Internetverbindung sehen, die von einem Tor-Server kommt, stammt diese in der Regel von jemandem, der Tor benutzt, und nicht vom Betreiber des Tor-Servers selbst. Weder The Tor Project, Inc. noch die Tor-Server-Betreiber besitzen Aufzeichnungen \u00fcber die Internetverbindungen im Tor-Netzwerk und k\u00f6nnen daher keine Auskunft \u00fcber den Ursprung der Internetverbindung geben. Mehr Informationen \u00fcber Tor erhalten Sie %s. The Tor Project, Inc. steht Ihnen jederzeit f\u00fcr %s zur Verf\u00fcgung. -footer.abouttor.body.link1=Datenpakete verschl\u00fcsselt und \u00fcber eine Reihe von Stationen geleitet werden bevor diese ihr Ziel erreichen -footer.abouttor.body.link2=hier -footer.abouttor.body.link3=Fragen +footer.abouttor.body.text=Tor ist ein internationales Softwareprojekt um Internetverbindungen zu anonymisieren indem Datenpakete verschl\u00fcsselt und \u00fcber eine Reihe von Stationen geleitet werden bevor diese ihr Ziel erreichen. Wenn Sie eine Internetverbindung sehen, die von einem Tor-Server kommt, stammt diese in der Regel von jemandem, der Tor benutzt, und nicht vom Betreiber des Tor-Servers selbst. Weder The Tor Project, Inc. noch die Tor-Server-Betreiber besitzen Aufzeichnungen \u00fcber die Internetverbindungen im Tor-Netzwerk und k\u00f6nnen daher keine Auskunft \u00fcber den Ursprung der Internetverbindung geben. Mehr Informationen \u00fcber Tor erhalten Sie %s. The Tor Project, Inc. steht Ihnen jederzeit f\u00fcr %s zur Verf\u00fcgung. +footer.abouttor.body.link1=hier +footer.abouttor.body.link2=Fragen footer.aboutexonerator.heading=\u00dcber ExoneraTor footer.aboutexonerator.body=Der ExoneraTor-Dienst basiert auf einer Datenbank von IP-Adressen, die Teil des Tor-Netzwerks sind oder waren. Dieser Dienst beantwortet die Frage, ob es an einem bestimmten Tag einen Tor-Server mit einer bestimmten IP-Adresse gegeben hat. ExoneraTor kann in einigen F\u00e4llen mehr als nur eine IP-Adresse pro Tor-Server speichern, falls ein Tor-Server eine andere IP-Adresse f\u00fcr ausgehende Verbindungen verwendet hat als die IP-Adresse um sich im Tor-Netzwerk zu registrieren. ExoneraTor speichert au\u00dferdem ob ein Tor-Server Verbindungen ins offene Internet zugelassen hat oder nicht. footer.language.name=Deutsch diff --git a/src/main/resources/web/properties/ExoneraTor_fr.properties b/src/main/resources/web/properties/ExoneraTor_fr.properties index 93c33bb..3aaf3e5 100644 --- a/src/main/resources/web/properties/ExoneraTor_fr.properties +++ b/src/main/resources/web/properties/ExoneraTor_fr.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Oui technicaldetails.exit.no=Non permanentlink.heading=Lien permanent footer.abouttor.heading=\u00c0 propos de Tor -footer.abouttor.body.text=Tor est un projet international d\u00e9veloppant des logiciels pour anonymiser le trafic Internet en %s. Par cons\u00e9quent, si vous voyez du trafic en provenance d'un relais Tor, ce trafic provient habituellement de quelqu'un qui utilise Tor, plut\u00f4t que de l'op\u00e9rateur du relais. Le Projet Tor et les op\u00e9rateurs de relais Tor n'ont aucune trace du trafic relay\u00e9 par le r\u00e9seau et ne peuvent donc fournir aucune information sur son origine. N'h\u00e9sitez pas \u00e0 vous %s et n'h\u00e9sitez pas \u00e0 %s pour obtenir d'avantage d'informations. -footer.abouttor.body.link1=chiffrant les paquets et en les faisant transiter par une s\u00e9rie de relais avant qu'ils n'atteignent leur destination -footer.abouttor.body.link2=renseigner d'avantage au sujet de Tor -footer.abouttor.body.link3=contacter \u00ab The Tor Project, Inc. \u00bb +footer.abouttor.body.text=Tor est un projet international d\u00e9veloppant des logiciels pour anonymiser le trafic Internet en chiffrant les paquets et en les faisant transiter par une s\u00e9rie de relais avant qu'ils n'atteignent leur destination. Par cons\u00e9quent, si vous voyez du trafic en provenance d'un relais Tor, ce trafic provient habituellement de quelqu'un qui utilise Tor, plut\u00f4t que de l'op\u00e9rateur du relais. Le Projet Tor et les op\u00e9rateurs de relais Tor n'ont aucune trace du trafic relay\u00e9 par le r\u00e9seau et ne peuvent donc fournir aucune information sur son origine. N'h\u00e9sitez pas \u00e0 vous %s et n'h\u00e9sitez pas \u00e0 %s pour obtenir d'avantage d'informations. +footer.abouttor.body.link1=renseigner d'avantage au sujet de Tor +footer.abouttor.body.link2=contacter \u00ab The Tor Project, Inc. \u00bb footer.aboutexonerator.heading=\u00c0 propos d'ExoneraTor footer.aboutexonerator.body=Le service ExoneraTor g\u00e8re une base de donn\u00e9es d'adresses IP qui ont fait partie du r\u00e9seau Tor. Il permet de savoir si un relais Tor fonctionnait pour une adresse IP donn\u00e9e \u00e0 une date pr\u00e9cise. ExoneraTor peut enregistrer plus d'une adresse IP par relais si ces derniers utilisent une adresse IP diff\u00e9rente pour se connecter \u00e0 Internet de celle utilis\u00e9e pour s'enregistrer sur le r\u00e9seau Tor. ExoneraTor enregistre \u00e9galement les dates et heures auxquelles un relais a permis de faire transiter du trafic en provenance de Tor vers Internet. footer.language.name=fran\u00e7ais diff --git a/src/main/resources/web/properties/ExoneraTor_ro.properties b/src/main/resources/web/properties/ExoneraTor_ro.properties index 3d58072..476f020 100644 --- a/src/main/resources/web/properties/ExoneraTor_ro.properties +++ b/src/main/resources/web/properties/ExoneraTor_ro.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Da technicaldetails.exit.no=Nu permanentlink.heading=Link permanent footer.abouttor.heading=Despre Tor -footer.abouttor.body.text=Tor este un proiect software interna\u021bional pentru anonimizarea traficului de internet prin %s.  De aceea, dac\u0103 vede\u021bi trafic de la un nod Tor, acest trafic provine mai degrab\u0103 de la un client care folose\u0219te Tor, nu de la operatorul releului.  The Tor Project, Inc. (organiza\u021bia) \u0219i operatorii nodurilor Tor nu de\u021bin informa\u021bii referitoare la traficul care trece prin re\u021bea \u0219i, implicit, prin nodurile operate de ace\u0219tia, de aceea nu pot furniza informa\u021bii despre originea acestuia. Asigura\u021bi-v\u0103 c\u0103 %s \u0219i nu ezita\u021bi s\u0103 %s pentru mai multe informa\u021bii. -footer.abouttor.body.link1=criptarea pachetelor de date \u0219i transmiterea acestora printr-o serie de hop-uri \u00eenainte ca ele s\u0103 ajung\u0103 la destina\u021bie -footer.abouttor.body.link2=Afla\u021bi mai multe despre Tor -footer.abouttor.body.link3=contacta\u021bi The Tor Project, Inc. +footer.abouttor.body.text=Tor este un proiect software interna\u021bional pentru anonimizarea traficului de internet prin criptarea pachetelor de date \u0219i transmiterea acestora printr-o serie de hop-uri \u00eenainte ca ele s\u0103 ajung\u0103 la destina\u021bie.  De aceea, dac\u0103 vede\u021bi trafic de la un nod Tor, acest trafic provine mai degrab\u0103 de la un client care folose\u0219te Tor, nu de la operatorul releului.  The Tor Project, Inc. (organiza\u021bia) \u0219i operatorii nodurilor Tor nu de\u021bin informa\u021bii referitoare la traficul care trece prin re\u021bea \u0219i, implicit, prin nodurile operate de ace\u0219tia, de aceea nu pot furniza informa\u021bii despre originea acestuia. Asigura\u021bi-v\u0103 c\u0103 %s \u0219i nu ezita\u021bi s\u0103 %s pentru mai multe informa\u021bii. +footer.abouttor.body.link1=Afla\u021bi mai multe despre Tor +footer.abouttor.body.link2=contacta\u021bi The Tor Project, Inc. footer.aboutexonerator.heading=Despre ExoneraTor footer.aboutexonerator.body=Serviciul ExoneraTor opereaz\u0103 o baz\u0103 de date de adrese IP care au fost parte din re\u021belaua Tor la un anumit moment.  Acest serviciu r\u0103spunde la \u00eentrebarea dac\u0103 a existat un nod Tor care a func\u021bionat pe o anumit\u0103 adres\u0103 IP la o anumit\u0103 dat\u0103.  ExoneraTor poate stoca mai mult de o adresa IP per nod dac\u0103 nodurile folosesc o alt\u0103 adres\u0103 IP pentru a ie\u0219i pe internet dec\u00e2t cea folosit\u0103 pentru \u00eenregistrarea \u00een re\u021beaua Tor \u0219i stocheaz\u0103 dac\u0103 un nod a permis tranzitarea de trafic din re\u021beaua Tor catre internetul public la acea dat\u0103. footer.language.name=rom\u00e2n\u0103 diff --git a/src/main/resources/web/properties/ExoneraTor_sv.properties b/src/main/resources/web/properties/ExoneraTor_sv.properties index 8e649bd..53427b6 100644 --- a/src/main/resources/web/properties/ExoneraTor_sv.properties +++ b/src/main/resources/web/properties/ExoneraTor_sv.properties @@ -43,10 +43,9 @@ technicaldetails.exit.yes=Ja technicaldetails.exit.no=Nej permanentlink.heading=Permanent l\u00e4nk footer.abouttor.heading=Om Tor -footer.abouttor.body.text=Tor \u00e4r ett internationellt mjukvaruprojekt f\u00f6r att anonymisera internettrafik genom att %s. Om du ser trafik fr\u00e5n ett Tor-rel\u00e4 s\u00e5 kommer den trafiken oftast fr\u00e5n n\u00e5gon som anv\u00e4nder sig av Tor snarare \u00e4n fr\u00e5n sj\u00e4lva rel\u00e4operat\u00f6ren. Vare sig Tor-projektet eller Tor-rel\u00e4operat\u00f6rer har n\u00e5gon historik \u00f6ver trafiken som passerar genom n\u00e4tverket och kan d\u00e4rf\u00f6r inte ge n\u00e5gon information om dess ursprung. Se till att %s och tveka inte att %s f\u00f6r ytterligare information. -footer.abouttor.body.link1=kryptera datat och skicka det genom flera hopp innan det n\u00e5r sin destination -footer.abouttor.body.link2=l\u00e4sa p\u00e5 om Tor -footer.abouttor.body.link3=kontakta Tor-projektet +footer.abouttor.body.text=Tor \u00e4r ett internationellt mjukvaruprojekt f\u00f6r att anonymisera internettrafik genom att kryptera datat och skicka det genom flera hopp innan det n\u00e5r sin destination. Om du ser trafik fr\u00e5n ett Tor-rel\u00e4 s\u00e5 kommer den trafiken oftast fr\u00e5n n\u00e5gon som anv\u00e4nder sig av Tor snarare \u00e4n fr\u00e5n sj\u00e4lva rel\u00e4operat\u00f6ren. Vare sig Tor-projektet eller Tor-rel\u00e4operat\u00f6rer har n\u00e5gon historik \u00f6ver trafiken som passerar genom n\u00e4tverket och kan d\u00e4rf\u00f6r inte ge n\u00e5gon information om dess ursprung. Se till att %s och tveka inte att %s f\u00f6r ytterligare information. +footer.abouttor.body.link1=l\u00e4sa p\u00e5 om Tor +footer.abouttor.body.link2=kontakta Tor-projektet footer.aboutexonerator.heading=Om ExoneraTor footer.aboutexonerator.body=Tj\u00e4nsten ExoneraTor uppr\u00e4tth\u00e5ller en databas \u00f6ver IP-adresser som har varit en del av Tor-n\u00e4tverket. Den ger svar p\u00e5 fr\u00e5gan om det var ett Tor-rel\u00e4 som anv\u00e4nde en viss IP-adress vid ett givet datum. ExoneraTor kan komma att lagra fler \u00e4n en IP-adress per rel\u00e4 f\u00f6r rel\u00e4n som anv\u00e4nder en annan IP-adress f\u00f6r att ansluta till internet \u00e4n den som anv\u00e4ndes f\u00f6r att registrera sig i Tor-n\u00e4tverket. Huruvida ett rel\u00e4 har till\u00e5tit Tor-trafik ut mot internet vid tidpunkten eller ej lagras ocks\u00e5. footer.language.name=svenska

1 0

[metrics-web/release] Re-add no-data-available.png for display in RS.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit 1d21cebf4b999ced613f444c3278f1b53acf9480 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Apr 3 09:38:56 2019 +0200 Re-add no-data-available.png for display in RS. Reported on metrics-team@. --- src/main/resources/web/images/no-data-available.png | Bin 0 -> 26209 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/main/resources/web/images/no-data-available.png b/src/main/resources/web/images/no-data-available.png new file mode 100644 index 0000000..36310ad Binary files /dev/null and b/src/main/resources/web/images/no-data-available.png differ

1 0

[metrics-web/release] Simplify Rserve setup.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit e82de493279a0e74b55e5fd66a4056a1cecf19c5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jan 11 11:39:12 2019 +0100 Simplify Rserve setup. --- src/main/R/rserver/Rserv.conf | 2 - src/main/R/rserver/graphs.R | 1539 ------------------------------------ src/main/R/rserver/rserve-init.R | 1609 +++++++++++++++++++++++++++++++++++++- src/main/R/rserver/tables.R | 58 -- 4 files changed, 1600 insertions(+), 1608 deletions(-) diff --git a/src/main/R/rserver/Rserv.conf b/src/main/R/rserver/Rserv.conf deleted file mode 100644 index 1fb3039..0000000 --- a/src/main/R/rserver/Rserv.conf +++ /dev/null @@ -1,2 +0,0 @@ -workdir /srv/metrics.torproject.org/metrics/website/rserve/workdir -source rserve-init.R diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R deleted file mode 100644 index 0d7a90c..0000000 --- a/src/main/R/rserver/graphs.R +++ /dev/null @@ -1,1539 +0,0 @@ -countrylist <- list( - "ad" = "Andorra", - "ae" = "the United Arab Emirates", - "af" = "Afghanistan", - "ag" = "Antigua and Barbuda", - "ai" = "Anguilla", - "al" = "Albania", - "am" = "Armenia", - "an" = "the Netherlands Antilles", - "ao" = "Angola", - "aq" = "Antarctica", - "ar" = "Argentina", - "as" = "American Samoa", - "at" = "Austria", - "au" = "Australia", - "aw" = "Aruba", - "ax" = "the Aland Islands", - "az" = "Azerbaijan", - "ba" = "Bosnia and Herzegovina", - "bb" = "Barbados", - "bd" = "Bangladesh", - "be" = "Belgium", - "bf" = "Burkina Faso", - "bg" = "Bulgaria", - "bh" = "Bahrain", - "bi" = "Burundi", - "bj" = "Benin", - "bl" = "Saint Bartelemey", - "bm" = "Bermuda", - "bn" = "Brunei", - "bo" = "Bolivia", - "bq" = "Bonaire, Sint Eustatius and Saba", - "br" = "Brazil", - "bs" = "the Bahamas", - "bt" = "Bhutan", - "bv" = "the Bouvet Island", - "bw" = "Botswana", - "by" = "Belarus", - "bz" = "Belize", - "ca" = "Canada", - "cc" = "the Cocos (Keeling) Islands", - "cd" = "the Democratic Republic of the Congo", - "cf" = "Central African Republic", - "cg" = "Congo", - "ch" = "Switzerland", - "ci" = "Côte d'Ivoire", - "ck" = "the Cook Islands", - "cl" = "Chile", - "cm" = "Cameroon", - "cn" = "China", - "co" = "Colombia", - "cr" = "Costa Rica", - "cu" = "Cuba", - "cv" = "Cape Verde", - "cw" = "Curaçao", - "cx" = "the Christmas Island", - "cy" = "Cyprus", - "cz" = "the Czech Republic", - "de" = "Germany", - "dj" = "Djibouti", - "dk" = "Denmark", - "dm" = "Dominica", - "do" = "the Dominican Republic", - "dz" = "Algeria", - "ec" = "Ecuador", - "ee" = "Estonia", - "eg" = "Egypt", - "eh" = "the Western Sahara", - "er" = "Eritrea", - "es" = "Spain", - "et" = "Ethiopia", - "fi" = "Finland", - "fj" = "Fiji", - "fk" = "the Falkland Islands (Malvinas)", - "fm" = "the Federated States of Micronesia", - "fo" = "the Faroe Islands", - "fr" = "France", - "ga" = "Gabon", - "gb" = "the United Kingdom", - "gd" = "Grenada", - "ge" = "Georgia", - "gf" = "French Guiana", - "gg" = "Guernsey", - "gh" = "Ghana", - "gi" = "Gibraltar", - "gl" = "Greenland", - "gm" = "Gambia", - "gn" = "Guinea", - "gp" = "Guadeloupe", - "gq" = "Equatorial Guinea", - "gr" = "Greece", - "gs" = "South Georgia and the South Sandwich Islands", - "gt" = "Guatemala", - "gu" = "Guam", - "gw" = "Guinea-Bissau", - "gy" = "Guyana", - "hk" = "Hong Kong", - "hm" = "Heard Island and McDonald Islands", - "hn" = "Honduras", - "hr" = "Croatia", - "ht" = "Haiti", - "hu" = "Hungary", - "id" = "Indonesia", - "ie" = "Ireland", - "il" = "Israel", - "im" = "the Isle of Man", - "in" = "India", - "io" = "the British Indian Ocean Territory", - "iq" = "Iraq", - "ir" = "Iran", - "is" = "Iceland", - "it" = "Italy", - "je" = "Jersey", - "jm" = "Jamaica", - "jo" = "Jordan", - "jp" = "Japan", - "ke" = "Kenya", - "kg" = "Kyrgyzstan", - "kh" = "Cambodia", - "ki" = "Kiribati", - "km" = "Comoros", - "kn" = "Saint Kitts and Nevis", - "kp" = "North Korea", - "kr" = "the Republic of Korea", - "kw" = "Kuwait", - "ky" = "the Cayman Islands", - "kz" = "Kazakhstan", - "la" = "Laos", - "lb" = "Lebanon", - "lc" = "Saint Lucia", - "li" = "Liechtenstein", - "lk" = "Sri Lanka", - "lr" = "Liberia", - "ls" = "Lesotho", - "lt" = "Lithuania", - "lu" = "Luxembourg", - "lv" = "Latvia", - "ly" = "Libya", - "ma" = "Morocco", - "mc" = "Monaco", - "md" = "the Republic of Moldova", - "me" = "Montenegro", - "mf" = "Saint Martin", - "mg" = "Madagascar", - "mh" = "the Marshall Islands", - "mk" = "Macedonia", - "ml" = "Mali", - "mm" = "Burma", - "mn" = "Mongolia", - "mo" = "Macau", - "mp" = "the Northern Mariana Islands", - "mq" = "Martinique", - "mr" = "Mauritania", - "ms" = "Montserrat", - "mt" = "Malta", - "mu" = "Mauritius", - "mv" = "the Maldives", - "mw" = "Malawi", - "mx" = "Mexico", - "my" = "Malaysia", - "mz" = "Mozambique", - "na" = "Namibia", - "nc" = "New Caledonia", - "ne" = "Niger", - "nf" = "Norfolk Island", - "ng" = "Nigeria", - "ni" = "Nicaragua", - "nl" = "the Netherlands", - "no" = "Norway", - "np" = "Nepal", - "nr" = "Nauru", - "nu" = "Niue", - "nz" = "New Zealand", - "om" = "Oman", - "pa" = "Panama", - "pe" = "Peru", - "pf" = "French Polynesia", - "pg" = "Papua New Guinea", - "ph" = "the Philippines", - "pk" = "Pakistan", - "pl" = "Poland", - "pm" = "Saint Pierre and Miquelon", - "pn" = "the Pitcairn Islands", - "pr" = "Puerto Rico", - "ps" = "the Palestinian Territory", - "pt" = "Portugal", - "pw" = "Palau", - "py" = "Paraguay", - "qa" = "Qatar", - "re" = "Reunion", - "ro" = "Romania", - "rs" = "Serbia", - "ru" = "Russia", - "rw" = "Rwanda", - "sa" = "Saudi Arabia", - "sb" = "the Solomon Islands", - "sc" = "the Seychelles", - "sd" = "Sudan", - "se" = "Sweden", - "sg" = "Singapore", - "sh" = "Saint Helena", - "si" = "Slovenia", - "sj" = "Svalbard and Jan Mayen", - "sk" = "Slovakia", - "sl" = "Sierra Leone", - "sm" = "San Marino", - "sn" = "Senegal", - "so" = "Somalia", - "sr" = "Suriname", - "ss" = "South Sudan", - "st" = "São Tomé and Príncipe", - "sv" = "El Salvador", - "sx" = "Sint Maarten", - "sy" = "the Syrian Arab Republic", - "sz" = "Swaziland", - "tc" = "Turks and Caicos Islands", - "td" = "Chad", - "tf" = "the French Southern Territories", - "tg" = "Togo", - "th" = "Thailand", - "tj" = "Tajikistan", - "tk" = "Tokelau", - "tl" = "East Timor", - "tm" = "Turkmenistan", - "tn" = "Tunisia", - "to" = "Tonga", - "tr" = "Turkey", - "tt" = "Trinidad and Tobago", - "tv" = "Tuvalu", - "tw" = "Taiwan", - "tz" = "the United Republic of Tanzania", - "ua" = "Ukraine", - "ug" = "Uganda", - "um" = "the United States Minor Outlying Islands", - "us" = "the United States", - "uy" = "Uruguay", - "uz" = "Uzbekistan", - "va" = "Vatican City", - "vc" = "Saint Vincent and the Grenadines", - "ve" = "Venezuela", - "vg" = "the British Virgin Islands", - "vi" = "the United States Virgin Islands", - "vn" = "Vietnam", - "vu" = "Vanuatu", - "wf" = "Wallis and Futuna", - "ws" = "Samoa", - "xk" = "Kosovo", - "ye" = "Yemen", - "yt" = "Mayotte", - "za" = "South Africa", - "zm" = "Zambia", - "zw" = "Zimbabwe") - -countryname <- function(country) { - res <- countrylist[[country]] - if (is.null(res)) - res <- "no-man's-land" - res -} - -# Helper function that takes date limits as input and returns major breaks as -# output. The main difference to the built-in major breaks is that we're trying -# harder to align major breaks with first days of weeks (Sundays), months, -# quarters, or years. -custom_breaks <- function(input) { - scales_index <- cut(as.numeric(max(input) - min(input)), - c(-1, 7, 12, 56, 180, 600, 2000, Inf), labels = FALSE) - from_print_format <- c("%F", "%F", "%Y-W%U-7", "%Y-%m-01", "%Y-01-01", - "%Y-01-01", "%Y-01-01")[scales_index] - from_parse_format <- ifelse(scales_index == 3, "%Y-W%U-%u", "%F") - by <- c("1 day", "2 days", "1 week", "1 month", "3 months", "1 year", - "2 years")[scales_index] - seq(as.Date(as.character(min(input), from_print_format), - format = from_parse_format), max(input), by = by) -} - -# Helper function that takes date limits as input and returns minor breaks as -# output. As opposed to the built-in minor breaks, we're not just adding one -# minor break half way through between two major breaks. Instead, we're plotting -# a minor break for every day, week, month, or quarter between two major breaks. -custom_minor_breaks <- function(input) { - scales_index <- cut(as.numeric(max(input) - min(input)), - c(-1, 7, 12, 56, 180, 600, 2000, Inf), labels = FALSE) - from_print_format <- c("%F", "%F", "%F", "%Y-W%U-7", "%Y-%m-01", "%Y-01-01", - "%Y-01-01")[scales_index] - from_parse_format <- ifelse(scales_index == 4, "%Y-W%U-%u", "%F") - by <- c("1 day", "1 day", "1 day", "1 week", "1 month", "3 months", - "1 year")[scales_index] - seq(as.Date(as.character(min(input), from_print_format), - format = from_parse_format), max(input), by = by) -} - -# Helper function that takes breaks as input and returns labels as output. We're -# going all ISO-8601 here, though we're not just writing %Y-%m-%d everywhere, -# but %Y-%m or %Y if all breaks are on the first of a month or even year. -custom_labels <- function(breaks) { - if (all(format(breaks, format = "%m-%d") == "01-01", na.rm = TRUE)) { - format(breaks, format = "%Y") - } else { - if (all(format(breaks, format = "%d") == "01", na.rm = TRUE)) { - format(breaks, format = "%Y-%m") - } else { - format(breaks, format = "%F") - } - } -} - -# Helper function to format numbers in non-scientific notation with spaces as -# thousands separator. -formatter <- function(x, ...) { - format(x, ..., scientific = FALSE, big.mark = " ") -} - -theme_update( - # Make plot title centered, and leave some room to the plot. - plot.title = element_text(hjust = 0.5, margin = margin(b = 11)), - - # Leave a little more room to the right for long x axis labels. - plot.margin = margin(5.5, 11, 5.5, 5.5) -) - -# Set the default line size of geom_line() to 1. -update_geom_defaults("line", list(size = 1)) - -copyright_notice <- "The Tor Project - https://metrics.torproject.org/" - -stats_dir <- "/srv/metrics.torproject.org/metrics/shared/stats/" - -rdata_dir <- "/srv/metrics.torproject.org/metrics/shared/RData/" - -# Helper function that copies the appropriate no data object to filename. -copy_no_data <- function(filename) { - len <- nchar(filename) - extension <- substr(filename, len - 3, len) - if (".csv" == extension) { - write("# No data available for the given parameters.", file=filename) - } else { - file.copy(paste(rdata_dir, "no-data-available", extension, sep = ""), - filename) - } -} - -# Helper function wrapping calls into error handling. -robust_call <- function(wrappee, filename) { - tryCatch(eval(wrappee), error = function(e) copy_no_data(filename), - finally = if (!file.exists(filename) || file.size(filename) == 0) { - copy_no_data(filename) - }) -} - -# Write the result of the given FUN, typically a prepare_ function, as .csv file -# to the given path_p. -write_data <- function(FUN, ..., path_p) { - FUN(...) %>% - write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") -} - -# Disable readr's automatic progress bar. -options(readr.show_progress = FALSE) - -prepare_networksize <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "networksize.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - relays = col_double(), - bridges = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) -} - -plot_networksize <- function(start_p, end_p, path_p) { - prepare_networksize(start_p, end_p) %>% - gather(variable, value, -date) %>% - complete(date = full_seq(date, period = 1), - variable = c("relays", "bridges")) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_hue("", breaks = c("relays", "bridges"), - labels = c("Relays", "Bridges")) + - ggtitle("Number of relays") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_versions <- function(start_p = NULL, end_p = NULL) { - read_csv(paste(stats_dir, "versions.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - version = col_character(), - relays = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) -} - -plot_versions <- function(start_p, end_p, path_p) { - s <- prepare_versions(start_p, end_p) - known_versions <- unique(s$version) - getPalette <- colorRampPalette(brewer.pal(12, "Paired")) - colours <- data.frame(breaks = known_versions, - values = rep(brewer.pal(min(12, length(known_versions)), "Paired"), - len = length(known_versions)), - stringsAsFactors = FALSE) - versions <- s[s$version %in% known_versions, ] - visible_versions <- sort(unique(versions$version)) - versions <- versions %>% - complete(date = full_seq(date, period = 1), nesting(version)) %>% - ggplot(aes(x = date, y = relays, colour = version)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_manual(name = "Tor version", - values = colours[colours$breaks %in% visible_versions, 2], - breaks = visible_versions) + - ggtitle("Relay versions") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_platforms <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "platforms.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - platform = col_factor(levels = NULL), - relays = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - mutate(platform = tolower(platform)) %>% - spread(platform, relays) -} - -plot_platforms <- function(start_p, end_p, path_p) { - prepare_platforms(start_p, end_p) %>% - gather(platform, relays, -date) %>% - complete(date = full_seq(date, period = 1), nesting(platform)) %>% - ggplot(aes(x = date, y = relays, colour = platform)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_manual(name = "Platform", - breaks = c("linux", "macos", "bsd", "windows", "other"), - labels = c("Linux", "macOS", "BSD", "Windows", "Other"), - values = c("linux" = "#56B4E9", "macos" = "#333333", "bsd" = "#E69F00", - "windows" = "#0072B2", "other" = "#009E73")) + - ggtitle("Relay platforms") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_dirbytes <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "bandwidth.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - isexit = col_logical(), - isguard = col_logical(), - bwread = col_skip(), - bwwrite = col_skip(), - dirread = col_double(), - dirwrite = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(is.na(isexit)) %>% - filter(is.na(isguard)) %>% - mutate(dirread = dirread * 8 / 1e9, - dirwrite = dirwrite * 8 / 1e9) %>% - select(date, dirread, dirwrite) -} - -plot_dirbytes <- function(start_p, end_p, path_p) { - prepare_dirbytes(start_p, end_p) %>% - gather(variable, value, -date) %>% - complete(date = full_seq(date, period = 1), nesting(variable)) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_hue(name = "", - breaks = c("dirwrite", "dirread"), - labels = c("Written dir bytes", "Read dir bytes")) + - ggtitle("Number of bytes spent on answering directory requests") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_relayflags <- function(start_p = NULL, end_p = NULL, flag_p = NULL) { - read_csv(file = paste(stats_dir, "relayflags.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - flag = col_factor(levels = NULL), - relays = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(flag_p)) flag %in% flag_p else TRUE) -} - -plot_relayflags <- function(start_p, end_p, flag_p, path_p) { - prepare_relayflags(start_p, end_p, flag_p) %>% - complete(date = full_seq(date, period = 1), flag = unique(flag)) %>% - ggplot(aes(x = date, y = relays, colour = flag)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_manual(name = "Relay flags", values = c("#E69F00", - "#56B4E9", "#009E73", "#EE6A50", "#000000", "#0072B2"), - breaks = flag_p, labels = flag_p) + - ggtitle("Number of relays with relay flags assigned") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_torperf <- function(start_p = NULL, end_p = NULL, server_p = NULL, - filesize_p = NULL) { - read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - filesize = col_double(), - source = col_character(), - server = col_character(), - q1 = col_double(), - md = col_double(), - q3 = col_double(), - timeouts = col_skip(), - failures = col_skip(), - requests = col_skip())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(server_p)) server == server_p else TRUE) %>% - filter(if (!is.null(filesize_p)) - filesize == ifelse(filesize_p == "50kb", 50 * 1024, - ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else - TRUE) %>% - transmute(date, filesize, source, server, q1 = q1 / 1e3, md = md / 1e3, - q3 = q3 / 1e3) -} - -plot_torperf <- function(start_p, end_p, server_p, filesize_p, path_p) { - prepare_torperf(start_p, end_p, server_p, filesize_p) %>% - filter(source != "") %>% - complete(date = full_seq(date, period = 1), nesting(source)) %>% - ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + - geom_ribbon(alpha = 0.5) + - geom_line(aes(colour = source), size = 0.75) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "s"), - limits = c(0, NA)) + - scale_fill_hue(name = "Source") + - scale_colour_hue(name = "Source") + - ggtitle(paste("Time to complete", - ifelse(filesize_p == "50kb", "50 KiB", - ifelse(filesize_p == "1mb", "1 MiB", "5 MiB")), - "request to", server_p, "server")) + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_torperf_failures <- function(start_p = NULL, end_p = NULL, - server_p = NULL, filesize_p = NULL) { - read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - filesize = col_double(), - source = col_character(), - server = col_character(), - q1 = col_skip(), - md = col_skip(), - q3 = col_skip(), - timeouts = col_double(), - failures = col_double(), - requests = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(filesize_p)) - filesize == ifelse(filesize_p == "50kb", 50 * 1024, - ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else - TRUE) %>% - filter(if (!is.null(server_p)) server == server_p else TRUE) %>% - filter(requests > 0) %>% - transmute(date, filesize, source, server, timeouts = timeouts / requests, - failures = failures / requests) -} - -plot_torperf_failures <- function(start_p, end_p, server_p, filesize_p, - path_p) { - prepare_torperf_failures(start_p, end_p, server_p, filesize_p) %>% - filter(source != "") %>% - gather(variable, value, -c(date, filesize, source, server)) %>% - mutate(variable = factor(variable, levels = c("timeouts", "failures"), - labels = c("Timeouts", "Failures"))) %>% - ggplot(aes(x = date, y = value, colour = source)) + - geom_point(size = 2, alpha = 0.5) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) + - scale_colour_hue(name = "Source") + - facet_grid(variable ~ .) + - ggtitle(paste("Timeouts and failures of", - ifelse(filesize_p == "50kb", "50 KiB", - ifelse(filesize_p == "1mb", "1 MiB", "5 MiB")), - "requests to", server_p, "server")) + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_onionperf_buildtimes <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "buildtimes.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - source = col_character(), - position = col_double(), - q1 = col_double(), - md = col_double(), - q3 = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) -} - -plot_onionperf_buildtimes <- function(start_p, end_p, path_p) { - prepare_onionperf_buildtimes(start_p, end_p) %>% - filter(source != "") %>% - mutate(date = as.Date(date), - position = factor(position, levels = seq(1, 3, 1), - labels = c("1st hop", "2nd hop", "3rd hop"))) %>% - complete(date = full_seq(date, period = 1), nesting(source, position)) %>% - ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + - geom_ribbon(alpha = 0.5) + - geom_line(aes(colour = source), size = 0.75) + - facet_grid(position ~ .) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "ms"), - limits = c(0, NA)) + - scale_fill_hue(name = "Source") + - scale_colour_hue(name = "Source") + - ggtitle("Circuit build times") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_onionperf_latencies <- function(start_p = NULL, end_p = NULL, - server_p = NULL) { - read_csv(file = paste(stats_dir, "latencies.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - source = col_character(), - server = col_character(), - q1 = col_double(), - md = col_double(), - q3 = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(server_p)) server == server_p else TRUE) -} - -plot_onionperf_latencies <- function(start_p, end_p, server_p, path_p) { - prepare_onionperf_latencies(start_p, end_p, server_p) %>% - filter(source != "") %>% - complete(date = full_seq(date, period = 1), nesting(source)) %>% - ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + - geom_ribbon(alpha = 0.5) + - geom_line(aes(colour = source), size = 0.75) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "ms"), - limits = c(0, NA)) + - scale_fill_hue(name = "Source") + - scale_colour_hue(name = "Source") + - ggtitle(paste("Circuit round-trip latencies to", server_p, "server")) + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_connbidirect <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "connbidirect2.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - direction = col_factor(levels = NULL), - quantile = col_double(), - fraction = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - mutate(quantile = paste("X", quantile, sep = ""), - fraction = fraction / 100) %>% - spread(quantile, fraction) %>% - rename(q1 = X0.25, md = X0.5, q3 = X0.75) -} - -plot_connbidirect <- function(start_p, end_p, path_p) { - prepare_connbidirect(start_p, end_p) %>% - complete(date = full_seq(date, period = 1), nesting(direction)) %>% - ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = direction)) + - geom_ribbon(alpha = 0.5) + - geom_line(aes(colour = direction), size = 0.75) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) + - scale_colour_hue(name = "Medians and interquartile ranges", - breaks = c("both", "write", "read"), - labels = c("Both reading and writing", "Mostly writing", - "Mostly reading")) + - scale_fill_hue(name = "Medians and interquartile ranges", - breaks = c("both", "write", "read"), - labels = c("Both reading and writing", "Mostly writing", - "Mostly reading")) + - ggtitle("Fraction of connections used uni-/bidirectionally") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_bandwidth_flags <- function(start_p = NULL, end_p = NULL) { - advbw <- read_csv(file = paste(stats_dir, "advbw.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - isexit = col_logical(), - isguard = col_logical(), - advbw = col_double())) %>% - transmute(date, have_guard_flag = isguard, have_exit_flag = isexit, - variable = "advbw", value = advbw * 8 / 1e9) - bwhist <- read_csv(file = paste(stats_dir, "bandwidth.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - isexit = col_logical(), - isguard = col_logical(), - bwread = col_double(), - bwwrite = col_double(), - dirread = col_double(), - dirwrite = col_double())) %>% - transmute(date, have_guard_flag = isguard, have_exit_flag = isexit, - variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9) - rbind(advbw, bwhist) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(!is.na(have_exit_flag)) %>% - filter(!is.na(have_guard_flag)) %>% - spread(variable, value) -} - -plot_bandwidth_flags <- function(start_p, end_p, path_p) { - prepare_bandwidth_flags(start_p, end_p) %>% - gather(variable, value, c(advbw, bwhist)) %>% - unite(flags, have_guard_flag, have_exit_flag) %>% - mutate(flags = factor(flags, - levels = c("FALSE_TRUE", "TRUE_TRUE", "TRUE_FALSE", "FALSE_FALSE"), - labels = c("Exit only", "Guard and Exit", "Guard only", - "Neither Guard nor Exit"))) %>% - mutate(variable = ifelse(variable == "advbw", - "Advertised bandwidth", "Consumed bandwidth")) %>% - ggplot(aes(x = date, y = value, fill = flags)) + - geom_area() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_fill_manual(name = "", - values = c("#03B3FF", "#39FF02", "#FFFF00", "#AAAA99")) + - facet_grid(variable ~ .) + - ggtitle("Advertised and consumed bandwidth by relay flags") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_userstats_relay_country <- function(start_p = NULL, end_p = NULL, - country_p = NULL, events_p = NULL) { - read_csv(file = paste(stats_dir, "clients.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - node = col_character(), - country = col_character(), - transport = col_character(), - version = col_character(), - lower = col_double(), - upper = col_double(), - clients = col_double(), - frac = col_double()), - na = character()) %>% - filter(node == "relay") %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(country_p)) - country == ifelse(country_p == "all", "", country_p) else TRUE) %>% - filter(transport == "") %>% - filter(version == "") %>% - select(date, country, clients, lower, upper, frac) %>% - rename(users = clients) -} - -plot_userstats_relay_country <- function(start_p, end_p, country_p, events_p, - path_p) { - u <- prepare_userstats_relay_country(start_p, end_p, country_p, events_p) %>% - complete(date = full_seq(date, period = 1)) - plot <- ggplot(u, aes(x = date, y = users)) - if (length(na.omit(u$users)) > 0 & events_p != "off" & - country_p != "all") { - upturns <- u[u$users > u$upper, c("date", "users")] - downturns <- u[u$users < u$lower, c("date", "users")] - if (events_p == "on") { - u[!is.na(u$lower) & u$lower < 0, "lower"] <- 0 - plot <- plot + - geom_ribbon(data = u, aes(ymin = lower, ymax = upper), fill = "gray") - } - if (length(upturns$date) > 0) - plot <- plot + - geom_point(data = upturns, aes(x = date, y = users), size = 5, - colour = "dodgerblue2") - if (length(downturns$date) > 0) - plot <- plot + - geom_point(data = downturns, aes(x = date, y = users), size = 5, - colour = "firebrick2") - } - plot <- plot + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - ggtitle(paste("Directly connecting users", - ifelse(country_p == "all", "", - paste(" from", countryname(country_p))), sep = "")) + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_userstats_bridge_country <- function(start_p = NULL, end_p = NULL, - country_p = NULL) { - read_csv(file = paste(stats_dir, "clients.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - node = col_character(), - country = col_character(), - transport = col_character(), - version = col_character(), - lower = col_double(), - upper = col_double(), - clients = col_double(), - frac = col_double()), - na = character()) %>% - filter(node == "bridge") %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(country_p)) - country == ifelse(country_p == "all", "", country_p) else TRUE) %>% - filter(transport == "") %>% - filter(version == "") %>% - select(date, country, clients, frac) %>% - rename(users = clients) -} - -plot_userstats_bridge_country <- function(start_p, end_p, country_p, path_p) { - prepare_userstats_bridge_country(start_p, end_p, country_p) %>% - complete(date = full_seq(date, period = 1)) %>% - ggplot(aes(x = date, y = users)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - ggtitle(paste("Bridge users", - ifelse(country_p == "all", "", - paste(" from", countryname(country_p))), sep = "")) + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_userstats_bridge_transport <- function(start_p = NULL, end_p = NULL, - transport_p = NULL) { - u <- read_csv(file = paste(stats_dir, "clients.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - node = col_character(), - country = col_character(), - transport = col_character(), - version = col_character(), - lower = col_double(), - upper = col_double(), - clients = col_double(), - frac = col_double())) %>% - filter(node == "bridge") %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(is.na(country)) %>% - filter(is.na(version)) %>% - filter(!is.na(transport)) %>% - select(date, transport, clients, frac) - if (is.null(transport_p) || "!<OR>" %in% transport_p) { - n <- u %>% - filter(transport != "<OR>") %>% - group_by(date, frac) %>% - summarize(clients = sum(clients)) - u <- rbind(u, data.frame(date = n$date, transport = "!<OR>", - clients = n$clients, frac = n$frac)) - } - u %>% - filter(if (!is.null(transport_p)) transport %in% transport_p else TRUE) %>% - select(date, transport, clients, frac) %>% - rename(users = clients) %>% - arrange(date, transport) -} - -plot_userstats_bridge_transport <- function(start_p, end_p, transport_p, - path_p) { - if (length(transport_p) > 1) { - title <- paste("Bridge users by transport") - } else { - title <- paste("Bridge users using", - ifelse(transport_p == "<??>", "unknown pluggable transport(s)", - ifelse(transport_p == "<OR>", "default OR protocol", - ifelse(transport_p == "!<OR>", "any pluggable transport", - ifelse(transport_p == "fte", "FTE", - ifelse(transport_p == "websocket", "Flash proxy/websocket", - paste("transport", transport_p))))))) - } - u <- prepare_userstats_bridge_transport(start_p, end_p, transport_p) %>% - complete(date = full_seq(date, period = 1), nesting(transport)) - if (length(transport_p) > 1) { - plot <- ggplot(u, aes(x = date, y = users, colour = transport)) - } else { - plot <- ggplot(u, aes(x = date, y = users)) - } - plot <- plot + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - ggtitle(title) + - labs(caption = copyright_notice) - if (length(transport_p) > 1) { - plot <- plot + - scale_colour_hue(name = "", breaks = transport_p, - labels = ifelse(transport_p == "<??>", "Unknown PT", - ifelse(transport_p == "<OR>", "Default OR protocol", - ifelse(transport_p == "!<OR>", "Any PT", - ifelse(transport_p == "fte", "FTE", - ifelse(transport_p == "websocket", "Flash proxy/websocket", - transport_p)))))) - } - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_userstats_bridge_version <- function(start_p = NULL, end_p = NULL, - version_p = NULL) { - read_csv(file = paste(stats_dir, "clients.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - node = col_character(), - country = col_character(), - transport = col_character(), - version = col_character(), - lower = col_double(), - upper = col_double(), - clients = col_double(), - frac = col_double())) %>% - filter(node == "bridge") %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(is.na(country)) %>% - filter(is.na(transport)) %>% - filter(if (!is.null(version_p)) version == version_p else TRUE) %>% - select(date, version, clients, frac) %>% - rename(users = clients) -} - -plot_userstats_bridge_version <- function(start_p, end_p, version_p, path_p) { - prepare_userstats_bridge_version(start_p, end_p, version_p) %>% - complete(date = full_seq(date, period = 1)) %>% - ggplot(aes(x = date, y = users)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - ggtitle(paste("Bridge users using IP", version_p, sep = "")) + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_userstats_bridge_combined <- function(start_p = NULL, end_p = NULL, - country_p = NULL) { - if (!is.null(country_p) && country_p == "all") { - prepare_userstats_bridge_country(start_p, end_p, country_p) - } else { - read_csv(file = paste(stats_dir, "userstats-combined.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - node = col_skip(), - country = col_character(), - transport = col_character(), - version = col_skip(), - frac = col_double(), - low = col_double(), - high = col_double()), - na = character()) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(country_p)) country == country_p else TRUE) %>% - select(date, country, transport, low, high, frac) %>% - arrange(date, country, transport) - } -} - -plot_userstats_bridge_combined <- function(start_p, end_p, country_p, path_p) { - if (country_p == "all") { - plot_userstats_bridge_country(start_p, end_p, country_p, path_p) - } else { - top <- 3 - u <- prepare_userstats_bridge_combined(start_p, end_p, country_p) - a <- aggregate(list(mid = (u$high + u$low) / 2), - by = list(transport = u$transport), FUN = sum) - a <- a[order(a$mid, decreasing = TRUE)[1:top], ] - u <- u[u$transport %in% a$transport, ] %>% - complete(date = full_seq(date, period = 1), nesting(country, transport)) - title <- paste("Bridge users by transport from ", - countryname(country_p), sep = "") - ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high, - fill = transport)) + - geom_ribbon(alpha = 0.5, size = 0.5) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", limits = c(0, NA), labels = formatter) + - scale_colour_hue("Top-3 transports") + - scale_fill_hue("Top-3 transports") + - ggtitle(title) + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) - } -} - -prepare_advbwdist_perc <- function(start_p = NULL, end_p = NULL, p_p = NULL) { - read_csv(file = paste(stats_dir, "advbwdist.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - isexit = col_logical(), - relay = col_skip(), - percentile = col_integer(), - advbw = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(p_p)) percentile %in% as.numeric(p_p) else - percentile != "") %>% - transmute(date, percentile = as.factor(percentile), - variable = ifelse(is.na(isexit), "all", "exits"), - advbw = advbw * 8 / 1e9) %>% - spread(variable, advbw) %>% - rename(p = percentile) -} - -plot_advbwdist_perc <- function(start_p, end_p, p_p, path_p) { - prepare_advbwdist_perc(start_p, end_p, p_p) %>% - gather(variable, advbw, -c(date, p)) %>% - mutate(variable = ifelse(variable == "all", "All relays", - "Exits only")) %>% - complete(date = full_seq(date, period = 1), nesting(p, variable)) %>% - ggplot(aes(x = date, y = advbw, colour = p)) + - facet_grid(variable ~ .) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_hue(name = "Percentile") + - ggtitle("Advertised bandwidth distribution") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_advbwdist_relay <- function(start_p = NULL, end_p = NULL, n_p = NULL) { - read_csv(file = paste(stats_dir, "advbwdist.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - isexit = col_logical(), - relay = col_integer(), - percentile = col_skip(), - advbw = col_double())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(if (!is.null(n_p)) relay %in% as.numeric(n_p) else - relay != "") %>% - transmute(date, relay = as.factor(relay), - variable = ifelse(is.na(isexit), "all", "exits"), - advbw = advbw * 8 / 1e9) %>% - spread(variable, advbw) %>% - rename(n = relay) -} - -plot_advbwdist_relay <- function(start_p, end_p, n_p, path_p) { - prepare_advbwdist_relay(start_p, end_p, n_p) %>% - gather(variable, advbw, -c(date, n)) %>% - mutate(variable = ifelse(variable == "all", "All relays", - "Exits only")) %>% - complete(date = full_seq(date, period = 1), nesting(n, variable)) %>% - ggplot(aes(x = date, y = advbw, colour = n)) + - facet_grid(variable ~ .) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_hue(name = "n") + - ggtitle("Advertised bandwidth of n-th fastest relays") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_hidserv_dir_onions_seen <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - type = col_factor(levels = NULL), - wmean = col_skip(), - wmedian = col_skip(), - wiqm = col_double(), - frac = col_double(), - stats = col_skip())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(type == "dir-onions-seen") %>% - transmute(date, onions = ifelse(frac >= 0.01, wiqm, NA), frac) -} - -plot_hidserv_dir_onions_seen <- function(start_p, end_p, path_p) { - prepare_hidserv_dir_onions_seen(start_p, end_p) %>% - complete(date = full_seq(date, period = 1)) %>% - ggplot(aes(x = date, y = onions)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", limits = c(0, NA), labels = formatter) + - ggtitle("Unique .onion addresses") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_hidserv_rend_relayed_cells <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""), - col_types = cols( - date = col_date(format = ""), - type = col_factor(levels = NULL), - wmean = col_skip(), - wmedian = col_skip(), - wiqm = col_double(), - frac = col_double(), - stats = col_skip())) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(type == "rend-relayed-cells") %>% - transmute(date, - relayed = ifelse(frac >= 0.01, wiqm * 8 * 512 / (86400 * 1e9), NA), frac) -} - -plot_hidserv_rend_relayed_cells <- function(start_p, end_p, path_p) { - prepare_hidserv_rend_relayed_cells(start_p, end_p) %>% - complete(date = full_seq(date, period = 1)) %>% - ggplot(aes(x = date, y = relayed)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - ggtitle("Onion-service traffic") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_webstats_tb <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), - col_types = cols( - log_date = col_date(format = ""), - request_type = col_factor(levels = NULL), - platform = col_skip(), - channel = col_skip(), - locale = col_skip(), - incremental = col_skip(), - count = col_double())) %>% - filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type %in% c("tbid", "tbsd", "tbup", "tbur")) %>% - group_by(log_date, request_type) %>% - summarize(count = sum(count)) %>% - spread(request_type, count) %>% - rename(date = log_date, initial_downloads = tbid, - signature_downloads = tbsd, update_pings = tbup, - update_requests = tbur) -} - -plot_webstats_tb <- function(start_p, end_p, path_p) { - prepare_webstats_tb(start_p, end_p) %>% - gather(request_type, count, -date) %>% - mutate(request_type = factor(request_type, - levels = c("initial_downloads", "signature_downloads", "update_pings", - "update_requests"), - labels = c("Initial downloads", "Signature downloads", "Update pings", - "Update requests"))) %>% - ungroup() %>% - complete(date = full_seq(date, period = 1), nesting(request_type)) %>% - ggplot(aes(x = date, y = count)) + - geom_point() + - geom_line() + - facet_grid(request_type ~ ., scales = "free_y") + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA)) + - ggtitle("Tor Browser downloads and updates") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_webstats_tb_platform <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), - col_types = cols( - log_date = col_date(format = ""), - request_type = col_factor(levels = NULL), - platform = col_factor(levels = NULL), - channel = col_skip(), - locale = col_skip(), - incremental = col_skip(), - count = col_double())) %>% - filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type %in% c("tbid", "tbup")) %>% - group_by(log_date, platform, request_type) %>% - summarize(count = sum(count)) %>% - spread(request_type, count, fill = 0) %>% - rename(date = log_date, initial_downloads = tbid, update_pings = tbup) -} - -plot_webstats_tb_platform <- function(start_p, end_p, path_p) { - prepare_webstats_tb_platform(start_p, end_p) %>% - gather(request_type, count, -c(date, platform)) %>% - mutate(request_type = factor(request_type, - levels = c("initial_downloads", "update_pings"), - labels = c("Initial downloads", "Update pings"))) %>% - ungroup() %>% - complete(date = full_seq(date, period = 1), - nesting(platform, request_type)) %>% - ggplot(aes(x = date, y = count, colour = platform)) + - geom_point() + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_hue(name = "Platform", - breaks = c("w", "m", "l", "o", ""), - labels = c("Windows", "macOS", "Linux", "Other", "Unknown")) + - facet_grid(request_type ~ ., scales = "free_y") + - theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA), - legend.position = "top") + - ggtitle("Tor Browser downloads and updates by platform") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_webstats_tb_locale <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), - col_types = cols( - log_date = col_date(format = ""), - request_type = col_factor(levels = NULL), - platform = col_skip(), - channel = col_skip(), - locale = col_factor(levels = NULL), - incremental = col_skip(), - count = col_double())) %>% - filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type %in% c("tbid", "tbup")) %>% - rename(date = log_date) %>% - group_by(date, locale, request_type) %>% - summarize(count = sum(count)) %>% - mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>% - spread(request_type, count, fill = 0) %>% - rename(initial_downloads = tbid, update_pings = tbup) -} - -plot_webstats_tb_locale <- function(start_p, end_p, path_p) { - d <- prepare_webstats_tb_locale(start_p, end_p) %>% - gather(request_type, count, -c(date, locale)) %>% - mutate(request_type = factor(request_type, - levels = c("initial_downloads", "update_pings"), - labels = c("Initial downloads", "Update pings"))) - e <- d - e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum) - e <- e[order(e$count, decreasing = TRUE), ] - e <- e[1:5, ] - d <- aggregate(list(count = d$count), by = list(date = d$date, - request_type = d$request_type, - locale = ifelse(d$locale %in% e$locale, d$locale, "(other)")), FUN = sum) - d %>% - complete(date = full_seq(date, period = 1), - nesting(locale, request_type)) %>% - ggplot(aes(x = date, y = count, colour = locale)) + - geom_point() + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_hue(name = "Locale", - breaks = c(e$locale, "(other)"), - labels = c(as.character(e$locale), "Other")) + - facet_grid(request_type ~ ., scales = "free_y") + - theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA), - legend.position = "top") + - guides(col = guide_legend(nrow = 1)) + - ggtitle("Tor Browser downloads and updates by locale") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), - col_types = cols( - log_date = col_date(format = ""), - request_type = col_factor(levels = NULL), - platform = col_skip(), - channel = col_skip(), - locale = col_skip(), - incremental = col_skip(), - count = col_double())) %>% - filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type %in% c("tmid", "tmup")) %>% - group_by(log_date, request_type) %>% - summarize(count = sum(count)) %>% - mutate(request_type = factor(request_type, levels = c("tmid", "tmup"))) %>% - spread(request_type, count, drop = FALSE, fill = 0) %>% - rename(date = log_date, initial_downloads = tmid, update_pings = tmup) -} - -plot_webstats_tm <- function(start_p, end_p, path_p) { - prepare_webstats_tm(start_p, end_p) %>% - gather(request_type, count, -date) %>% - mutate(request_type = factor(request_type, - levels = c("initial_downloads", "update_pings"), - labels = c("Initial downloads", "Update pings"))) %>% - ungroup() %>% - complete(date = full_seq(date, period = 1), nesting(request_type)) %>% - ggplot(aes(x = date, y = count)) + - geom_point() + - geom_line() + - facet_grid(request_type ~ ., scales = "free_y") + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA)) + - ggtitle("Tor Messenger downloads and updates") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_relays_ipv6 <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), - col_types = cols( - valid_after_date = col_date(format = ""), - server = col_factor(levels = NULL), - guard_relay = col_skip(), - exit_relay = col_skip(), - announced_ipv6 = col_logical(), - exiting_ipv6_relay = col_logical(), - reachable_ipv6_relay = col_logical(), - server_count_sum_avg = col_double(), - advertised_bandwidth_bytes_sum_avg = col_skip())) %>% - filter(if (!is.null(start_p)) - valid_after_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) - valid_after_date <= as.Date(end_p) else TRUE) %>% - filter(server == "relay") %>% - group_by(valid_after_date) %>% - summarize(total = sum(server_count_sum_avg), - announced = sum(server_count_sum_avg[announced_ipv6]), - reachable = sum(server_count_sum_avg[reachable_ipv6_relay]), - exiting = sum(server_count_sum_avg[exiting_ipv6_relay])) %>% - rename(date = valid_after_date) -} - -plot_relays_ipv6 <- function(start_p, end_p, path_p) { - prepare_relays_ipv6(start_p, end_p) %>% - complete(date = full_seq(date, period = 1)) %>% - gather(category, count, -date) %>% - ggplot(aes(x = date, y = count, colour = category)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_hue(name = "", h.start = 90, - breaks = c("total", "announced", "reachable", "exiting"), - labels = c("Total (IPv4) OR", "IPv6 announced OR", "IPv6 reachable OR", - "IPv6 exiting")) + - ggtitle("Relays by IP version") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_bridges_ipv6 <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), - col_types = cols( - valid_after_date = col_date(format = ""), - server = col_factor(levels = NULL), - guard_relay = col_skip(), - exit_relay = col_skip(), - announced_ipv6 = col_logical(), - exiting_ipv6_relay = col_skip(), - reachable_ipv6_relay = col_skip(), - server_count_sum_avg = col_double(), - advertised_bandwidth_bytes_sum_avg = col_skip())) %>% - filter(if (!is.null(start_p)) - valid_after_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) - valid_after_date <= as.Date(end_p) else TRUE) %>% - filter(server == "bridge") %>% - group_by(valid_after_date) %>% - summarize(total = sum(server_count_sum_avg), - announced = sum(server_count_sum_avg[announced_ipv6])) %>% - rename(date = valid_after_date) -} - -plot_bridges_ipv6 <- function(start_p, end_p, path_p) { - prepare_bridges_ipv6(start_p, end_p) %>% - complete(date = full_seq(date, period = 1)) %>% - gather(category, count, -date) %>% - ggplot(aes(x = date, y = count, colour = category)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_hue(name = "", h.start = 90, - breaks = c("total", "announced"), - labels = c("Total (IPv4) OR", "IPv6 announced OR")) + - ggtitle("Bridges by IP version") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_advbw_ipv6 <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), - col_types = cols( - valid_after_date = col_date(format = ""), - server = col_factor(levels = NULL), - guard_relay = col_logical(), - exit_relay = col_logical(), - announced_ipv6 = col_logical(), - exiting_ipv6_relay = col_logical(), - reachable_ipv6_relay = col_logical(), - server_count_sum_avg = col_skip(), - advertised_bandwidth_bytes_sum_avg = col_double())) %>% - filter(if (!is.null(start_p)) - valid_after_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) - valid_after_date <= as.Date(end_p) else TRUE) %>% - filter(server == "relay") %>% - mutate(advertised_bandwidth_bytes_sum_avg = - advertised_bandwidth_bytes_sum_avg * 8 / 1e9) %>% - group_by(valid_after_date) %>% - summarize(total = sum(advertised_bandwidth_bytes_sum_avg), - total_guard = sum(advertised_bandwidth_bytes_sum_avg[guard_relay]), - total_exit = sum(advertised_bandwidth_bytes_sum_avg[exit_relay]), - reachable_guard = sum(advertised_bandwidth_bytes_sum_avg[ - reachable_ipv6_relay & guard_relay]), - reachable_exit = sum(advertised_bandwidth_bytes_sum_avg[ - reachable_ipv6_relay & exit_relay]), - exiting = sum(advertised_bandwidth_bytes_sum_avg[ - exiting_ipv6_relay])) %>% - rename(date = valid_after_date) -} - -plot_advbw_ipv6 <- function(start_p, end_p, path_p) { - prepare_advbw_ipv6(start_p, end_p) %>% - complete(date = full_seq(date, period = 1)) %>% - gather(category, advbw, -date) %>% - ggplot(aes(x = date, y = advbw, colour = category)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_hue(name = "", h.start = 90, - breaks = c("total", "total_guard", "total_exit", "reachable_guard", - "reachable_exit", "exiting"), - labels = c("Total (IPv4) OR", "Guard total (IPv4)", "Exit total (IPv4)", - "Reachable guard IPv6 OR", "Reachable exit IPv6 OR", "IPv6 exiting")) + - ggtitle("Advertised bandwidth by IP version") + - labs(caption = copyright_notice) + - theme(legend.position = "top") + - guides(colour = guide_legend(nrow = 2, byrow = TRUE)) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -prepare_totalcw <- function(start_p = NULL, end_p = NULL) { - read_csv(file = paste(stats_dir, "totalcw.csv", sep = ""), - col_types = cols( - valid_after_date = col_date(format = ""), - nickname = col_character(), - have_guard_flag = col_logical(), - have_exit_flag = col_logical(), - measured_sum_avg = col_double())) %>% - filter(if (!is.null(start_p)) - valid_after_date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) - valid_after_date <= as.Date(end_p) else TRUE) %>% - group_by(valid_after_date, nickname) %>% - summarize(measured_sum_avg = sum(measured_sum_avg)) %>% - rename(date = valid_after_date, totalcw = measured_sum_avg) %>% - arrange(date, nickname) -} - -plot_totalcw <- function(start_p, end_p, path_p) { - prepare_totalcw(start_p, end_p) %>% - mutate(nickname = ifelse(is.na(nickname), "consensus", nickname)) %>% - mutate(nickname = factor(nickname, - levels = c("consensus", unique(nickname[nickname != "consensus"])))) %>% - ungroup() %>% - complete(date = full_seq(date, period = 1), nesting(nickname)) %>% - ggplot(aes(x = date, y = totalcw, colour = nickname)) + - geom_line(na.rm = TRUE) + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + - scale_colour_hue(name = "") + - ggtitle("Total consensus weights across bandwidth authorities") + - labs(caption = copyright_notice) - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - - diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index f160698..57e14f5 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -1,12 +1,1603 @@ -##Pre-loaded libraries and graphing functions to speed things up +require(ggplot2) +require(RColorBrewer) +require(scales) +require(dplyr) +require(tidyr) +require(readr) -library("ggplot2") -library("RColorBrewer") -library("scales") -library(dplyr) -library(tidyr) -library(readr) +countrylist <- list( + "ad" = "Andorra", + "ae" = "the United Arab Emirates", + "af" = "Afghanistan", + "ag" = "Antigua and Barbuda", + "ai" = "Anguilla", + "al" = "Albania", + "am" = "Armenia", + "an" = "the Netherlands Antilles", + "ao" = "Angola", + "aq" = "Antarctica", + "ar" = "Argentina", + "as" = "American Samoa", + "at" = "Austria", + "au" = "Australia", + "aw" = "Aruba", + "ax" = "the Aland Islands", + "az" = "Azerbaijan", + "ba" = "Bosnia and Herzegovina", + "bb" = "Barbados", + "bd" = "Bangladesh", + "be" = "Belgium", + "bf" = "Burkina Faso", + "bg" = "Bulgaria", + "bh" = "Bahrain", + "bi" = "Burundi", + "bj" = "Benin", + "bl" = "Saint Bartelemey", + "bm" = "Bermuda", + "bn" = "Brunei", + "bo" = "Bolivia", + "bq" = "Bonaire, Sint Eustatius and Saba", + "br" = "Brazil", + "bs" = "the Bahamas", + "bt" = "Bhutan", + "bv" = "the Bouvet Island", + "bw" = "Botswana", + "by" = "Belarus", + "bz" = "Belize", + "ca" = "Canada", + "cc" = "the Cocos (Keeling) Islands", + "cd" = "the Democratic Republic of the Congo", + "cf" = "Central African Republic", + "cg" = "Congo", + "ch" = "Switzerland", + "ci" = "Côte d'Ivoire", + "ck" = "the Cook Islands", + "cl" = "Chile", + "cm" = "Cameroon", + "cn" = "China", + "co" = "Colombia", + "cr" = "Costa Rica", + "cu" = "Cuba", + "cv" = "Cape Verde", + "cw" = "Curaçao", + "cx" = "the Christmas Island", + "cy" = "Cyprus", + "cz" = "the Czech Republic", + "de" = "Germany", + "dj" = "Djibouti", + "dk" = "Denmark", + "dm" = "Dominica", + "do" = "the Dominican Republic", + "dz" = "Algeria", + "ec" = "Ecuador", + "ee" = "Estonia", + "eg" = "Egypt", + "eh" = "the Western Sahara", + "er" = "Eritrea", + "es" = "Spain", + "et" = "Ethiopia", + "fi" = "Finland", + "fj" = "Fiji", + "fk" = "the Falkland Islands (Malvinas)", + "fm" = "the Federated States of Micronesia", + "fo" = "the Faroe Islands", + "fr" = "France", + "ga" = "Gabon", + "gb" = "the United Kingdom", + "gd" = "Grenada", + "ge" = "Georgia", + "gf" = "French Guiana", + "gg" = "Guernsey", + "gh" = "Ghana", + "gi" = "Gibraltar", + "gl" = "Greenland", + "gm" = "Gambia", + "gn" = "Guinea", + "gp" = "Guadeloupe", + "gq" = "Equatorial Guinea", + "gr" = "Greece", + "gs" = "South Georgia and the South Sandwich Islands", + "gt" = "Guatemala", + "gu" = "Guam", + "gw" = "Guinea-Bissau", + "gy" = "Guyana", + "hk" = "Hong Kong", + "hm" = "Heard Island and McDonald Islands", + "hn" = "Honduras", + "hr" = "Croatia", + "ht" = "Haiti", + "hu" = "Hungary", + "id" = "Indonesia", + "ie" = "Ireland", + "il" = "Israel", + "im" = "the Isle of Man", + "in" = "India", + "io" = "the British Indian Ocean Territory", + "iq" = "Iraq", + "ir" = "Iran", + "is" = "Iceland", + "it" = "Italy", + "je" = "Jersey", + "jm" = "Jamaica", + "jo" = "Jordan", + "jp" = "Japan", + "ke" = "Kenya", + "kg" = "Kyrgyzstan", + "kh" = "Cambodia", + "ki" = "Kiribati", + "km" = "Comoros", + "kn" = "Saint Kitts and Nevis", + "kp" = "North Korea", + "kr" = "the Republic of Korea", + "kw" = "Kuwait", + "ky" = "the Cayman Islands", + "kz" = "Kazakhstan", + "la" = "Laos", + "lb" = "Lebanon", + "lc" = "Saint Lucia", + "li" = "Liechtenstein", + "lk" = "Sri Lanka", + "lr" = "Liberia", + "ls" = "Lesotho", + "lt" = "Lithuania", + "lu" = "Luxembourg", + "lv" = "Latvia", + "ly" = "Libya", + "ma" = "Morocco", + "mc" = "Monaco", + "md" = "the Republic of Moldova", + "me" = "Montenegro", + "mf" = "Saint Martin", + "mg" = "Madagascar", + "mh" = "the Marshall Islands", + "mk" = "Macedonia", + "ml" = "Mali", + "mm" = "Burma", + "mn" = "Mongolia", + "mo" = "Macau", + "mp" = "the Northern Mariana Islands", + "mq" = "Martinique", + "mr" = "Mauritania", + "ms" = "Montserrat", + "mt" = "Malta", + "mu" = "Mauritius", + "mv" = "the Maldives", + "mw" = "Malawi", + "mx" = "Mexico", + "my" = "Malaysia", + "mz" = "Mozambique", + "na" = "Namibia", + "nc" = "New Caledonia", + "ne" = "Niger", + "nf" = "Norfolk Island", + "ng" = "Nigeria", + "ni" = "Nicaragua", + "nl" = "the Netherlands", + "no" = "Norway", + "np" = "Nepal", + "nr" = "Nauru", + "nu" = "Niue", + "nz" = "New Zealand", + "om" = "Oman", + "pa" = "Panama", + "pe" = "Peru", + "pf" = "French Polynesia", + "pg" = "Papua New Guinea", + "ph" = "the Philippines", + "pk" = "Pakistan", + "pl" = "Poland", + "pm" = "Saint Pierre and Miquelon", + "pn" = "the Pitcairn Islands", + "pr" = "Puerto Rico", + "ps" = "the Palestinian Territory", + "pt" = "Portugal", + "pw" = "Palau", + "py" = "Paraguay", + "qa" = "Qatar", + "re" = "Reunion", + "ro" = "Romania", + "rs" = "Serbia", + "ru" = "Russia", + "rw" = "Rwanda", + "sa" = "Saudi Arabia", + "sb" = "the Solomon Islands", + "sc" = "the Seychelles", + "sd" = "Sudan", + "se" = "Sweden", + "sg" = "Singapore", + "sh" = "Saint Helena", + "si" = "Slovenia", + "sj" = "Svalbard and Jan Mayen", + "sk" = "Slovakia", + "sl" = "Sierra Leone", + "sm" = "San Marino", + "sn" = "Senegal", + "so" = "Somalia", + "sr" = "Suriname", + "ss" = "South Sudan", + "st" = "São Tomé and Príncipe", + "sv" = "El Salvador", + "sx" = "Sint Maarten", + "sy" = "the Syrian Arab Republic", + "sz" = "Swaziland", + "tc" = "Turks and Caicos Islands", + "td" = "Chad", + "tf" = "the French Southern Territories", + "tg" = "Togo", + "th" = "Thailand", + "tj" = "Tajikistan", + "tk" = "Tokelau", + "tl" = "East Timor", + "tm" = "Turkmenistan", + "tn" = "Tunisia", + "to" = "Tonga", + "tr" = "Turkey", + "tt" = "Trinidad and Tobago", + "tv" = "Tuvalu", + "tw" = "Taiwan", + "tz" = "the United Republic of Tanzania", + "ua" = "Ukraine", + "ug" = "Uganda", + "um" = "the United States Minor Outlying Islands", + "us" = "the United States", + "uy" = "Uruguay", + "uz" = "Uzbekistan", + "va" = "Vatican City", + "vc" = "Saint Vincent and the Grenadines", + "ve" = "Venezuela", + "vg" = "the British Virgin Islands", + "vi" = "the United States Virgin Islands", + "vn" = "Vietnam", + "vu" = "Vanuatu", + "wf" = "Wallis and Futuna", + "ws" = "Samoa", + "xk" = "Kosovo", + "ye" = "Yemen", + "yt" = "Mayotte", + "za" = "South Africa", + "zm" = "Zambia", + "zw" = "Zimbabwe") -source('graphs.R') -source('tables.R') +countryname <- function(country) { + res <- countrylist[[country]] + if (is.null(res)) + res <- "no-man's-land" + res +} + +# Helper function that takes date limits as input and returns major breaks as +# output. The main difference to the built-in major breaks is that we're trying +# harder to align major breaks with first days of weeks (Sundays), months, +# quarters, or years. +custom_breaks <- function(input) { + scales_index <- cut(as.numeric(max(input) - min(input)), + c(-1, 7, 12, 56, 180, 600, 2000, Inf), labels = FALSE) + from_print_format <- c("%F", "%F", "%Y-W%U-7", "%Y-%m-01", "%Y-01-01", + "%Y-01-01", "%Y-01-01")[scales_index] + from_parse_format <- ifelse(scales_index == 3, "%Y-W%U-%u", "%F") + by <- c("1 day", "2 days", "1 week", "1 month", "3 months", "1 year", + "2 years")[scales_index] + seq(as.Date(as.character(min(input), from_print_format), + format = from_parse_format), max(input), by = by) +} + +# Helper function that takes date limits as input and returns minor breaks as +# output. As opposed to the built-in minor breaks, we're not just adding one +# minor break half way through between two major breaks. Instead, we're plotting +# a minor break for every day, week, month, or quarter between two major breaks. +custom_minor_breaks <- function(input) { + scales_index <- cut(as.numeric(max(input) - min(input)), + c(-1, 7, 12, 56, 180, 600, 2000, Inf), labels = FALSE) + from_print_format <- c("%F", "%F", "%F", "%Y-W%U-7", "%Y-%m-01", "%Y-01-01", + "%Y-01-01")[scales_index] + from_parse_format <- ifelse(scales_index == 4, "%Y-W%U-%u", "%F") + by <- c("1 day", "1 day", "1 day", "1 week", "1 month", "3 months", + "1 year")[scales_index] + seq(as.Date(as.character(min(input), from_print_format), + format = from_parse_format), max(input), by = by) +} + +# Helper function that takes breaks as input and returns labels as output. We're +# going all ISO-8601 here, though we're not just writing %Y-%m-%d everywhere, +# but %Y-%m or %Y if all breaks are on the first of a month or even year. +custom_labels <- function(breaks) { + if (all(format(breaks, format = "%m-%d") == "01-01", na.rm = TRUE)) { + format(breaks, format = "%Y") + } else { + if (all(format(breaks, format = "%d") == "01", na.rm = TRUE)) { + format(breaks, format = "%Y-%m") + } else { + format(breaks, format = "%F") + } + } +} + +# Helper function to format numbers in non-scientific notation with spaces as +# thousands separator. +formatter <- function(x, ...) { + format(x, ..., scientific = FALSE, big.mark = " ") +} + +theme_update( + # Make plot title centered, and leave some room to the plot. + plot.title = element_text(hjust = 0.5, margin = margin(b = 11)), + + # Leave a little more room to the right for long x axis labels. + plot.margin = margin(5.5, 11, 5.5, 5.5) +) + +# Set the default line size of geom_line() to 1. +update_geom_defaults("line", list(size = 1)) + +copyright_notice <- "The Tor Project - https://metrics.torproject.org/" + +stats_dir <- "/srv/metrics.torproject.org/metrics/shared/stats/" + +rdata_dir <- "/srv/metrics.torproject.org/metrics/shared/RData/" + +# Helper function that copies the appropriate no data object to filename. +copy_no_data <- function(filename) { + len <- nchar(filename) + extension <- substr(filename, len - 3, len) + if (".csv" == extension) { + write("# No data available for the given parameters.", file=filename) + } else { + file.copy(paste(rdata_dir, "no-data-available", extension, sep = ""), + filename) + } +} + +# Helper function wrapping calls into error handling. +robust_call <- function(wrappee, filename) { + tryCatch(eval(wrappee), error = function(e) copy_no_data(filename), + finally = if (!file.exists(filename) || file.size(filename) == 0) { + copy_no_data(filename) + }) +} + +# Write the result of the given FUN, typically a prepare_ function, as .csv file +# to the given path_p. +write_data <- function(FUN, ..., path_p) { + FUN(...) %>% + write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") +} + +# Disable readr's automatic progress bar. +options(readr.show_progress = FALSE) + +prepare_networksize <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "networksize.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + relays = col_double(), + bridges = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) +} + +plot_networksize <- function(start_p, end_p, path_p) { + prepare_networksize(start_p, end_p) %>% + gather(variable, value, -date) %>% + complete(date = full_seq(date, period = 1), + variable = c("relays", "bridges")) %>% + ggplot(aes(x = date, y = value, colour = variable)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_hue("", breaks = c("relays", "bridges"), + labels = c("Relays", "Bridges")) + + ggtitle("Number of relays") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_versions <- function(start_p = NULL, end_p = NULL) { + read_csv(paste(stats_dir, "versions.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + version = col_character(), + relays = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) +} + +plot_versions <- function(start_p, end_p, path_p) { + s <- prepare_versions(start_p, end_p) + known_versions <- unique(s$version) + getPalette <- colorRampPalette(brewer.pal(12, "Paired")) + colours <- data.frame(breaks = known_versions, + values = rep(brewer.pal(min(12, length(known_versions)), "Paired"), + len = length(known_versions)), + stringsAsFactors = FALSE) + versions <- s[s$version %in% known_versions, ] + visible_versions <- sort(unique(versions$version)) + versions <- versions %>% + complete(date = full_seq(date, period = 1), nesting(version)) %>% + ggplot(aes(x = date, y = relays, colour = version)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_manual(name = "Tor version", + values = colours[colours$breaks %in% visible_versions, 2], + breaks = visible_versions) + + ggtitle("Relay versions") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_platforms <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "platforms.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + platform = col_factor(levels = NULL), + relays = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + mutate(platform = tolower(platform)) %>% + spread(platform, relays) +} + +plot_platforms <- function(start_p, end_p, path_p) { + prepare_platforms(start_p, end_p) %>% + gather(platform, relays, -date) %>% + complete(date = full_seq(date, period = 1), nesting(platform)) %>% + ggplot(aes(x = date, y = relays, colour = platform)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_manual(name = "Platform", + breaks = c("linux", "macos", "bsd", "windows", "other"), + labels = c("Linux", "macOS", "BSD", "Windows", "Other"), + values = c("linux" = "#56B4E9", "macos" = "#333333", "bsd" = "#E69F00", + "windows" = "#0072B2", "other" = "#009E73")) + + ggtitle("Relay platforms") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_dirbytes <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "bandwidth.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + isexit = col_logical(), + isguard = col_logical(), + bwread = col_skip(), + bwwrite = col_skip(), + dirread = col_double(), + dirwrite = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(is.na(isexit)) %>% + filter(is.na(isguard)) %>% + mutate(dirread = dirread * 8 / 1e9, + dirwrite = dirwrite * 8 / 1e9) %>% + select(date, dirread, dirwrite) +} + +plot_dirbytes <- function(start_p, end_p, path_p) { + prepare_dirbytes(start_p, end_p) %>% + gather(variable, value, -date) %>% + complete(date = full_seq(date, period = 1), nesting(variable)) %>% + ggplot(aes(x = date, y = value, colour = variable)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_colour_hue(name = "", + breaks = c("dirwrite", "dirread"), + labels = c("Written dir bytes", "Read dir bytes")) + + ggtitle("Number of bytes spent on answering directory requests") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_relayflags <- function(start_p = NULL, end_p = NULL, flag_p = NULL) { + read_csv(file = paste(stats_dir, "relayflags.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + flag = col_factor(levels = NULL), + relays = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(flag_p)) flag %in% flag_p else TRUE) +} + +plot_relayflags <- function(start_p, end_p, flag_p, path_p) { + prepare_relayflags(start_p, end_p, flag_p) %>% + complete(date = full_seq(date, period = 1), flag = unique(flag)) %>% + ggplot(aes(x = date, y = relays, colour = flag)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_manual(name = "Relay flags", values = c("#E69F00", + "#56B4E9", "#009E73", "#EE6A50", "#000000", "#0072B2"), + breaks = flag_p, labels = flag_p) + + ggtitle("Number of relays with relay flags assigned") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_torperf <- function(start_p = NULL, end_p = NULL, server_p = NULL, + filesize_p = NULL) { + read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + filesize = col_double(), + source = col_character(), + server = col_character(), + q1 = col_double(), + md = col_double(), + q3 = col_double(), + timeouts = col_skip(), + failures = col_skip(), + requests = col_skip())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(server_p)) server == server_p else TRUE) %>% + filter(if (!is.null(filesize_p)) + filesize == ifelse(filesize_p == "50kb", 50 * 1024, + ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else + TRUE) %>% + transmute(date, filesize, source, server, q1 = q1 / 1e3, md = md / 1e3, + q3 = q3 / 1e3) +} + +plot_torperf <- function(start_p, end_p, server_p, filesize_p, path_p) { + prepare_torperf(start_p, end_p, server_p, filesize_p) %>% + filter(source != "") %>% + complete(date = full_seq(date, period = 1), nesting(source)) %>% + ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + + geom_ribbon(alpha = 0.5) + + geom_line(aes(colour = source), size = 0.75) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "s"), + limits = c(0, NA)) + + scale_fill_hue(name = "Source") + + scale_colour_hue(name = "Source") + + ggtitle(paste("Time to complete", + ifelse(filesize_p == "50kb", "50 KiB", + ifelse(filesize_p == "1mb", "1 MiB", "5 MiB")), + "request to", server_p, "server")) + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_torperf_failures <- function(start_p = NULL, end_p = NULL, + server_p = NULL, filesize_p = NULL) { + read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + filesize = col_double(), + source = col_character(), + server = col_character(), + q1 = col_skip(), + md = col_skip(), + q3 = col_skip(), + timeouts = col_double(), + failures = col_double(), + requests = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(filesize_p)) + filesize == ifelse(filesize_p == "50kb", 50 * 1024, + ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else + TRUE) %>% + filter(if (!is.null(server_p)) server == server_p else TRUE) %>% + filter(requests > 0) %>% + transmute(date, filesize, source, server, timeouts = timeouts / requests, + failures = failures / requests) +} + +plot_torperf_failures <- function(start_p, end_p, server_p, filesize_p, + path_p) { + prepare_torperf_failures(start_p, end_p, server_p, filesize_p) %>% + filter(source != "") %>% + gather(variable, value, -c(date, filesize, source, server)) %>% + mutate(variable = factor(variable, levels = c("timeouts", "failures"), + labels = c("Timeouts", "Failures"))) %>% + ggplot(aes(x = date, y = value, colour = source)) + + geom_point(size = 2, alpha = 0.5) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) + + scale_colour_hue(name = "Source") + + facet_grid(variable ~ .) + + ggtitle(paste("Timeouts and failures of", + ifelse(filesize_p == "50kb", "50 KiB", + ifelse(filesize_p == "1mb", "1 MiB", "5 MiB")), + "requests to", server_p, "server")) + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_onionperf_buildtimes <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "buildtimes.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + source = col_character(), + position = col_double(), + q1 = col_double(), + md = col_double(), + q3 = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) +} + +plot_onionperf_buildtimes <- function(start_p, end_p, path_p) { + prepare_onionperf_buildtimes(start_p, end_p) %>% + filter(source != "") %>% + mutate(date = as.Date(date), + position = factor(position, levels = seq(1, 3, 1), + labels = c("1st hop", "2nd hop", "3rd hop"))) %>% + complete(date = full_seq(date, period = 1), nesting(source, position)) %>% + ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + + geom_ribbon(alpha = 0.5) + + geom_line(aes(colour = source), size = 0.75) + + facet_grid(position ~ .) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "ms"), + limits = c(0, NA)) + + scale_fill_hue(name = "Source") + + scale_colour_hue(name = "Source") + + ggtitle("Circuit build times") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_onionperf_latencies <- function(start_p = NULL, end_p = NULL, + server_p = NULL) { + read_csv(file = paste(stats_dir, "latencies.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + source = col_character(), + server = col_character(), + q1 = col_double(), + md = col_double(), + q3 = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(server_p)) server == server_p else TRUE) +} + +plot_onionperf_latencies <- function(start_p, end_p, server_p, path_p) { + prepare_onionperf_latencies(start_p, end_p, server_p) %>% + filter(source != "") %>% + complete(date = full_seq(date, period = 1), nesting(source)) %>% + ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + + geom_ribbon(alpha = 0.5) + + geom_line(aes(colour = source), size = 0.75) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "ms"), + limits = c(0, NA)) + + scale_fill_hue(name = "Source") + + scale_colour_hue(name = "Source") + + ggtitle(paste("Circuit round-trip latencies to", server_p, "server")) + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_connbidirect <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "connbidirect2.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + direction = col_factor(levels = NULL), + quantile = col_double(), + fraction = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + mutate(quantile = paste("X", quantile, sep = ""), + fraction = fraction / 100) %>% + spread(quantile, fraction) %>% + rename(q1 = X0.25, md = X0.5, q3 = X0.75) +} + +plot_connbidirect <- function(start_p, end_p, path_p) { + prepare_connbidirect(start_p, end_p) %>% + complete(date = full_seq(date, period = 1), nesting(direction)) %>% + ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = direction)) + + geom_ribbon(alpha = 0.5) + + geom_line(aes(colour = direction), size = 0.75) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) + + scale_colour_hue(name = "Medians and interquartile ranges", + breaks = c("both", "write", "read"), + labels = c("Both reading and writing", "Mostly writing", + "Mostly reading")) + + scale_fill_hue(name = "Medians and interquartile ranges", + breaks = c("both", "write", "read"), + labels = c("Both reading and writing", "Mostly writing", + "Mostly reading")) + + ggtitle("Fraction of connections used uni-/bidirectionally") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_bandwidth_flags <- function(start_p = NULL, end_p = NULL) { + advbw <- read_csv(file = paste(stats_dir, "advbw.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + isexit = col_logical(), + isguard = col_logical(), + advbw = col_double())) %>% + transmute(date, have_guard_flag = isguard, have_exit_flag = isexit, + variable = "advbw", value = advbw * 8 / 1e9) + bwhist <- read_csv(file = paste(stats_dir, "bandwidth.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + isexit = col_logical(), + isguard = col_logical(), + bwread = col_double(), + bwwrite = col_double(), + dirread = col_double(), + dirwrite = col_double())) %>% + transmute(date, have_guard_flag = isguard, have_exit_flag = isexit, + variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9) + rbind(advbw, bwhist) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(!is.na(have_exit_flag)) %>% + filter(!is.na(have_guard_flag)) %>% + spread(variable, value) +} + +plot_bandwidth_flags <- function(start_p, end_p, path_p) { + prepare_bandwidth_flags(start_p, end_p) %>% + gather(variable, value, c(advbw, bwhist)) %>% + unite(flags, have_guard_flag, have_exit_flag) %>% + mutate(flags = factor(flags, + levels = c("FALSE_TRUE", "TRUE_TRUE", "TRUE_FALSE", "FALSE_FALSE"), + labels = c("Exit only", "Guard and Exit", "Guard only", + "Neither Guard nor Exit"))) %>% + mutate(variable = ifelse(variable == "advbw", + "Advertised bandwidth", "Consumed bandwidth")) %>% + ggplot(aes(x = date, y = value, fill = flags)) + + geom_area() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_fill_manual(name = "", + values = c("#03B3FF", "#39FF02", "#FFFF00", "#AAAA99")) + + facet_grid(variable ~ .) + + ggtitle("Advertised and consumed bandwidth by relay flags") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_userstats_relay_country <- function(start_p = NULL, end_p = NULL, + country_p = NULL, events_p = NULL) { + read_csv(file = paste(stats_dir, "clients.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + node = col_character(), + country = col_character(), + transport = col_character(), + version = col_character(), + lower = col_double(), + upper = col_double(), + clients = col_double(), + frac = col_double()), + na = character()) %>% + filter(node == "relay") %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(country_p)) + country == ifelse(country_p == "all", "", country_p) else TRUE) %>% + filter(transport == "") %>% + filter(version == "") %>% + select(date, country, clients, lower, upper, frac) %>% + rename(users = clients) +} + +plot_userstats_relay_country <- function(start_p, end_p, country_p, events_p, + path_p) { + u <- prepare_userstats_relay_country(start_p, end_p, country_p, events_p) %>% + complete(date = full_seq(date, period = 1)) + plot <- ggplot(u, aes(x = date, y = users)) + if (length(na.omit(u$users)) > 0 & events_p != "off" & + country_p != "all") { + upturns <- u[u$users > u$upper, c("date", "users")] + downturns <- u[u$users < u$lower, c("date", "users")] + if (events_p == "on") { + u[!is.na(u$lower) & u$lower < 0, "lower"] <- 0 + plot <- plot + + geom_ribbon(data = u, aes(ymin = lower, ymax = upper), fill = "gray") + } + if (length(upturns$date) > 0) + plot <- plot + + geom_point(data = upturns, aes(x = date, y = users), size = 5, + colour = "dodgerblue2") + if (length(downturns$date) > 0) + plot <- plot + + geom_point(data = downturns, aes(x = date, y = users), size = 5, + colour = "firebrick2") + } + plot <- plot + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + ggtitle(paste("Directly connecting users", + ifelse(country_p == "all", "", + paste(" from", countryname(country_p))), sep = "")) + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_userstats_bridge_country <- function(start_p = NULL, end_p = NULL, + country_p = NULL) { + read_csv(file = paste(stats_dir, "clients.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + node = col_character(), + country = col_character(), + transport = col_character(), + version = col_character(), + lower = col_double(), + upper = col_double(), + clients = col_double(), + frac = col_double()), + na = character()) %>% + filter(node == "bridge") %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(country_p)) + country == ifelse(country_p == "all", "", country_p) else TRUE) %>% + filter(transport == "") %>% + filter(version == "") %>% + select(date, country, clients, frac) %>% + rename(users = clients) +} + +plot_userstats_bridge_country <- function(start_p, end_p, country_p, path_p) { + prepare_userstats_bridge_country(start_p, end_p, country_p) %>% + complete(date = full_seq(date, period = 1)) %>% + ggplot(aes(x = date, y = users)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + ggtitle(paste("Bridge users", + ifelse(country_p == "all", "", + paste(" from", countryname(country_p))), sep = "")) + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_userstats_bridge_transport <- function(start_p = NULL, end_p = NULL, + transport_p = NULL) { + u <- read_csv(file = paste(stats_dir, "clients.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + node = col_character(), + country = col_character(), + transport = col_character(), + version = col_character(), + lower = col_double(), + upper = col_double(), + clients = col_double(), + frac = col_double())) %>% + filter(node == "bridge") %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(is.na(country)) %>% + filter(is.na(version)) %>% + filter(!is.na(transport)) %>% + select(date, transport, clients, frac) + if (is.null(transport_p) || "!<OR>" %in% transport_p) { + n <- u %>% + filter(transport != "<OR>") %>% + group_by(date, frac) %>% + summarize(clients = sum(clients)) + u <- rbind(u, data.frame(date = n$date, transport = "!<OR>", + clients = n$clients, frac = n$frac)) + } + u %>% + filter(if (!is.null(transport_p)) transport %in% transport_p else TRUE) %>% + select(date, transport, clients, frac) %>% + rename(users = clients) %>% + arrange(date, transport) +} + +plot_userstats_bridge_transport <- function(start_p, end_p, transport_p, + path_p) { + if (length(transport_p) > 1) { + title <- paste("Bridge users by transport") + } else { + title <- paste("Bridge users using", + ifelse(transport_p == "<??>", "unknown pluggable transport(s)", + ifelse(transport_p == "<OR>", "default OR protocol", + ifelse(transport_p == "!<OR>", "any pluggable transport", + ifelse(transport_p == "fte", "FTE", + ifelse(transport_p == "websocket", "Flash proxy/websocket", + paste("transport", transport_p))))))) + } + u <- prepare_userstats_bridge_transport(start_p, end_p, transport_p) %>% + complete(date = full_seq(date, period = 1), nesting(transport)) + if (length(transport_p) > 1) { + plot <- ggplot(u, aes(x = date, y = users, colour = transport)) + } else { + plot <- ggplot(u, aes(x = date, y = users)) + } + plot <- plot + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + ggtitle(title) + + labs(caption = copyright_notice) + if (length(transport_p) > 1) { + plot <- plot + + scale_colour_hue(name = "", breaks = transport_p, + labels = ifelse(transport_p == "<??>", "Unknown PT", + ifelse(transport_p == "<OR>", "Default OR protocol", + ifelse(transport_p == "!<OR>", "Any PT", + ifelse(transport_p == "fte", "FTE", + ifelse(transport_p == "websocket", "Flash proxy/websocket", + transport_p)))))) + } + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_userstats_bridge_version <- function(start_p = NULL, end_p = NULL, + version_p = NULL) { + read_csv(file = paste(stats_dir, "clients.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + node = col_character(), + country = col_character(), + transport = col_character(), + version = col_character(), + lower = col_double(), + upper = col_double(), + clients = col_double(), + frac = col_double())) %>% + filter(node == "bridge") %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(is.na(country)) %>% + filter(is.na(transport)) %>% + filter(if (!is.null(version_p)) version == version_p else TRUE) %>% + select(date, version, clients, frac) %>% + rename(users = clients) +} + +plot_userstats_bridge_version <- function(start_p, end_p, version_p, path_p) { + prepare_userstats_bridge_version(start_p, end_p, version_p) %>% + complete(date = full_seq(date, period = 1)) %>% + ggplot(aes(x = date, y = users)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + ggtitle(paste("Bridge users using IP", version_p, sep = "")) + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_userstats_bridge_combined <- function(start_p = NULL, end_p = NULL, + country_p = NULL) { + if (!is.null(country_p) && country_p == "all") { + prepare_userstats_bridge_country(start_p, end_p, country_p) + } else { + read_csv(file = paste(stats_dir, "userstats-combined.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + node = col_skip(), + country = col_character(), + transport = col_character(), + version = col_skip(), + frac = col_double(), + low = col_double(), + high = col_double()), + na = character()) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(country_p)) country == country_p else TRUE) %>% + select(date, country, transport, low, high, frac) %>% + arrange(date, country, transport) + } +} + +plot_userstats_bridge_combined <- function(start_p, end_p, country_p, path_p) { + if (country_p == "all") { + plot_userstats_bridge_country(start_p, end_p, country_p, path_p) + } else { + top <- 3 + u <- prepare_userstats_bridge_combined(start_p, end_p, country_p) + a <- aggregate(list(mid = (u$high + u$low) / 2), + by = list(transport = u$transport), FUN = sum) + a <- a[order(a$mid, decreasing = TRUE)[1:top], ] + u <- u[u$transport %in% a$transport, ] %>% + complete(date = full_seq(date, period = 1), nesting(country, transport)) + title <- paste("Bridge users by transport from ", + countryname(country_p), sep = "") + ggplot(u, aes(x = as.Date(date), ymin = low, ymax = high, + fill = transport)) + + geom_ribbon(alpha = 0.5, size = 0.5) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", limits = c(0, NA), labels = formatter) + + scale_colour_hue("Top-3 transports") + + scale_fill_hue("Top-3 transports") + + ggtitle(title) + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) + } +} + +prepare_advbwdist_perc <- function(start_p = NULL, end_p = NULL, p_p = NULL) { + read_csv(file = paste(stats_dir, "advbwdist.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + isexit = col_logical(), + relay = col_skip(), + percentile = col_integer(), + advbw = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(p_p)) percentile %in% as.numeric(p_p) else + percentile != "") %>% + transmute(date, percentile = as.factor(percentile), + variable = ifelse(is.na(isexit), "all", "exits"), + advbw = advbw * 8 / 1e9) %>% + spread(variable, advbw) %>% + rename(p = percentile) +} + +plot_advbwdist_perc <- function(start_p, end_p, p_p, path_p) { + prepare_advbwdist_perc(start_p, end_p, p_p) %>% + gather(variable, advbw, -c(date, p)) %>% + mutate(variable = ifelse(variable == "all", "All relays", + "Exits only")) %>% + complete(date = full_seq(date, period = 1), nesting(p, variable)) %>% + ggplot(aes(x = date, y = advbw, colour = p)) + + facet_grid(variable ~ .) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_colour_hue(name = "Percentile") + + ggtitle("Advertised bandwidth distribution") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_advbwdist_relay <- function(start_p = NULL, end_p = NULL, n_p = NULL) { + read_csv(file = paste(stats_dir, "advbwdist.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + isexit = col_logical(), + relay = col_integer(), + percentile = col_skip(), + advbw = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(n_p)) relay %in% as.numeric(n_p) else + relay != "") %>% + transmute(date, relay = as.factor(relay), + variable = ifelse(is.na(isexit), "all", "exits"), + advbw = advbw * 8 / 1e9) %>% + spread(variable, advbw) %>% + rename(n = relay) +} + +plot_advbwdist_relay <- function(start_p, end_p, n_p, path_p) { + prepare_advbwdist_relay(start_p, end_p, n_p) %>% + gather(variable, advbw, -c(date, n)) %>% + mutate(variable = ifelse(variable == "all", "All relays", + "Exits only")) %>% + complete(date = full_seq(date, period = 1), nesting(n, variable)) %>% + ggplot(aes(x = date, y = advbw, colour = n)) + + facet_grid(variable ~ .) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_colour_hue(name = "n") + + ggtitle("Advertised bandwidth of n-th fastest relays") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_hidserv_dir_onions_seen <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + type = col_factor(levels = NULL), + wmean = col_skip(), + wmedian = col_skip(), + wiqm = col_double(), + frac = col_double(), + stats = col_skip())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(type == "dir-onions-seen") %>% + transmute(date, onions = ifelse(frac >= 0.01, wiqm, NA), frac) +} + +plot_hidserv_dir_onions_seen <- function(start_p, end_p, path_p) { + prepare_hidserv_dir_onions_seen(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% + ggplot(aes(x = date, y = onions)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", limits = c(0, NA), labels = formatter) + + ggtitle("Unique .onion addresses") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_hidserv_rend_relayed_cells <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "hidserv.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + type = col_factor(levels = NULL), + wmean = col_skip(), + wmedian = col_skip(), + wiqm = col_double(), + frac = col_double(), + stats = col_skip())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(type == "rend-relayed-cells") %>% + transmute(date, + relayed = ifelse(frac >= 0.01, wiqm * 8 * 512 / (86400 * 1e9), NA), frac) +} + +plot_hidserv_rend_relayed_cells <- function(start_p, end_p, path_p) { + prepare_hidserv_rend_relayed_cells(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% + ggplot(aes(x = date, y = relayed)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + ggtitle("Onion-service traffic") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_webstats_tb <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), + col_types = cols( + log_date = col_date(format = ""), + request_type = col_factor(levels = NULL), + platform = col_skip(), + channel = col_skip(), + locale = col_skip(), + incremental = col_skip(), + count = col_double())) %>% + filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% + filter(request_type %in% c("tbid", "tbsd", "tbup", "tbur")) %>% + group_by(log_date, request_type) %>% + summarize(count = sum(count)) %>% + spread(request_type, count) %>% + rename(date = log_date, initial_downloads = tbid, + signature_downloads = tbsd, update_pings = tbup, + update_requests = tbur) +} + +plot_webstats_tb <- function(start_p, end_p, path_p) { + prepare_webstats_tb(start_p, end_p) %>% + gather(request_type, count, -date) %>% + mutate(request_type = factor(request_type, + levels = c("initial_downloads", "signature_downloads", "update_pings", + "update_requests"), + labels = c("Initial downloads", "Signature downloads", "Update pings", + "Update requests"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), nesting(request_type)) %>% + ggplot(aes(x = date, y = count)) + + geom_point() + + geom_line() + + facet_grid(request_type ~ ., scales = "free_y") + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), + strip.background = element_rect(fill = NA)) + + ggtitle("Tor Browser downloads and updates") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_webstats_tb_platform <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), + col_types = cols( + log_date = col_date(format = ""), + request_type = col_factor(levels = NULL), + platform = col_factor(levels = NULL), + channel = col_skip(), + locale = col_skip(), + incremental = col_skip(), + count = col_double())) %>% + filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% + filter(request_type %in% c("tbid", "tbup")) %>% + group_by(log_date, platform, request_type) %>% + summarize(count = sum(count)) %>% + spread(request_type, count, fill = 0) %>% + rename(date = log_date, initial_downloads = tbid, update_pings = tbup) +} + +plot_webstats_tb_platform <- function(start_p, end_p, path_p) { + prepare_webstats_tb_platform(start_p, end_p) %>% + gather(request_type, count, -c(date, platform)) %>% + mutate(request_type = factor(request_type, + levels = c("initial_downloads", "update_pings"), + labels = c("Initial downloads", "Update pings"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), + nesting(platform, request_type)) %>% + ggplot(aes(x = date, y = count, colour = platform)) + + geom_point() + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_hue(name = "Platform", + breaks = c("w", "m", "l", "o", ""), + labels = c("Windows", "macOS", "Linux", "Other", "Unknown")) + + facet_grid(request_type ~ ., scales = "free_y") + + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), + strip.background = element_rect(fill = NA), + legend.position = "top") + + ggtitle("Tor Browser downloads and updates by platform") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_webstats_tb_locale <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), + col_types = cols( + log_date = col_date(format = ""), + request_type = col_factor(levels = NULL), + platform = col_skip(), + channel = col_skip(), + locale = col_factor(levels = NULL), + incremental = col_skip(), + count = col_double())) %>% + filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% + filter(request_type %in% c("tbid", "tbup")) %>% + rename(date = log_date) %>% + group_by(date, locale, request_type) %>% + summarize(count = sum(count)) %>% + mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>% + spread(request_type, count, fill = 0) %>% + rename(initial_downloads = tbid, update_pings = tbup) +} + +plot_webstats_tb_locale <- function(start_p, end_p, path_p) { + d <- prepare_webstats_tb_locale(start_p, end_p) %>% + gather(request_type, count, -c(date, locale)) %>% + mutate(request_type = factor(request_type, + levels = c("initial_downloads", "update_pings"), + labels = c("Initial downloads", "Update pings"))) + e <- d + e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum) + e <- e[order(e$count, decreasing = TRUE), ] + e <- e[1:5, ] + d <- aggregate(list(count = d$count), by = list(date = d$date, + request_type = d$request_type, + locale = ifelse(d$locale %in% e$locale, d$locale, "(other)")), FUN = sum) + d %>% + complete(date = full_seq(date, period = 1), + nesting(locale, request_type)) %>% + ggplot(aes(x = date, y = count, colour = locale)) + + geom_point() + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_hue(name = "Locale", + breaks = c(e$locale, "(other)"), + labels = c(as.character(e$locale), "Other")) + + facet_grid(request_type ~ ., scales = "free_y") + + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), + strip.background = element_rect(fill = NA), + legend.position = "top") + + guides(col = guide_legend(nrow = 1)) + + ggtitle("Tor Browser downloads and updates by locale") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), + col_types = cols( + log_date = col_date(format = ""), + request_type = col_factor(levels = NULL), + platform = col_skip(), + channel = col_skip(), + locale = col_skip(), + incremental = col_skip(), + count = col_double())) %>% + filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% + filter(request_type %in% c("tmid", "tmup")) %>% + group_by(log_date, request_type) %>% + summarize(count = sum(count)) %>% + mutate(request_type = factor(request_type, levels = c("tmid", "tmup"))) %>% + spread(request_type, count, drop = FALSE, fill = 0) %>% + rename(date = log_date, initial_downloads = tmid, update_pings = tmup) +} + +plot_webstats_tm <- function(start_p, end_p, path_p) { + prepare_webstats_tm(start_p, end_p) %>% + gather(request_type, count, -date) %>% + mutate(request_type = factor(request_type, + levels = c("initial_downloads", "update_pings"), + labels = c("Initial downloads", "Update pings"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), nesting(request_type)) %>% + ggplot(aes(x = date, y = count)) + + geom_point() + + geom_line() + + facet_grid(request_type ~ ., scales = "free_y") + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), + strip.background = element_rect(fill = NA)) + + ggtitle("Tor Messenger downloads and updates") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_relays_ipv6 <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), + col_types = cols( + valid_after_date = col_date(format = ""), + server = col_factor(levels = NULL), + guard_relay = col_skip(), + exit_relay = col_skip(), + announced_ipv6 = col_logical(), + exiting_ipv6_relay = col_logical(), + reachable_ipv6_relay = col_logical(), + server_count_sum_avg = col_double(), + advertised_bandwidth_bytes_sum_avg = col_skip())) %>% + filter(if (!is.null(start_p)) + valid_after_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) + valid_after_date <= as.Date(end_p) else TRUE) %>% + filter(server == "relay") %>% + group_by(valid_after_date) %>% + summarize(total = sum(server_count_sum_avg), + announced = sum(server_count_sum_avg[announced_ipv6]), + reachable = sum(server_count_sum_avg[reachable_ipv6_relay]), + exiting = sum(server_count_sum_avg[exiting_ipv6_relay])) %>% + rename(date = valid_after_date) +} + +plot_relays_ipv6 <- function(start_p, end_p, path_p) { + prepare_relays_ipv6(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% + gather(category, count, -date) %>% + ggplot(aes(x = date, y = count, colour = category)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_hue(name = "", h.start = 90, + breaks = c("total", "announced", "reachable", "exiting"), + labels = c("Total (IPv4) OR", "IPv6 announced OR", "IPv6 reachable OR", + "IPv6 exiting")) + + ggtitle("Relays by IP version") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_bridges_ipv6 <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), + col_types = cols( + valid_after_date = col_date(format = ""), + server = col_factor(levels = NULL), + guard_relay = col_skip(), + exit_relay = col_skip(), + announced_ipv6 = col_logical(), + exiting_ipv6_relay = col_skip(), + reachable_ipv6_relay = col_skip(), + server_count_sum_avg = col_double(), + advertised_bandwidth_bytes_sum_avg = col_skip())) %>% + filter(if (!is.null(start_p)) + valid_after_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) + valid_after_date <= as.Date(end_p) else TRUE) %>% + filter(server == "bridge") %>% + group_by(valid_after_date) %>% + summarize(total = sum(server_count_sum_avg), + announced = sum(server_count_sum_avg[announced_ipv6])) %>% + rename(date = valid_after_date) +} + +plot_bridges_ipv6 <- function(start_p, end_p, path_p) { + prepare_bridges_ipv6(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% + gather(category, count, -date) %>% + ggplot(aes(x = date, y = count, colour = category)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_hue(name = "", h.start = 90, + breaks = c("total", "announced"), + labels = c("Total (IPv4) OR", "IPv6 announced OR")) + + ggtitle("Bridges by IP version") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_advbw_ipv6 <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "ipv6servers.csv", sep = ""), + col_types = cols( + valid_after_date = col_date(format = ""), + server = col_factor(levels = NULL), + guard_relay = col_logical(), + exit_relay = col_logical(), + announced_ipv6 = col_logical(), + exiting_ipv6_relay = col_logical(), + reachable_ipv6_relay = col_logical(), + server_count_sum_avg = col_skip(), + advertised_bandwidth_bytes_sum_avg = col_double())) %>% + filter(if (!is.null(start_p)) + valid_after_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) + valid_after_date <= as.Date(end_p) else TRUE) %>% + filter(server == "relay") %>% + mutate(advertised_bandwidth_bytes_sum_avg = + advertised_bandwidth_bytes_sum_avg * 8 / 1e9) %>% + group_by(valid_after_date) %>% + summarize(total = sum(advertised_bandwidth_bytes_sum_avg), + total_guard = sum(advertised_bandwidth_bytes_sum_avg[guard_relay]), + total_exit = sum(advertised_bandwidth_bytes_sum_avg[exit_relay]), + reachable_guard = sum(advertised_bandwidth_bytes_sum_avg[ + reachable_ipv6_relay & guard_relay]), + reachable_exit = sum(advertised_bandwidth_bytes_sum_avg[ + reachable_ipv6_relay & exit_relay]), + exiting = sum(advertised_bandwidth_bytes_sum_avg[ + exiting_ipv6_relay])) %>% + rename(date = valid_after_date) +} + +plot_advbw_ipv6 <- function(start_p, end_p, path_p) { + prepare_advbw_ipv6(start_p, end_p) %>% + complete(date = full_seq(date, period = 1)) %>% + gather(category, advbw, -date) %>% + ggplot(aes(x = date, y = advbw, colour = category)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_colour_hue(name = "", h.start = 90, + breaks = c("total", "total_guard", "total_exit", "reachable_guard", + "reachable_exit", "exiting"), + labels = c("Total (IPv4) OR", "Guard total (IPv4)", "Exit total (IPv4)", + "Reachable guard IPv6 OR", "Reachable exit IPv6 OR", "IPv6 exiting")) + + ggtitle("Advertised bandwidth by IP version") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + + guides(colour = guide_legend(nrow = 2, byrow = TRUE)) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +prepare_totalcw <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "totalcw.csv", sep = ""), + col_types = cols( + valid_after_date = col_date(format = ""), + nickname = col_character(), + have_guard_flag = col_logical(), + have_exit_flag = col_logical(), + measured_sum_avg = col_double())) %>% + filter(if (!is.null(start_p)) + valid_after_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) + valid_after_date <= as.Date(end_p) else TRUE) %>% + group_by(valid_after_date, nickname) %>% + summarize(measured_sum_avg = sum(measured_sum_avg)) %>% + rename(date = valid_after_date, totalcw = measured_sum_avg) %>% + arrange(date, nickname) +} + +plot_totalcw <- function(start_p, end_p, path_p) { + prepare_totalcw(start_p, end_p) %>% + mutate(nickname = ifelse(is.na(nickname), "consensus", nickname)) %>% + mutate(nickname = factor(nickname, + levels = c("consensus", unique(nickname[nickname != "consensus"])))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), nesting(nickname)) %>% + ggplot(aes(x = date, y = totalcw, colour = nickname)) + + geom_line(na.rm = TRUE) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + scale_colour_hue(name = "") + + ggtitle("Total consensus weights across bandwidth authorities") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + +countrynames <- function(countries) { + sapply(countries, countryname) +} + +write_userstats <- function(start, end, node, path) { + end <- min(end, as.character(Sys.Date())) + c <- read.csv(paste("/srv/metrics.torproject.org/metrics/shared/stats/", + "clients.csv", sep = ""), stringsAsFactors = FALSE) + c <- c[c$date >= start & c$date <= end & c$country != '' & + c$transport == '' & c$version == '' & c$node == node, ] + u <- data.frame(country = c$country, users = c$clients, + stringsAsFactors = FALSE) + u <- u[!is.na(u$users), ] + u <- aggregate(list(users = u$users), by = list(country = u$country), + mean) + total <- sum(u$users) + u <- u[!(u$country %in% c("zy", "??", "a1", "a2", "o1", "ap", "eu")), ] + u <- u[order(u$users, decreasing = TRUE), ] + u <- u[1:10, ] + u <- data.frame( + cc = as.character(u$country), + country = sub('the ', '', countrynames(as.character(u$country))), + abs = round(u$users), + rel = sprintf("%.2f", round(100 * u$users / total, 2))) + write.csv(u, path, quote = FALSE, row.names = FALSE) +} + +write_userstats_relay <- function(start, end, path) { + write_userstats(start, end, 'relay', path) +} + +write_userstats_bridge <- function(start, end, path) { + write_userstats(start, end, 'bridge', path) +} + +write_userstats_censorship_events <- function(start, end, path) { + end <- min(end, as.character(Sys.Date())) + c <- read.csv(paste("/srv/metrics.torproject.org/metrics/shared/stats/", + "clients.csv", sep = ""), stringsAsFactors = FALSE) + c <- c[c$date >= start & c$date <= end & c$country != '' & + c$transport == '' & c$version == '' & c$node == 'relay', ] + r <- data.frame(date = c$date, country = c$country, + upturn = ifelse(!is.na(c$upper) & + c$clients > c$upper, 1, 0), + downturn = ifelse(!is.na(c$lower) & + c$clients < c$lower, 1, 0)) + r <- aggregate(r[, c("upturn", "downturn")], + by = list(country = r$country), sum) + r <- r[(r$country %in% names(countrylist)), ] + r <- r[order(r$downturn, r$upturn, decreasing = TRUE), ] + r <- r[1:10, ] + r <- data.frame(cc = r$country, + country = sub('the ', '', countrynames(as.character(r$country))), + downturns = r$downturn, + upturns = r$upturn) + write.csv(r, path, quote = FALSE, row.names = FALSE) +} diff --git a/src/main/R/rserver/tables.R b/src/main/R/rserver/tables.R deleted file mode 100644 index 28bd3d5..0000000 --- a/src/main/R/rserver/tables.R +++ /dev/null @@ -1,58 +0,0 @@ -countrynames <- function(countries) { - sapply(countries, countryname) -} - -write_userstats <- function(start, end, node, path) { - end <- min(end, as.character(Sys.Date())) - c <- read.csv(paste("/srv/metrics.torproject.org/metrics/shared/stats/", - "clients.csv", sep = ""), stringsAsFactors = FALSE) - c <- c[c$date >= start & c$date <= end & c$country != '' & - c$transport == '' & c$version == '' & c$node == node, ] - u <- data.frame(country = c$country, users = c$clients, - stringsAsFactors = FALSE) - u <- u[!is.na(u$users), ] - u <- aggregate(list(users = u$users), by = list(country = u$country), - mean) - total <- sum(u$users) - u <- u[!(u$country %in% c("zy", "??", "a1", "a2", "o1", "ap", "eu")), ] - u <- u[order(u$users, decreasing = TRUE), ] - u <- u[1:10, ] - u <- data.frame( - cc = as.character(u$country), - country = sub('the ', '', countrynames(as.character(u$country))), - abs = round(u$users), - rel = sprintf("%.2f", round(100 * u$users / total, 2))) - write.csv(u, path, quote = FALSE, row.names = FALSE) -} - -write_userstats_relay <- function(start, end, path) { - write_userstats(start, end, 'relay', path) -} - -write_userstats_bridge <- function(start, end, path) { - write_userstats(start, end, 'bridge', path) -} - -write_userstats_censorship_events <- function(start, end, path) { - end <- min(end, as.character(Sys.Date())) - c <- read.csv(paste("/srv/metrics.torproject.org/metrics/shared/stats/", - "clients.csv", sep = ""), stringsAsFactors = FALSE) - c <- c[c$date >= start & c$date <= end & c$country != '' & - c$transport == '' & c$version == '' & c$node == 'relay', ] - r <- data.frame(date = c$date, country = c$country, - upturn = ifelse(!is.na(c$upper) & - c$clients > c$upper, 1, 0), - downturn = ifelse(!is.na(c$lower) & - c$clients < c$lower, 1, 0)) - r <- aggregate(r[, c("upturn", "downturn")], - by = list(country = r$country), sum) - r <- r[(r$country %in% names(countrylist)), ] - r <- r[order(r$downturn, r$upturn, decreasing = TRUE), ] - r <- r[1:10, ] - r <- data.frame(cc = r$country, - country = sub('the ', '', countrynames(as.character(r$country))), - downturns = r$downturn, - upturns = r$upturn) - write.csv(r, path, quote = FALSE, row.names = FALSE) -} -

1 0

[metrics-web/release] Update CollecTor page to include bandwidth files.
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit d31b049cb1ae146a5bb4e418dcda91aabf0275d4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 15 16:53:29 2019 +0200 Update CollecTor page to include bandwidth files. Fixes #30507. --- src/main/resources/web/jsps/collector.jsp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index 1adc9b3..a05b7e0 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -192,8 +192,9 @@ </tr> <tr> <td><a href="#type-bandwidth-file">Bandwidth Files</a></td> - <td><code>@type bandwidth-file 1.2</code></td> - <td></td> + <td><code>@type bandwidth-file 1.0</code></td> + <td><a href="/collector/recent/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs pull-left"> recent</a> + <a href="/collector/archive/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs pull-right"> archive</a></td> </tr> </tbody> </table> @@ -778,15 +779,16 @@ detail on a separate <a href="web-server-logs.html">page</a>. </h2> <h3 id="type-bandwidth-file" class="hover">Bandwidth Files -<code>@type bandwidth-file 1.2</code> +<code>@type bandwidth-file 1.0</code> +<a href="/collector/recent/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs"> recent</a> +<a href="/collector/archive/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs"> archive</a> <a href="#type-bandwidth-file" class="anchor">#</a> </h3> Bandwidth authority metrics as defined in the <a href="https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt">bandwidth-file-spec</a>. These are available from a DirPort's -<code>/tor/status-vote/next/bandwidth</code> url and CollecTor (both are -<a href="https://trac.torproject.org/projects/tor/ticket/21377">pending</a>). +<code>/tor/status-vote/next/bandwidth</code> url and CollecTor.

1 0

[metrics-web/release] Replace "brackets" with "parentheses"
by karsten＠torproject.org 10 Nov '19

10 Nov '19

commit 7cfb22744fc9a7801bf5df171ff6cbcbd80c0b97 Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Jun 5 13:30:07 2019 +0100 Replace "brackets" with "parentheses" Fixes: #29486 --- src/main/resources/web/templates/rs/search/do.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/templates/rs/search/do.html b/src/main/resources/web/templates/rs/search/do.html index 40745a6..1fe06b4 100644 --- a/src/main/resources/web/templates/rs/search/do.html +++ b/src/main/resources/web/templates/rs/search/do.html @@ -137,7 +137,7 @@ <% }); %> </tbody> </table> -&dagger;The number shown in brackets is the total effective +&dagger;The number shown in parentheses is the total effective family size for the relay including the relay itself. A size of 1 indicates that the relay does not have any other effective family members. Note that bridge relays do not advertise family members in their descriptors and so there

1 0