commit 8076ce4f8e7c2d987bfb44afbcd15d122dc44b83 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Dec 6 22:23:28 2017 +0100 Add graphs based on servers-ipv6.csv. Heavily based on input from teor. Implements #23761. --- src/main/R/rserver/graphs.R | 118 ++++++++++++++++++++++++++++ src/main/R/rserver/rserve-init.R | 2 + src/main/resources/web.xml | 9 +++ src/main/resources/web/json/categories.json | 3 + src/main/resources/web/json/metrics.json | 44 ++++++++++- 5 files changed, 175 insertions(+), 1 deletion(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index 00471cf..3805f63 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -1211,3 +1211,121 @@ plot_webstats_tm <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 150) } +plot_relays_ipv6 <- function(start, end, path) { + all_relay_data <- read.csv( + "/srv/metrics.torproject.org/metrics/shared/stats/ipv6servers.csv", + colClasses = c("valid_after_date" = "Date")) %>% + filter(server == "relay") + start_date <- max(as.Date(start), min(all_relay_data$valid_after_date)) + end_date <- min(as.Date(end), max(all_relay_data$valid_after_date), + Sys.Date() - 2) + date_breaks <- date_breaks(as.numeric(end_date - start_date)) + all_relay_data %>% + filter(valid_after_date >= start_date, valid_after_date <= end_date) %>% + group_by(valid_after_date) %>% + summarize(total = sum(server_count_sum_avg), + announced = sum(server_count_sum_avg[announced_ipv6 == 't']), + reachable = sum(server_count_sum_avg[reachable_ipv6_relay == 't']), + exiting = sum(server_count_sum_avg[exiting_ipv6_relay == 't'])) %>% + merge(data.frame(valid_after_date = seq(start_date, end_date, + by = "1 day")), all = TRUE) %>% + gather(total, announced, reachable, exiting, key = "category", + value = "count") %>% + ggplot(aes(x = valid_after_date, y = count, colour = category)) + + geom_line(size = 1) + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = ""), + labels = date_format(date_breaks$format), + date_breaks = date_breaks$major, + date_minor_breaks = date_breaks$minor) + + scale_y_continuous(name = "") + + scale_colour_hue(name = "", h.start = 90, + breaks = c("total", "announced", "reachable", "exiting"), + labels = c("Total (IPv4) OR", "IPv6 announced OR", "IPv6 reachable OR", + "IPv6 exititing")) + + expand_limits(y = 0) + + ggtitle("Relays by IP version") + + theme(legend.position = "top") + ggsave(filename = path, width = 8, height = 5, dpi = 150) +} + +plot_bridges_ipv6 <- function(start, end, path) { + all_bridge_data <- read.csv( + "/srv/metrics.torproject.org/metrics/shared/stats/ipv6servers.csv", + colClasses = c("valid_after_date" = "Date")) %>% + filter(server == "bridge") + start_date <- max(as.Date(start), min(all_bridge_data$valid_after_date)) + end_date <- min(as.Date(end), max(all_bridge_data$valid_after_date), + Sys.Date() - 2) + date_breaks <- date_breaks(as.numeric(end_date - start_date)) + all_bridge_data %>% + filter(valid_after_date >= start_date, valid_after_date <= end_date) %>% + group_by(valid_after_date) %>% + summarize(total = sum(server_count_sum_avg), + announced = sum(server_count_sum_avg[announced_ipv6 == 't'])) %>% + merge(data.frame(valid_after_date = seq(start_date, end_date, + by = "1 day")), all = TRUE) %>% + gather(total, announced, key = "category", value = "count") %>% + ggplot(aes(x = valid_after_date, y = count, colour = category)) + + geom_line(size = 1) + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = ""), + labels = date_format(date_breaks$format), + date_breaks = date_breaks$major, + date_minor_breaks = date_breaks$minor) + + scale_y_continuous(name = "") + + scale_colour_hue(name = "", h.start = 90, + breaks = c("total", "announced"), + labels = c("Total (IPv4) OR", "IPv6 announced OR")) + + expand_limits(y = 0) + + ggtitle("Bridges by IP version") + + theme(legend.position = "top") + ggsave(filename = path, width = 8, height = 5, dpi = 150) +} + +plot_advbw_ipv6 <- function(start, end, path) { + all_relay_data <- read.csv( + "/srv/metrics.torproject.org/metrics/shared/stats/ipv6servers.csv", + colClasses = c("valid_after_date" = "Date")) %>% + filter(server == "relay") + start_date <- max(as.Date(start), min(all_relay_data$valid_after_date)) + end_date <- min(as.Date(end), max(all_relay_data$valid_after_date), + Sys.Date() - 2) + date_breaks <- date_breaks(as.numeric(end_date - start_date)) + all_relay_data %>% + filter(valid_after_date >= start_date, valid_after_date <= end_date) %>% + group_by(valid_after_date) %>% + summarize(total = sum(advertised_bandwidth_bytes_sum_avg), + total_guard = sum(advertised_bandwidth_bytes_sum_avg[guard_relay != 'f']), + total_exit = sum(advertised_bandwidth_bytes_sum_avg[exit_relay != 'f']), + reachable_guard = sum(advertised_bandwidth_bytes_sum_avg[ + reachable_ipv6_relay != 'f' & guard_relay != 'f']), + reachable_exit = sum(advertised_bandwidth_bytes_sum_avg[ + reachable_ipv6_relay != 'f' & exit_relay != 'f']), + exiting = sum(advertised_bandwidth_bytes_sum_avg[ + exiting_ipv6_relay != 'f'])) %>% + merge(data.frame(valid_after_date = seq(start_date, end_date, + by = "1 day")), all = TRUE) %>% + gather(total, total_guard, total_exit, reachable_guard, reachable_exit, + exiting, key = "category", value = "count") %>% + ggplot(aes(x = valid_after_date, y = (count * 8) / 1e9, + colour = category)) + + geom_line(size = 1) + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = ""), + labels = date_format(date_breaks$format), + date_breaks = date_breaks$major, + date_minor_breaks = date_breaks$minor) + + scale_y_continuous(name = "Bandwidth (Gbit/s)") + + scale_colour_hue(name = "", h.start = 90, + breaks = c("total", "total_guard", "total_exit", "reachable_guard", + "reachable_exit", "exiting"), + labels = c("Total (IPv4) OR", "Guard total (IPv4)", "Exit total (IPv4)", + "Reachable guard IPv6 OR", "Reachable exit IPv6 OR", "IPv6 exiting")) + + expand_limits(y = 0) + + ggtitle("Advertised bandwidth by IP version") + + theme(legend.position = "top") + + guides(colour = guide_legend(nrow = 2, byrow = TRUE)) + ggsave(filename = path, width = 8, height = 5, dpi = 150) +} + diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 5cdf9c2..ede7c73 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -4,6 +4,8 @@ library("ggplot2") library("reshape") library("RColorBrewer") library("scales") +library(dplyr) +library(tidyr) source('graphs.R') source('tables.R') diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index 26acb8f..09ab129 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -51,6 +51,9 @@ <url-pattern>/webstats-tb-platform.html</url-pattern> <url-pattern>/webstats-tb-locale.html</url-pattern> <url-pattern>/webstats-tm.html</url-pattern> + <url-pattern>/relays-ipv6.html</url-pattern> + <url-pattern>/bridges-ipv6.html</url-pattern> + <url-pattern>/advbw-ipv6.html</url-pattern> </servlet-mapping> <servlet> @@ -168,6 +171,12 @@ <url-pattern>/webstats-tb-locale.pdf</url-pattern> <url-pattern>/webstats-tm.png</url-pattern> <url-pattern>/webstats-tm.pdf</url-pattern> + <url-pattern>/relays-ipv6.pdf</url-pattern> + <url-pattern>/relays-ipv6.png</url-pattern> + <url-pattern>/bridges-ipv6.pdf</url-pattern> + <url-pattern>/bridges-ipv6.png</url-pattern> + <url-pattern>/advbw-ipv6.pdf</url-pattern> + <url-pattern>/advbw-ipv6.png</url-pattern> </servlet-mapping> <servlet> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index e71c1d6..82cf1f4 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -28,6 +28,8 @@ "relayflags", "versions", "platforms", + "relays-ipv6", + "bridges-ipv6", "uptimes", "networkchurn", "bubbles" @@ -42,6 +44,7 @@ "metrics": [ "bandwidth", "bandwidth-flags", + "advbw-ipv6", "advbwdist-perc", "advbwdist-relay", "bwhist-flags", diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 1320f6c..ccd6d53 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -30,7 +30,7 @@ }, { "id": "versions", - "title": "Relays by version", + "title": "Relays by tor version", "type": "Graph", "description": "<p>This graph shows the number of running <a href=\"glossary.html#relay\">relays</a> by tor software version. Relays report their tor software version when they announce themselves in the network. More details on when these versions were declared stable or unstable can be found on the <a href=\"https://www.torproject.org/download/download.html\">download page</a> and in the <a href=\"https://gitweb.torproject.org/tor.git/tree/ChangeLog\">changes file</a>.</p>", "function": "plot_versions", @@ -57,6 +57,34 @@ ] }, { + "id": "relays-ipv6", + "title": "Relays by IP version", + "type": "Graph", + "description": "<p>This graph shows the number of <a href=\"glossary.html#relay\">relays</a> supporting IPv6 as compared to all relays. A relay can support IPv6 by announcing an IPv6 address and port for the OR protocol, which may then be confirmed as reachable by the <a href=\"glossary.html#directory-authority\">directory authorities</a>, and by permitting exiting to IPv6 targets. These sets are not distinct, because relays can have various combinations of announced/confirmed OR ports and exit policies.</p>", + "function": "plot_relays_ipv6", + "parameters": [ + "start", + "end" + ], + "data": [ + "ipv6servers" + ] + }, + { + "id": "bridges-ipv6", + "title": "Bridges by IP version", + "type": "Graph", + "description": "<p>This graph shows the number of <a href=\"glossary.html#bridge\">bridges</a> supporting IPv6 as compared to all bridges. A bridge can support IPv6 by announcing an IPv6 address and port for the OR protocol.</p>", + "function": "plot_bridges_ipv6", + "parameters": [ + "start", + "end" + ], + "data": [ + "ipv6servers" + ] + }, + { "id": "bandwidth", "title": "Total relay bandwidth", "type": "Graph", @@ -113,6 +141,20 @@ ] }, { + "id": "advbw-ipv6", + "title": "Advertised bandwidth by IP version", + "type": "Graph", + "description": "<p>This graph shows total <a href=\"glossary.html#advertised-bandwidth\">advertised bandwidth</a> by relays supporting IPv6 as compared to all relays. A relay can support IPv6 by announcing an IPv6 address and port for the OR protocol, which may then be confirmed as reachable by the <a href=\"glossary.html#directory-authority\">directory authorities</a>, and by permitting exiting to IPv6 targets. In some cases, relay sets are broken down by whether relays got the \"Guard\" and/or \"Exit\" <a href=\"glossary.html#relay-flag\">relay flags</a> indicating their special qualification for the first or last position in a <a href=\"glossary.html#circuit\">circuit</a>. These sets are not distinct, because relays can have various combinations of announced/confirmed OR ports, exit policies, and relay flags.</p>", + "function": "plot_advbw_ipv6", + "parameters": [ + "start", + "end" + ], + "data": [ + "ipv6servers" + ] + }, + { "id": "advbwdist-perc", "title": "Advertised bandwidth distribution", "type": "Graph",