commit 59b800a491f93d08bcec6bda1b393a762156226d Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Dec 12 12:08:52 2018 +0100

Remove bandwidth and bwhist-flags graphs.

The "Advertised and consumed bandwidth by relay flags" graph now contains everything that's contained in the "Total relay bandwidth" and the "Consumed bandwidth by Exit/Guard flag combination" graphs. Removing these two graphs as obsolete.

Also update documentation for the newly deployed "Advertised and consumed bandwidth by relay flags" graph.

Part of #28353. --- src/main/R/rserver/graphs.R | 79 ---------------------- src/main/resources/web.xml | 8 --- src/main/resources/web/json/categories.json | 2 - src/main/resources/web/json/metrics.json | 22 ------ .../resources/web/jsps/reproducible-metrics.jsp | 17 ++--- src/main/resources/web/jsps/stats.jsp | 68 ++----------------- 6 files changed, 13 insertions(+), 183 deletions(-)

diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index cb40d52..1ca9357 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -448,85 +448,6 @@ write_platforms <- function(start_p = NULL, end_p = NULL, path_p) { write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") }

-prepare_bandwidth <- function(start_p, end_p) { - advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""), - colClasses = c("date" = "Date")) %>% - transmute(date, variable = "advbw", value = advbw * 8 / 1e9) - bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), - colClasses = c("date" = "Date")) %>% - transmute(date, variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9) - rbind(advbw, bwhist) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(!is.na(value)) %>% - group_by(date, variable) %>% - summarize(value = sum(value)) %>% - spread(variable, value) -} - -plot_bandwidth <- function(start_p, end_p, path_p) { - prepare_bandwidth(start_p, end_p) %>% - gather(variable, value, -date) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_hue(name = "", h.start = 90, - breaks = c("advbw", "bwhist"), - labels = c("Advertised bandwidth", "Bandwidth history")) + - ggtitle("Total relay bandwidth") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -write_bandwidth <- function(start_p = NULL, end_p = NULL, path_p) { - prepare_bandwidth(start_p, end_p) %>% - write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") -} - -prepare_bwhist_flags <- function(start_p, end_p) { - read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), - colClasses = c("date" = "Date")) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(isexit != "") %>% - filter(isguard != "") %>% - mutate(variable = ifelse(isexit == "t", - ifelse(isguard == "t", "guard_and_exit", "exit_only"), - ifelse(isguard == "t", "guard_only", "middle_only")), - value = (bwread + bwwrite) * 8 / 2e9) %>% - select(date, variable, value) -} - -plot_bwhist_flags <- function(start_p, end_p, path_p) { - prepare_bwhist_flags(start_p, end_p) %>% - complete(date = full_seq(date, period = 1), - variable = unique(variable)) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_manual(name = "", - breaks = c("exit_only", "guard_and_exit", "guard_only", "middle_only"), - labels = c("Exit only", "Guard & Exit", "Guard only", "Middle only"), - values = c("#E69F00", "#56B4E9", "#009E73", "#0072B2")) + - ggtitle("Bandwidth history by relay flags") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -write_bwhist_flags <- function(start_p = NULL, end_p = NULL, path_p) { - prepare_bwhist_flags(start_p, end_p) %>% - spread(variable, value) %>% - write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") -} - prepare_dirbytes <- function(start_p, end_p, path_p) { read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) %>% diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index 6ff82ad..b643b89 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -30,8 +30,6 @@ <url-pattern>/relayflags.html</url-pattern> <url-pattern>/versions.html</url-pattern> <url-pattern>/platforms.html</url-pattern> - <url-pattern>/bandwidth.html</url-pattern> - <url-pattern>/bwhist-flags.html</url-pattern> <url-pattern>/bandwidth-flags.html</url-pattern> <url-pattern>/dirbytes.html</url-pattern> <url-pattern>/advbwdist-perc.html</url-pattern> @@ -119,12 +117,6 @@ <url-pattern>/platforms.png</url-pattern> <url-pattern>/platforms.pdf</url-pattern> <url-pattern>/platforms.csv</url-pattern> - <url-pattern>/bandwidth.png</url-pattern> - <url-pattern>/bandwidth.pdf</url-pattern> - <url-pattern>/bandwidth.csv</url-pattern> - <url-pattern>/bwhist-flags.png</url-pattern> - <url-pattern>/bwhist-flags.pdf</url-pattern> - <url-pattern>/bwhist-flags.csv</url-pattern> <url-pattern>/bandwidth-flags.png</url-pattern> <url-pattern>/bandwidth-flags.pdf</url-pattern> <url-pattern>/bandwidth-flags.csv</url-pattern> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index 73d2f01..d19aeca 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -43,12 +43,10 @@ "summary": "How much traffic the Tor network can handle and how much traffic there is.", "description": "We measure total available bandwidth and current capacity by aggregating what relays and bridges report to directory authorities.", "metrics": [ - "bandwidth", "bandwidth-flags", "advbw-ipv6", "advbwdist-perc", "advbwdist-relay", - "bwhist-flags", "dirbytes", "connbidirect", "uncharted-data-flow" diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 5173ae4..5c99546 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -67,28 +67,6 @@ ] }, { - "id": "bandwidth", - "title": "Total relay bandwidth (deprecated)", - "type": "Graph", - "description": "<p>This graph shows the total <a href="glossary.html#advertised-bandwidth">advertised</a> and <a href="glossary.html#bandwidth-history">consumed bandwidth</a> of all <a href="glossary.html#relay">relays</a> in the network. <strong>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> graph.</strong></p>", - "function": "bandwidth", - "parameters": [ - "start", - "end" - ] - }, - { - "id": "bwhist-flags", - "title": "Consumed bandwidth by Exit/Guard flag combination (deprecated)", - "type": "Graph", - "description": "<p>This graph shows the <a href="glossary.html#bandwidth-history">consumed bandwidth</a> reported by relays, subdivided into four distinct subsets by assigned "Exit" and/or "Guard" <a href="glossary.html#relay-flag">flags</a>. <strong>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> graph.</strong></p>", - "function": "bwhist_flags", - "parameters": [ - "start", - "end" - ] - }, - { "id": "bandwidth-flags", "title": "Advertised and consumed bandwidth by relay flags", "type": "Graph", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 24bdba0..3838972 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -431,8 +431,7 @@ Relays self-report their advertised bandwidth in their server descriptors which <p>The following description applies to the following graphs:</p>

<ul> -<li>Total relay bandwidth (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Advertised and consumed bandwidth by relay flag (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Advertised and consumed bandwidth by relay flags (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth by IP version <a href="/advbw-ipv6.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth distribution <a href="/advbwdist-perc.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth of n-th fastest relays <a href="/advbwdist-relay.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> @@ -469,7 +468,7 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with

<h4>Step 3: Compute daily averages</h4>

-<p>The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a>, and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p> +<p>The first two graphs described here, namely <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p>

<p>In order to compute these averages, first match consensus entries with server descriptors by SHA-1 digest. Every consensus entry references exactly one server descriptor, and a server descriptor may be referenced from an arbitrary number of consensus entries. @@ -514,9 +513,7 @@ Relays self-report bandwidth histories as part of their extra-info descriptors, <p>The following description applies to the following graphs:</p>

<ul> -<li>Total relay bandwidth (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Advertised and consumed bandwidth by relay flag (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Consumed bandwidth by Exit/Guard flag combination <a href="/bwhist-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Advertised and consumed bandwidth by relay flags (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> </ul>

@@ -552,13 +549,13 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with

<h4>Step 3: Compute daily totals</h4>

-<p>The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a>, and <a href="/bwhist-flags.html">Consumed bandwidth by Exit/Guard flag combination</a>, show daily totals of all bytes written or read by relays. -For all three graphs, we sum up all read and written bytes on a given day and divide the result by 2. +<p>The first graph described here, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a>, shows daily totals of all bytes written or read by relays. +For this graph, we sum up all read and written bytes on a given day and divide the result by 2. However, we only include bandwidth histories for a given day if a relay was listed as running in a consensus at least once on that day. We attribute bandwidth to guards and/or exits if a relay was a guard and/or exit at least in one consensus on a day.</p>

-<p>The fourth graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. -As opposed to the first three graphs, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. +<p>The second graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. +As opposed to the first graph, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. Also, we compute total read directory and total written directory bytes for this fourth graph, not an average of the two.</p>

<h3 id="connbidirect" class="hover">Connection usage diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index a78da8a..6c1955a 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -49,7 +49,7 @@ https://metrics.torproject.org/identifier.csv <li><b>September 15, 2018:</b> Removed all pre-aggregated CSV files.</li> <li><b>October 28, 2018:</b> Added and/or removed columns to <a href="#webstats-tb-platform">Tor Browser downloads and updates by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads and updates by locale</a> graphs.</li> <li><b>December 20, 2018:</b> Removed source parameters and output rows with aggregates over all sources from <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> -<li><b>December 20, 2018 (scheduled):</b> Remove two graphs <a href="#bandwidth">Total relay bandwidth</a> and <a href="#bwhist-flags">Consumed bandwidth by Exit/Guard flag combination</a>, and update the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flag</a> graph to cover all data previously contained in the first two graphs.</li> +<li><b>December 20, 2018:</b> Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph to cover all data previously contained in the first two graphs.</li> </ul>

</div> @@ -332,32 +332,7 @@ Servers <a href="#servers" name="servers" class="anchor">#</a></h2> <h2><i class="fa fa-road fa-fw" aria-hidden="true"></i> Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2>

-<h3>Total relay bandwidth -<a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> -<a href="/bandwidth.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> -<a href="#bandwidth" name="bandwidth" class="anchor">#</a></h3> - -<div class="bs-callout bs-callout-warning"> -<h3>Deprecated</h3> -<p>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph.</p> -</div> - -<h4>Parameters</h4> - -<ul> -<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> -<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> -</ul> - -<h4>Columns</h4> - -<ul> -<li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide.</li> -<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of all relays.</li> -</ul> - -<h3>Advertised and consumed bandwidth by relay flag +<h3>Advertised and consumed bandwidth by relay flags <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/bandwidth-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> <a href="#bandwidth-flags" name="bandwidth-flags" class="anchor">#</a></h3> @@ -373,14 +348,10 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2>

<ul> <li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>guard_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Guard"</b> relay flag are capable to provide. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>guard_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Guard"</b> relay flag. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>exit_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Exit"</b> relay flag are capable to provide. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>exit_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Exit"</b> relay flag. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>have_guard_flag:</b> Whether relays included in this row had the <code>"Guard"</code> relay flag assigned (<code>"t"</code>) or not (<code>"f"</code>). <span class="blue">This column is going to be added after December 20, 2018.</span></li> -<li><b>have_exit_flag:</b> Whether relays included in this row had the <code>"Exit"</code> relay flag assigned and at the same time the <code>"BadExit"</code> not assigned (<code>"t"</code>) or not (<code>"f"</code>). <span class="blue">This column is going to be added after December 20, 2018.</span></li> -<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide. <span class="blue">This column is going to be added after December 20, 2018.</span></li> -<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic. <span class="blue">This column is going to be added after December 20, 2018.</span></li> +<li><b>have_guard_flag:</b> Whether relays included in this row had the <code>"Guard"</code> relay flag assigned (<code>"t"</code>) or not (<code>"f"</code>).</li> +<li><b>have_exit_flag:</b> Whether relays included in this row had the <code>"Exit"</code> relay flag assigned and at the same time the <code>"BadExit"</code> not assigned (<code>"t"</code>) or not (<code>"f"</code>).</li> +<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide.</li> +<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic.</li> </ul>

<h3>Advertised bandwidth by IP version @@ -451,33 +422,6 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> <li><b>exits:</b> Advertised bandwidth in Gbit/s of n-th fastest relay with the <b>"Exit"</b> relay flag.</li> </ul>

-<h3>Consumed bandwidth by Exit/Guard flag combination -<a href="/bwhist-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> -<a href="/bwhist-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> -<a href="#bwhist-flags" name="bwhist-flags" class="anchor">#</a></h3> - -<div class="bs-callout bs-callout-warning"> -<h3>Deprecated</h3> -<p>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph.</p> -</div> - -<h4>Parameters</h4> - -<ul> -<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> -<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> -</ul> - -<h4>Columns</h4> - -<ul> -<li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>exit_only:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays without <b>"Guard"</b> and with <b>"Exit"</b> relay flag.</li> -<li><b>guard_and_exit:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with both <b>"Guard"</b> and <b>"Exit"</b> relay flag.</li> -<li><b>guard_only:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with <b>"Guard"</b> and without <b>"Exit"</b> relay flag.</li> -<li><b>middle_only:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with neither <b>"Guard"</b> nor <b>"Exit"</b> relay flag.</li> -</ul> - <h3>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/dirbytes.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a>