[tor-commits] [metrics-web/release] Re-add the total relay bandwidth graph.

karsten at torproject.org karsten at torproject.org
Sat Nov 9 21:45:07 UTC 2019


commit c71f0325e1f666ebf7a2fcc73724c2747b290690
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon Aug 5 10:48:41 2019 +0200

    Re-add the total relay bandwidth graph.
    
    Implements 30883.
---
 src/main/R/rserver/rserve-init.R                   | 24 ++++++++++++++++++++++
 src/main/resources/web.xml                         |  4 ++++
 src/main/resources/web/json/categories.json        |  1 +
 src/main/resources/web/json/metrics.json           | 11 ++++++++++
 .../resources/web/jsps/reproducible-metrics.jsp    | 12 ++++++-----
 src/main/resources/web/jsps/stats.jsp              | 21 +++++++++++++++++++
 6 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R
index f74fd03..af74ef7 100644
--- a/src/main/R/rserver/rserve-init.R
+++ b/src/main/R/rserver/rserve-init.R
@@ -829,6 +829,30 @@ plot_bandwidth_flags <- function(start_p, end_p, path_p) {
   ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
 }
 
+prepare_bandwidth <- function(start_p = NULL, end_p = NULL) {
+  prepare_bandwidth_flags(start_p, end_p) %>%
+    group_by(date) %>%
+    summarize(advbw = sum(advbw), bwhist = sum(bwhist))
+}
+
+plot_bandwidth <- function(start_p, end_p, path_p) {
+  prepare_bandwidth(start_p, end_p) %>%
+    gather(variable, value, -date) %>%
+    ggplot(aes(x = date, y = value, colour = variable)) +
+    geom_line() +
+    scale_x_date(name = "", breaks = custom_breaks,
+      labels = custom_labels, minor_breaks = custom_minor_breaks) +
+    scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"),
+      limits = c(0, NA)) +
+    scale_colour_hue(name = "", h.start = 90,
+        breaks = c("advbw", "bwhist"),
+        labels = c("Advertised bandwidth", "Bandwidth history")) +
+    ggtitle("Total relay bandwidth") +
+    labs(caption = copyright_notice) +
+    theme(legend.position = "top")
+  ggsave(filename = path_p, width = 8, height = 5, dpi = 150)
+}
+
 prepare_userstats_relay_country <- function(start_p = NULL, end_p = NULL,
     country_p = NULL, events_p = NULL) {
   read_csv(file = paste(stats_dir, "clients.csv", sep = ""),
diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml
index f92813a..2c68fd7 100644
--- a/src/main/resources/web.xml
+++ b/src/main/resources/web.xml
@@ -30,6 +30,7 @@
     <url-pattern>/relayflags.html</url-pattern>
     <url-pattern>/versions.html</url-pattern>
     <url-pattern>/platforms.html</url-pattern>
+    <url-pattern>/bandwidth.html</url-pattern>
     <url-pattern>/bandwidth-flags.html</url-pattern>
     <url-pattern>/dirbytes.html</url-pattern>
     <url-pattern>/advbwdist-perc.html</url-pattern>
@@ -118,6 +119,9 @@
     <url-pattern>/platforms.png</url-pattern>
     <url-pattern>/platforms.pdf</url-pattern>
     <url-pattern>/platforms.csv</url-pattern>
+    <url-pattern>/bandwidth.png</url-pattern>
+    <url-pattern>/bandwidth.pdf</url-pattern>
+    <url-pattern>/bandwidth.csv</url-pattern>
     <url-pattern>/bandwidth-flags.png</url-pattern>
     <url-pattern>/bandwidth-flags.pdf</url-pattern>
     <url-pattern>/bandwidth-flags.csv</url-pattern>
diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json
index ad0df11..89742bc 100644
--- a/src/main/resources/web/json/categories.json
+++ b/src/main/resources/web/json/categories.json
@@ -43,6 +43,7 @@
     "summary": "How much traffic the Tor network can handle and how much traffic there is.",
     "description": "We measure total available bandwidth and current capacity by aggregating what relays and bridges report to directory authorities.",
     "metrics": [
+      "bandwidth",
       "bandwidth-flags",
       "advbw-ipv6",
       "advbwdist-perc",
diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json
index 08e8c19..e6bab04 100644
--- a/src/main/resources/web/json/metrics.json
+++ b/src/main/resources/web/json/metrics.json
@@ -67,6 +67,17 @@
     ]
   },
   {
+    "id": "bandwidth",
+    "title": "Total relay bandwidth",
+    "type": "Graph",
+    "description": "<p>This graph shows the total <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of all <a href=\"glossary.html#relay\">relays</a> in the network.</p>",
+    "function": "bandwidth",
+    "parameters": [
+      "start",
+      "end"
+    ]
+  },
+  {
     "id": "bandwidth-flags",
     "title": "Advertised and consumed bandwidth by relay flags",
     "type": "Graph",
diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp
index aee4d5c..d6e7d0e 100644
--- a/src/main/resources/web/jsps/reproducible-metrics.jsp
+++ b/src/main/resources/web/jsps/reproducible-metrics.jsp
@@ -431,6 +431,7 @@ Relays self-report their advertised bandwidth in their server descriptors which
 <p>The following description applies to the following graphs:</p>
 
 <ul>
+<li>Total relay bandwidth (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
 <li>Advertised and consumed bandwidth by relay flags (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
 <li>Advertised bandwidth by IP version <a href="/advbw-ipv6.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
 <li>Advertised bandwidth distribution <a href="/advbwdist-perc.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
@@ -468,7 +469,7 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with
 
 <h4>Step 3: Compute daily averages</h4>
 
-<p>The first two graphs described here, namely <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p>
+<p>The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p>
 
 <p>In order to compute these averages, first match consensus entries with server descriptors by SHA-1 digest.
 Every consensus entry references exactly one server descriptor, and a server descriptor may be referenced from an arbitrary number of consensus entries.
@@ -513,6 +514,7 @@ Relays self-report bandwidth histories as part of their extra-info descriptors,
 <p>The following description applies to the following graphs:</p>
 
 <ul>
+<li>Total relay bandwidth (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
 <li>Advertised and consumed bandwidth by relay flags (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
 <li>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li>
 </ul>
@@ -549,13 +551,13 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with
 
 <h4>Step 3: Compute daily totals</h4>
 
-<p>The first graph described here, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a>, shows daily totals of all bytes written or read by relays.
-For this graph, we sum up all read and written bytes on a given day and divide the result by 2.
+<p>The first two graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a> and <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a> show daily totals of all bytes written or read by relays.
+For both graphs we sum up all read and written bytes on a given day and divide the result by 2.
 However, we only include bandwidth histories for a given day if a relay was listed as running in a consensus at least once on that day.
 We attribute bandwidth to guards and/or exits if a relay was a guard and/or exit at least in one consensus on a day.</p>
 
-<p>The second graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests.
-As opposed to the first graph, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus.
+<p>The third graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests.
+As opposed to the first two graphs, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus.
 Also, we compute total read directory and total written directory bytes for this fourth graph, not an average of the two.</p>
 
 <h3 id="connbidirect" class="hover">Connection usage
diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp
index 5d653cb..eac75e3 100644
--- a/src/main/resources/web/jsps/stats.jsp
+++ b/src/main/resources/web/jsps/stats.jsp
@@ -52,6 +52,7 @@ https://metrics.torproject.org/identifier.csv
 <li><b>December 20, 2018:</b> Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph to cover all data previously contained in the first two graphs.</li>
 <li><b>May 29, 2019:</b> Extended <a href="#onionperf-latencies">Circuit round-trip latencies</a> graph to contain high/low values.</li>
 <li><b>June 2, 2019:</b> Added <a href="#onionperf-throughput">Throughput</a> graph.</li>
+<li><b>August 5, 2019:</b> Re-added the <a href="#bandwidth">Total relay bandwidth</a> graph due to popular demand.</li>
 </ul>
 
 </div>
@@ -334,6 +335,26 @@ Servers <a href="#servers" name="servers" class="anchor">#</a></h2>
 <h2><i class="fa fa-road fa-fw" aria-hidden="true"></i>
 Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2>
 
+<h3>Total relay bandwidth
+<a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a>
+<a href="/bandwidth.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a>
+<a href="#bandwidth" name="bandwidth" class="anchor">#</a></h3>
+
+<h4>Parameters</h4>
+
+<ul>
+<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li>
+<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li>
+</ul>
+
+<h4>Columns</h4>
+
+<ul>
+<li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li>
+<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide.</li>
+<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of all relays.</li>
+</ul>
+
 <h3>Advertised and consumed bandwidth by relay flags
 <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a>
 <a href="/bandwidth-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a>





More information about the tor-commits mailing list