August 2019 - tor-commits - lists.torproject.org

[metrics-web/master] Re-add the total relay bandwidth graph.
by karsten＠torproject.org 06 Aug '19

06 Aug '19

commit c71f0325e1f666ebf7a2fcc73724c2747b290690 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 5 10:48:41 2019 +0200 Re-add the total relay bandwidth graph. Implements 30883. --- src/main/R/rserver/rserve-init.R | 24 ++++++++++++++++++++++ src/main/resources/web.xml | 4 ++++ src/main/resources/web/json/categories.json | 1 + src/main/resources/web/json/metrics.json | 11 ++++++++++ .../resources/web/jsps/reproducible-metrics.jsp | 12 ++++++----- src/main/resources/web/jsps/stats.jsp | 21 +++++++++++++++++++ 6 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index f74fd03..af74ef7 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -829,6 +829,30 @@ plot_bandwidth_flags <- function(start_p, end_p, path_p) { ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } +prepare_bandwidth <- function(start_p = NULL, end_p = NULL) { + prepare_bandwidth_flags(start_p, end_p) %>% + group_by(date) %>% + summarize(advbw = sum(advbw), bwhist = sum(bwhist)) +} + +plot_bandwidth <- function(start_p, end_p, path_p) { + prepare_bandwidth(start_p, end_p) %>% + gather(variable, value, -date) %>% + ggplot(aes(x = date, y = value, colour = variable)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_colour_hue(name = "", h.start = 90, + breaks = c("advbw", "bwhist"), + labels = c("Advertised bandwidth", "Bandwidth history")) + + ggtitle("Total relay bandwidth") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + prepare_userstats_relay_country <- function(start_p = NULL, end_p = NULL, country_p = NULL, events_p = NULL) { read_csv(file = paste(stats_dir, "clients.csv", sep = ""), diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index f92813a..2c68fd7 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -30,6 +30,7 @@ <url-pattern>/relayflags.html</url-pattern> <url-pattern>/versions.html</url-pattern> <url-pattern>/platforms.html</url-pattern> + <url-pattern>/bandwidth.html</url-pattern> <url-pattern>/bandwidth-flags.html</url-pattern> <url-pattern>/dirbytes.html</url-pattern> <url-pattern>/advbwdist-perc.html</url-pattern> @@ -118,6 +119,9 @@ <url-pattern>/platforms.png</url-pattern> <url-pattern>/platforms.pdf</url-pattern> <url-pattern>/platforms.csv</url-pattern> + <url-pattern>/bandwidth.png</url-pattern> + <url-pattern>/bandwidth.pdf</url-pattern> + <url-pattern>/bandwidth.csv</url-pattern> <url-pattern>/bandwidth-flags.png</url-pattern> <url-pattern>/bandwidth-flags.pdf</url-pattern> <url-pattern>/bandwidth-flags.csv</url-pattern> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index ad0df11..89742bc 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -43,6 +43,7 @@ "summary": "How much traffic the Tor network can handle and how much traffic there is.", "description": "We measure total available bandwidth and current capacity by aggregating what relays and bridges report to directory authorities.", "metrics": [ + "bandwidth", "bandwidth-flags", "advbw-ipv6", "advbwdist-perc", diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 08e8c19..e6bab04 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -67,6 +67,17 @@ ] }, { + "id": "bandwidth", + "title": "Total relay bandwidth", + "type": "Graph", + "description": "<p>This graph shows the total <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of all <a href=\"glossary.html#relay\">relays</a> in the network.</p>", + "function": "bandwidth", + "parameters": [ + "start", + "end" + ] + }, + { "id": "bandwidth-flags", "title": "Advertised and consumed bandwidth by relay flags", "type": "Graph", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index aee4d5c..d6e7d0e 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -431,6 +431,7 @@ Relays self-report their advertised bandwidth in their server descriptors which <p>The following description applies to the following graphs:</p> <ul> +<li>Total relay bandwidth (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised and consumed bandwidth by relay flags (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth by IP version <a href="/advbw-ipv6.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth distribution <a href="/advbwdist-perc.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> @@ -468,7 +469,7 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with <h4>Step 3: Compute daily averages</h4> -<p>The first two graphs described here, namely <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p> +<p>The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p> <p>In order to compute these averages, first match consensus entries with server descriptors by SHA-1 digest. Every consensus entry references exactly one server descriptor, and a server descriptor may be referenced from an arbitrary number of consensus entries. @@ -513,6 +514,7 @@ Relays self-report bandwidth histories as part of their extra-info descriptors, <p>The following description applies to the following graphs:</p> <ul> +<li>Total relay bandwidth (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised and consumed bandwidth by relay flags (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> </ul> @@ -549,13 +551,13 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with <h4>Step 3: Compute daily totals</h4> -<p>The first graph described here, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a>, shows daily totals of all bytes written or read by relays. -For this graph, we sum up all read and written bytes on a given day and divide the result by 2. +<p>The first two graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a> and <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a> show daily totals of all bytes written or read by relays. +For both graphs we sum up all read and written bytes on a given day and divide the result by 2. However, we only include bandwidth histories for a given day if a relay was listed as running in a consensus at least once on that day. We attribute bandwidth to guards and/or exits if a relay was a guard and/or exit at least in one consensus on a day.</p> -<p>The second graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. -As opposed to the first graph, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. +<p>The third graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. +As opposed to the first two graphs, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. Also, we compute total read directory and total written directory bytes for this fourth graph, not an average of the two.</p> <h3 id="connbidirect" class="hover">Connection usage diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 5d653cb..eac75e3 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -52,6 +52,7 @@ https://metrics.torproject.org/identifier.csv <li><b>December 20, 2018:</b> Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph to cover all data previously contained in the first two graphs.</li> <li><b>May 29, 2019:</b> Extended <a href="#onionperf-latencies">Circuit round-trip latencies</a> graph to contain high/low values.</li> <li><b>June 2, 2019:</b> Added <a href="#onionperf-throughput">Throughput</a> graph.</li> +<li><b>August 5, 2019:</b> Re-added the <a href="#bandwidth">Total relay bandwidth</a> graph due to popular demand.</li> </ul> </div> @@ -334,6 +335,26 @@ Servers <a href="#servers" name="servers" class="anchor">#</a></h2> <h2><i class="fa fa-road fa-fw" aria-hidden="true"></i> Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> +<h3>Total relay bandwidth +<a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> +<a href="/bandwidth.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> +<a href="#bandwidth" name="bandwidth" class="anchor">#</a></h3> + +<h4>Parameters</h4> + +<ul> +<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> +<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> +</ul> + +<h4>Columns</h4> + +<ul> +<li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> +<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide.</li> +<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of all relays.</li> +</ul> + <h3>Advertised and consumed bandwidth by relay flags <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/bandwidth-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a>

1 0

[tor/release-0.4.1] make dist: only include files from practracker dir intentionally.
by nickm＠torproject.org 06 Aug '19

06 Aug '19

commit 394528241947800a0ac881b8890e5af387e0e9d7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 1 13:45:45 2019 -0400 make dist: only include files from practracker dir intentionally. Previously, we included temporary files and whatnot, which is not good. Fixes bug 31311; bugfix on 0.4.1.1-alpha. --- Makefile.am | 9 ++++++--- changes/ticket31311 | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3f3de34ce..10bd4b45c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -165,9 +165,12 @@ EXTRA_DIST+= \ README \ ReleaseNotes \ scripts/maint/checkIncludes.py \ - scripts/maint/checkSpace.pl \ - scripts/maint/practracker - + scripts/maint/checkSpace.pl \ + scripts/maint/practracker/exceptions.txt \ + scripts/maint/practracker/metrics.py \ + scripts/maint/practracker/practracker.py \ + scripts/maint/practracker/problem.py \ + scripts/maint/practracker/util.py ## This tells etags how to find mockable function definitions. AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s' diff --git a/changes/ticket31311 b/changes/ticket31311 new file mode 100644 index 000000000..88dfb8573 --- /dev/null +++ b/changes/ticket31311 @@ -0,0 +1,3 @@ + o Minor bugfixes (distribution): + - Do not ship any temporary files found in the scripts/maint/practracker + directory. Fixes bug 31311; bugfix on 0.4.1.1-alpha.

1 0

[tor/release-0.4.1] Merge branch 'ticket31311_041' into maint-0.4.1
by nickm＠torproject.org 06 Aug '19

06 Aug '19

commit 21531f35f3c6d7d7a86f474e5a2a4ff02388bea0 Merge: 74c059596 394528241 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 6 09:19:05 2019 -0400 Merge branch 'ticket31311_041' into maint-0.4.1 Makefile.am | 9 ++++++--- changes/ticket31311 | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-)

1 0

[tor/release-0.4.1] Merge branch 'maint-0.4.1' into release-0.4.1
by nickm＠torproject.org 06 Aug '19

06 Aug '19

commit 5a918ced20942e6d09f3cddc54bb7893f319b8ed Merge: 53abbc784 21531f35f Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 6 09:19:11 2019 -0400 Merge branch 'maint-0.4.1' into release-0.4.1 Makefile.am | 9 ++++++--- changes/ticket31311 | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-)

1 0

[tor/maint-0.4.1] make dist: only include files from practracker dir intentionally.
by nickm＠torproject.org 06 Aug '19

06 Aug '19

commit 394528241947800a0ac881b8890e5af387e0e9d7 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 1 13:45:45 2019 -0400 make dist: only include files from practracker dir intentionally. Previously, we included temporary files and whatnot, which is not good. Fixes bug 31311; bugfix on 0.4.1.1-alpha. --- Makefile.am | 9 ++++++--- changes/ticket31311 | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3f3de34ce..10bd4b45c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -165,9 +165,12 @@ EXTRA_DIST+= \ README \ ReleaseNotes \ scripts/maint/checkIncludes.py \ - scripts/maint/checkSpace.pl \ - scripts/maint/practracker - + scripts/maint/checkSpace.pl \ + scripts/maint/practracker/exceptions.txt \ + scripts/maint/practracker/metrics.py \ + scripts/maint/practracker/practracker.py \ + scripts/maint/practracker/problem.py \ + scripts/maint/practracker/util.py ## This tells etags how to find mockable function definitions. AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s' diff --git a/changes/ticket31311 b/changes/ticket31311 new file mode 100644 index 000000000..88dfb8573 --- /dev/null +++ b/changes/ticket31311 @@ -0,0 +1,3 @@ + o Minor bugfixes (distribution): + - Do not ship any temporary files found in the scripts/maint/practracker + directory. Fixes bug 31311; bugfix on 0.4.1.1-alpha.

1 0

[tor/maint-0.4.1] Merge branch 'ticket31311_041' into maint-0.4.1
by nickm＠torproject.org 06 Aug '19

06 Aug '19

commit 21531f35f3c6d7d7a86f474e5a2a4ff02388bea0 Merge: 74c059596 394528241 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Aug 6 09:19:05 2019 -0400 Merge branch 'ticket31311_041' into maint-0.4.1 Makefile.am | 9 ++++++--- changes/ticket31311 | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-)

1 0

[translation/torbutton-aboutdialogdtd] Update translations for torbutton-aboutdialogdtd
by translation＠torproject.org 06 Aug '19

06 Aug '19

commit f26fe7a78e89191946b1c0486f2564d04b6b2558 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Aug 6 13:19:29 2019 +0000 Update translations for torbutton-aboutdialogdtd --- en/aboutdialog.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/en/aboutdialog.dtd b/en/aboutdialog.dtd index 5099ad74b..844f4e070 100644 --- a/en/aboutdialog.dtd +++ b/en/aboutdialog.dtd @@ -1,6 +1,6 @@ -<!ENTITY project.start "&brandShortName; is developed by "> +<!ENTITY project.start "&brandShortName; is developed by ">  -<!ENTITY project.tpoLink "the &vendorShortName;"> +<!ENTITY project.tpoLink "the &vendorShortName;"> <!ENTITY project.end ", a nonprofit working to defend your privacy and freedom online."> <!ENTITY help.start "Want to help? ">

1 0

[translation/tba-torbrowserstringsdtd_completed] Update translations for tba-torbrowserstringsdtd_completed
by translation＠torproject.org 06 Aug '19

06 Aug '19

commit a4c0fc8f9f5b2169e56b633f291b937e4cfa9061 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Aug 6 13:18:02 2019 +0000 Update translations for tba-torbrowserstringsdtd_completed --- en/torbrowser_strings.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/en/torbrowser_strings.dtd b/en/torbrowser_strings.dtd index 1db742f53..dd3f0cc06 100644 --- a/en/torbrowser_strings.dtd +++ b/en/torbrowser_strings.dtd @@ -51,7 +51,7 @@ <!ENTITY pref_tor_network_bridges_enabled_change_builtin "You\'re using a built-in bridge to connect to Tor. Change"> <!ENTITY pref_tor_network_bridges_enabled_change_custom "You\'re using a custom bridge to connect to Tor. Change"> <!ENTITY pref_tor_network_using_multiple_provided_bridges "You\'re using multiple custom bridges."> -<!ENTITY pref_tor_network_using_a_provided_bridge "You\'re using &formatS; bridge."> +<!ENTITY pref_tor_network_using_a_provided_bridge "You\'re using &formatS; bridge."> <!ENTITY tor_notify_user_about_error "An error occurred, please swipe for more information.">

1 0

[translation/tba-torbrowserstringsdtd] Update translations for tba-torbrowserstringsdtd
by translation＠torproject.org 06 Aug '19

06 Aug '19

commit c032c5a4bba99b3b27693f4294ad482af2a90192 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Aug 6 13:17:54 2019 +0000 Update translations for tba-torbrowserstringsdtd --- en/torbrowser_strings.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/en/torbrowser_strings.dtd b/en/torbrowser_strings.dtd index 1db742f53..dd3f0cc06 100644 --- a/en/torbrowser_strings.dtd +++ b/en/torbrowser_strings.dtd @@ -51,7 +51,7 @@ <!ENTITY pref_tor_network_bridges_enabled_change_builtin "You\'re using a built-in bridge to connect to Tor. Change"> <!ENTITY pref_tor_network_bridges_enabled_change_custom "You\'re using a custom bridge to connect to Tor. Change"> <!ENTITY pref_tor_network_using_multiple_provided_bridges "You\'re using multiple custom bridges."> -<!ENTITY pref_tor_network_using_a_provided_bridge "You\'re using &formatS; bridge."> +<!ENTITY pref_tor_network_using_a_provided_bridge "You\'re using &formatS; bridge."> <!ENTITY tor_notify_user_about_error "An error occurred, please swipe for more information.">

1 0

[translation/torbutton-aboutdialogdtd_completed] Update translations for torbutton-aboutdialogdtd_completed
by translation＠torproject.org 06 Aug '19

06 Aug '19

commit 73740cbcae2bbce659d14cc74d73ffa273c9cfbc Author: Translation commit bot <translation(a)torproject.org> Date: Tue Aug 6 12:49:40 2019 +0000 Update translations for torbutton-aboutdialogdtd_completed --- en/aboutdialog.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/en/aboutdialog.dtd b/en/aboutdialog.dtd index 5099ad74b..844f4e070 100644 --- a/en/aboutdialog.dtd +++ b/en/aboutdialog.dtd @@ -1,6 +1,6 @@ -<!ENTITY project.start "&brandShortName; is developed by "> +<!ENTITY project.start "&brandShortName; is developed by ">  -<!ENTITY project.tpoLink "the &vendorShortName;"> +<!ENTITY project.tpoLink "the &vendorShortName;"> <!ENTITY project.end ", a nonprofit working to defend your privacy and freedom online."> <!ENTITY help.start "Want to help? ">

1 0