November 2019 - tor-commits - lists.torproject.org

[metrics-web/release] Add OnionPerf throughput graph.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit d4452b38183e50d20c29bad6fff51bae58ebde0c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jun 2 15:56:50 2019 +0200 Add OnionPerf throughput graph. We calculate throughput from the time between receiving 0.5 and 1 MiB of a response, which obviously excludes any measurements with responses smaller than 1 MiB. From the FILESIZE and DATAPERC* fields we can compute the number of milliseconds that have elapsed between receiving bytes 524,288 and 1,048,576, which is a total of 524,288 bytes or 4,194,304 bits. We divide the value 4,194,304 by this time difference to obtain throughput in bits per millisecond which happens to be the same value as the number of kilobits per second. Implements #29772. --- src/main/R/rserver/rserve-init.R | 40 +++++++++++++++++++ .../torproject/metrics/stats/onionperf/Main.java | 30 +++++++++++++++ src/main/resources/web.xml | 4 ++ src/main/resources/web/json/categories.json | 3 +- src/main/resources/web/json/metrics.json | 12 ++++++ .../resources/web/jsps/reproducible-metrics.jsp | 9 ++++- src/main/resources/web/jsps/stats.jsp | 26 +++++++++++++ src/main/sql/onionperf/init-onionperf.sql | 45 ++++++++++++++++++++++ src/submods/metrics-lib | 2 +- 9 files changed, 168 insertions(+), 3 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 88aa5b9..f74fd03 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -697,6 +697,46 @@ plot_onionperf_latencies <- function(start_p, end_p, server_p, path_p) { ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } +prepare_onionperf_throughput <- function(start_p = NULL, end_p = NULL, + server_p = NULL) { + read_csv(file = paste(stats_dir, "onionperf-throughput.csv", sep = ""), + col_types = cols( + date = col_date(format = ""), + source = col_character(), + server = col_character(), + low = col_double(), + q1 = col_double(), + md = col_double(), + q3 = col_double(), + high = col_double())) %>% + filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% + filter(if (!is.null(server_p)) server == server_p else TRUE) +} + +plot_onionperf_throughput <- function(start_p, end_p, server_p, path_p) { + prepare_onionperf_throughput(start_p, end_p, server_p) %>% + complete(date = full_seq(date, period = 1), nesting(source)) %>% + ggplot(aes(x = date, ymin = q1 / 1000, ymax = q3 / 1000, fill = source)) + + geom_ribbon(alpha = 0.5) + + geom_line(aes(y = md / 1000, colour = source), size = 0.75) + + geom_line(aes(y = high / 1000, colour = source), size = 0.375) + + geom_line(aes(y = low / 1000, colour = source), size = 0.375) + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Mbps"), + limits = c(0, NA)) + + scale_fill_hue(name = "Source") + + scale_colour_hue(name = "Source") + + facet_grid(source ~ ., scales = "free", space = "free") + + ggtitle(paste("Throughput when downloading from", server_p, "server")) + + labs(caption = copyright_notice) + + theme(legend.position = "none", + strip.text.y = element_text(angle = 0, hjust = 0), + strip.background = element_rect(fill = NA)) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + prepare_connbidirect <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "connbidirect2.csv", sep = ""), col_types = cols( diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index a75cd1b..8fb762d 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -55,6 +55,9 @@ public class Main { queryBuildTimes(connection)); writeStatistics(new File(baseDir, "stats/latencies.csv").toPath(), queryLatencies(connection)); + writeStatistics( + new File(baseDir, "stats/onionperf-throughput.csv").toPath(), + queryThroughput(connection)); disconnectFromDatabase(connection); log.info("Terminated onionperf module."); } @@ -321,6 +324,33 @@ public class Main { return statistics; } + static List<String> queryThroughput(Connection connection) + throws SQLException { + log.info("Querying throughput statistics from database."); + List<String> statistics = new ArrayList<>(); + statistics.add("date,source,server,low,q1,md,q3,high"); + Statement st = connection.createStatement(); + String queryString = "SELECT date, source, server, low, q1, md, q3, high " + + "FROM throughput_stats"; + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + statistics.add(String.format("%s,%s,%s,%d,%d,%d,%d,%d", + dateFormat.format(rs.getDate("date", calendar)), + getStringFromResultSet(rs, "source"), + rs.getString("server"), + rs.getInt("low"), + rs.getInt("q1"), + rs.getInt("md"), + rs.getInt("q3"), + rs.getInt("high"))); + } + } + return statistics; + } + /** Retrieves the <code>String</code> value of the designated column in the * current row of the given <code>ResultSet</code> object, or returns the * empty string if the retrieved value was <code>NULL</code>. */ diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index b643b89..045dd2e 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -43,6 +43,7 @@ <url-pattern>/torperf-failures.html</url-pattern> <url-pattern>/onionperf-buildtimes.html</url-pattern> <url-pattern>/onionperf-latencies.html</url-pattern> + <url-pattern>/onionperf-throughput.html</url-pattern> <url-pattern>/connbidirect.html</url-pattern> <url-pattern>/hidserv-dir-onions-seen.html</url-pattern> <url-pattern>/hidserv-rend-relayed-cells.html</url-pattern> @@ -135,6 +136,9 @@ <url-pattern>/onionperf-latencies.png</url-pattern> <url-pattern>/onionperf-latencies.pdf</url-pattern> <url-pattern>/onionperf-latencies.csv</url-pattern> + <url-pattern>/onionperf-throughput.png</url-pattern> + <url-pattern>/onionperf-throughput.pdf</url-pattern> + <url-pattern>/onionperf-throughput.csv</url-pattern> <url-pattern>/connbidirect.png</url-pattern> <url-pattern>/connbidirect.pdf</url-pattern> <url-pattern>/connbidirect.csv</url-pattern> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index d19aeca..ad0df11 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -62,7 +62,8 @@ "torperf", "torperf-failures", "onionperf-buildtimes", - "onionperf-latencies" + "onionperf-latencies", + "onionperf-throughput" ] }, { diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 006de8b..bfcde22 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -309,6 +309,18 @@ ] }, { + "id": "onionperf-throughput", + "title": "Throughput", + "type": "Graph", + "description": "This graph shows throughput when downloading static files of different sizes over Tor, either from a server on the public internet or from a version 2 onion server. Throughput is calculated from the time between receiving 0.5 and 1 MiB of the response. The graph shows the median of measurements as thick line, the range of measurements from first to third quartile as ribbon, and the highest and lowest non-outlier measurements as thin lines.", + "function": "onionperf_throughput", + "parameters": [ + "start", + "end", + "server" + ] + }, + { "id": "connbidirect", "title": "Fraction of connections used uni-/bidirectionally", "type": "Graph", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index a833b31..aee4d5c 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -619,6 +619,7 @@ Here we explain how we evaluate Torperf/OnionPerf measurement to obtain the same <li>Timeouts and failures of downloading files over Tor <a href="/torperf-failures.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Circuit build times <a href="/onionperf-buildtimes.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Circuit round-trip latencies <a href="/onionperf-latencies.html" class="btn btn-primary btn-xs"> graph</a></li> +<li>Throughput <a href="/onionperf-throughput.html" class="btn btn-primary btn-xs"> graph</a></li> </ul> <h4>Step 1: Parse OnionPerf and/or Torperf measurement results</h4> @@ -636,17 +637,23 @@ Here we explain how we evaluate Torperf/OnionPerf measurement to obtain the same <li><code>DATACOMPLETE</code>: Download end time that is only set if the request succeeded.</li> <li><code>READBYTES</code>: Total number of bytes read, which indicates whether this request succeeded (if ≥ <code>FILESIZE</code>) or failed.</li> <li><code>DIDTIMEOUT</code>: 1 if the request timed out, 0 otherwise.</li> +<li><code>DATAPERCx</code>: Time when x% of expected bytes were read for x = { 10, 20, 50, 100 }.</li> <li><code>BUILDTIMES</code>: Comma-separated list of times when circuit hops were built, which includes all circuits used for making measurement requests, successful or not.</li> <li><code>ENDPOINTREMOTE</code>: Hostname, IP address, and port that was used to connect to the remote server; we use this to distinguish a request to a public server (if <code>ENDPOINTREMOTE</code> is not present or does not contain <code>".onion"</code> as substring) or to an onion server.</li> </ul> <h4>Step 2: Aggregate measurement results</h4> -Each of the measurement results parsed in the previous steps constitutes a single measurement. +Each of the measurement results parsed in the previous step constitutes a single measurement. We're first interested in statistics on download times for the <a href="/torperf.html">Time to download files over Tor</a> graph. Therefore we consider only measurements with <code>DATACOMPLETE > START</code>, for which we calculate the download time as: <code>DATACOMPLETE - START</code>. We then compute the 25th, 50th, and 75th percentile of download times by sorting download times, determining the percentile rank, and using linear interpolation between adjacent ranks. +Next we're interested in the average throughput of measurements for the <a href="/onionperf-throughput.html">Throughput</a> graph. +We calculate throughput from the time between receiving 0.5 and 1 MiB of a response, which obviously excludes any measurements with responses smaller than 1 MiB. +From <code>DATAPERC50</code> and <code>DATAPERC100</code> (if <code>FILESIZE = 1048576</code>) or <code>DATAPERC10</code> and <code>DATAPERC20</code> (if <code>FILESIZE = 5242880</code>) we can compute the number of milliseconds that have elapsed between receiving bytes 524,288 and 1,048,576, which is a total of 524,288 bytes or 4,194,304 bits. +We divide the value 4,194,304 by this time difference to obtain throughput in bits per millisecond which happens to be the same value as the number of kilobits per second. + We're also interested in circuit round-trip latencies for the <a href="/onionperf-latencies.html">Circuit round-trip latencies</a> graph. We measure circuit latency as the time between sending the HTTP request and receiving the HTTP response header. We calculate latencies as <code>DATARESPONSE - DATAREQUEST</code> for measurements with non-zero values for both timestamps. diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 443c292..68c4114 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -570,6 +570,32 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>high: Highest latency within 1.5 IQR of upper quartile (upper whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> </ul> +<h3>Throughput +<a href="/onionperf-throughput.html" class="btn btn-primary btn-xs"> graph</a> +<a href="/onionperf-throughput.csv" class="btn btn-primary btn-xs"> data</a> +<a href="#onionperf-throughput" name="onionperf-throughput" class="anchor">#</a></h3> + +<h4>Parameters</h4> + +<ul> +<li>start: First UTC date (YYYY-MM-DD) to include in the file.</li> +<li>end: Last UTC date (YYYY-MM-DD) to include in the file.</li> +<li>server: Either "public" for requests to a server on the public internet, or "onion" for requests to a version 2 onion server.</li> +</ul> + +<h4>Columns</h4> + +<ul> +<li>date: UTC date (YYYY-MM-DD) when download performance was measured.</li> +<li>source: Name of the OnionPerf or Torperf service performing measurements.</li> +<li>server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server.</li> +<li>low: Lowest measured throughput within 1.5 IQR of lower quartile (lower whisker in a boxplot) in kilobits per second.</li> +<li>q1: First quartile of measured throughput in kilobits per second.</li> +<li>md: Median of measured throughput in kilobits per second.</li> +<li>q3: Third quartile of measured throughput in kilobits per second.</li> +<li>high: Highest measured throughput within 1.5 IQR of upper quartile (upper whisker in a boxplot) in kilobits per second.</li> +</ul> + </div> <div class="container"> diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index 536cd45..b7c41d6 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -164,3 +164,48 @@ FROM filtered_measurements NATURAL JOIN quartiles GROUP BY 1, 2, 3 ORDER BY date, source, server; +-- Explanation of the number 4194304 below for computing kbps: From the FILESIZE +-- and DATAPERC* fields we can compute the number of milliseconds that have +-- elapsed between receiving bytes 524,288 and 1,048,576, which is a total +-- amount of 524,288 bytes or 4,194,304 bits. If we divide that value by +-- 4,194,304 we obtain the number of milliseconds that have elapsed for +-- downloading 1 bit, which happens to be the same value as the number of +-- seconds for downloading 1 kilobit. We want the reciprocal of that value which +-- has the unit kilobits per second. +CREATE OR REPLACE VIEW throughput_stats AS +WITH filtered_measurements AS ( + SELECT DATE(start) AS date, + source, + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' + ELSE 'public' END AS server, + CASE WHEN filesize = 1048576 AND dataperc100 > dataperc50 + THEN 4194304 / (dataperc100 - dataperc50) + WHEN filesize = 5242880 AND dataperc20 > dataperc10 + THEN 4194304 / (dataperc20 - dataperc10) + ELSE NULL END AS kbps + FROM measurements + WHERE DATE(start) < current_date - 1 + AND endpointremote NOT SIMILAR TO '_{56}.onion%' +), quartiles AS ( + SELECT date, + source, + server, + PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) + WITHIN GROUP(ORDER BY kbps) AS q + FROM filtered_measurements + GROUP BY date, source, server +) +SELECT date, + source, + server, + MIN(CASE WHEN kbps >= q[1] - ((q[3] - q[1]) * 1.5) + THEN kbps ELSE NULL END) AS low, + TRUNC(AVG(q[1])) AS q1, + TRUNC(AVG(q[2])) AS md, + TRUNC(AVG(q[3])) AS q3, + MAX(CASE WHEN kbps <= q[3] + ((q[3] - q[1]) * 1.5) + THEN kbps ELSE NULL END) AS high +FROM filtered_measurements NATURAL JOIN quartiles +GROUP BY date, source, server +ORDER BY date, source, server; + diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 3693e10..e723c06 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 3693e107a3aff7473200ece3ba3889dc9462c7b3 +Subproject commit e723c065b764ecfbb3bb96d4c491e67398b7f21b

1 0

[metrics-web/release] Update to latest metrics-lib.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 37190035e688cc0a313bb06e10324c8fc609a0b6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 16 16:39:37 2019 +0200 Update to latest metrics-lib. --- src/submods/metrics-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 53b16d1..7e396f0 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 53b16d192da4a7fa5c1987a89a60257061602882 +Subproject commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f

1 0

[metrics-web/release] Fix links on CollecTor page, again.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit c3d9c42d0743e9a15c117ab778802fbf7afc3ff1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 21 08:33:05 2019 +0200 Fix links on CollecTor page, again. Include metrics-lib-*-thin.jar rather than metrics-lib-*.jar. Re-closes #24792. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index d8c0c07..038a82f 100644 --- a/build.xml +++ b/build.xml @@ -83,7 +83,7 @@ <include name="jetty-plus${jetty.version}.jar"/> <include name="jetty-jndi${jetty.version}.jar"/> <include name="apache-jsp${jetty.version}.jar" /> - <include name="metrics-lib-${metricslibversion}.jar"/> + <include name="metrics-lib-${metricslibversion}-thin.jar"/> <include name="taglibs-standard-spec-1.2.5.jar"/> <include name="tomcat-embed-jasper-8.5.14.jar" /> <include name="tomcat-embed-el-8.5.14.jar" />

1 0

[metrics-web/release] Tweak documentation of new(ly updated) graphs.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 3f252fce7f4c3bcfc1a1f53ca0c8f2ff9df31659 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jun 11 16:36:06 2019 +0200 Tweak documentation of new(ly updated) graphs. --- src/main/resources/web/jsps/stats.jsp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 68c4114..5d653cb 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -50,6 +50,8 @@ https://metrics.torproject.org/identifier.csv <li>October 28, 2018: Added and/or removed columns to <a href="#webstats-tb-platform">Tor Browser downloads and updates by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads and updates by locale</a> graphs.</li> <li>December 20, 2018: Removed source parameters and output rows with aggregates over all sources from <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> <li>December 20, 2018: Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph to cover all data previously contained in the first two graphs.</li> +<li>May 29, 2019: Extended <a href="#onionperf-latencies">Circuit round-trip latencies</a> graph to contain high/low values.</li> +<li>June 2, 2019: Added <a href="#onionperf-throughput">Throughput</a> graph.</li> </ul> </div> @@ -563,11 +565,11 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>date: UTC date (YYYY-MM-DD) when download performance was measured.</li> <li>source: Name of the OnionPerf or Torperf service performing measurements.</li> <li>server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server.</li> -<li>low: Lowest latency within 1.5 IQR of lower quartile (lower whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> +<li>low: Lowest time in milliseconds between sending the HTTP request and receiving the HTTP response header within 1.5 IQR of lower quartile (lower whisker in a boxplot).</li> <li>q1: First quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>md: Median of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>q3: Third quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> -<li>high: Highest latency within 1.5 IQR of upper quartile (upper whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> +<li>high: Highest time in milliseconds between sending the HTTP request and receiving the HTTP response header within 1.5 IQR of upper quartile (upper whisker in a boxplot).</li> </ul> <h3>Throughput @@ -589,11 +591,11 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>date: UTC date (YYYY-MM-DD) when download performance was measured.</li> <li>source: Name of the OnionPerf or Torperf service performing measurements.</li> <li>server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server.</li> -<li>low: Lowest measured throughput within 1.5 IQR of lower quartile (lower whisker in a boxplot) in kilobits per second.</li> +<li>low: Lowest measured throughput in kilobits per second within 1.5 IQR of lower quartile (lower whisker in a boxplot).</li> <li>q1: First quartile of measured throughput in kilobits per second.</li> <li>md: Median of measured throughput in kilobits per second.</li> <li>q3: Third quartile of measured throughput in kilobits per second.</li> -<li>high: Highest measured throughput within 1.5 IQR of upper quartile (upper whisker in a boxplot) in kilobits per second.</li> +<li>high: Highest measured throughput in kilobits per second within 1.5 IQR of upper quartile (upper whisker in a boxplot).</li> </ul> </div>

1 0

[metrics-web/release] Tweak the new guidelines a bit.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 9955072a6cf65e7012c317ccd8ebc616a211364f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 29 10:25:22 2019 +0200 Tweak the new guidelines a bit. --- src/main/resources/web/jsps/metrics-guidelines.jsp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/web/jsps/metrics-guidelines.jsp b/src/main/resources/web/jsps/metrics-guidelines.jsp index 0f06fe5..eff0336 100644 --- a/src/main/resources/web/jsps/metrics-guidelines.jsp +++ b/src/main/resources/web/jsps/metrics-guidelines.jsp @@ -50,7 +50,7 @@ This section aims to help you organize your thoughts before making a request to the Metrics team. It might be that there are good reasons that something is not done in one of the preferred ways, but ideally data collection tools can - be written with this guidelines in mind. + be written with these guidelines in mind. <ul> <li>What is your data about? Is it about servers or users or both? Is it @@ -76,8 +76,8 @@ <li>Is the data you're planning to give us too sensitive? If so, can you sanitize it yourself before giving it to us (we can help you with that), or does the sanitizing need to happen on our side (we should still involve you in this - case)? -There are currently cases where Tor Metrics performs the sanitization of + case)? + There are currently cases where Tor Metrics performs the sanitization of data before archiving, but the preferred system would sanitize the data as close to the source as possible to minimize the possibility that sensitive data could be leaked.</li> @@ -97,9 +97,9 @@ data, maybe. And if we need to include fancy crypto libraries in order to process your data, then for sure. Any intuitions you have about possible difficulties would be good to know, even if things turn out to be easier in - the end. -As far as possible, use simple formats for providing data. The Tor Directory - Protocol meta-format [<a href="#ref-2">2</a>, §1.2] is a simple format for which we already have + the end. + As far as possible, use simple formats for providing data. The Tor Directory + Protocol meta-format [<a href="#ref-2">2</a>, §1.2] is a simple format for which we already have parsers. Without good reason, do not serialize to formats such as YAML, TOML, etc. as this would require adding a new parsing library into Tor Metrics just to parse the new data.</li>

1 0

[metrics-web/release] Re-add the total relay bandwidth graph.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit c71f0325e1f666ebf7a2fcc73724c2747b290690 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 5 10:48:41 2019 +0200 Re-add the total relay bandwidth graph. Implements 30883. --- src/main/R/rserver/rserve-init.R | 24 ++++++++++++++++++++++ src/main/resources/web.xml | 4 ++++ src/main/resources/web/json/categories.json | 1 + src/main/resources/web/json/metrics.json | 11 ++++++++++ .../resources/web/jsps/reproducible-metrics.jsp | 12 ++++++----- src/main/resources/web/jsps/stats.jsp | 21 +++++++++++++++++++ 6 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index f74fd03..af74ef7 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -829,6 +829,30 @@ plot_bandwidth_flags <- function(start_p, end_p, path_p) { ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } +prepare_bandwidth <- function(start_p = NULL, end_p = NULL) { + prepare_bandwidth_flags(start_p, end_p) %>% + group_by(date) %>% + summarize(advbw = sum(advbw), bwhist = sum(bwhist)) +} + +plot_bandwidth <- function(start_p, end_p, path_p) { + prepare_bandwidth(start_p, end_p) %>% + gather(variable, value, -date) %>% + ggplot(aes(x = date, y = value, colour = variable)) + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), + limits = c(0, NA)) + + scale_colour_hue(name = "", h.start = 90, + breaks = c("advbw", "bwhist"), + labels = c("Advertised bandwidth", "Bandwidth history")) + + ggtitle("Total relay bandwidth") + + labs(caption = copyright_notice) + + theme(legend.position = "top") + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + prepare_userstats_relay_country <- function(start_p = NULL, end_p = NULL, country_p = NULL, events_p = NULL) { read_csv(file = paste(stats_dir, "clients.csv", sep = ""), diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index f92813a..2c68fd7 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -30,6 +30,7 @@ <url-pattern>/relayflags.html</url-pattern> <url-pattern>/versions.html</url-pattern> <url-pattern>/platforms.html</url-pattern> + <url-pattern>/bandwidth.html</url-pattern> <url-pattern>/bandwidth-flags.html</url-pattern> <url-pattern>/dirbytes.html</url-pattern> <url-pattern>/advbwdist-perc.html</url-pattern> @@ -118,6 +119,9 @@ <url-pattern>/platforms.png</url-pattern> <url-pattern>/platforms.pdf</url-pattern> <url-pattern>/platforms.csv</url-pattern> + <url-pattern>/bandwidth.png</url-pattern> + <url-pattern>/bandwidth.pdf</url-pattern> + <url-pattern>/bandwidth.csv</url-pattern> <url-pattern>/bandwidth-flags.png</url-pattern> <url-pattern>/bandwidth-flags.pdf</url-pattern> <url-pattern>/bandwidth-flags.csv</url-pattern> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index ad0df11..89742bc 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -43,6 +43,7 @@ "summary": "How much traffic the Tor network can handle and how much traffic there is.", "description": "We measure total available bandwidth and current capacity by aggregating what relays and bridges report to directory authorities.", "metrics": [ + "bandwidth", "bandwidth-flags", "advbw-ipv6", "advbwdist-perc", diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 08e8c19..e6bab04 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -67,6 +67,17 @@ ] }, { + "id": "bandwidth", + "title": "Total relay bandwidth", + "type": "Graph", + "description": "This graph shows the total <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of all <a href=\"glossary.html#relay\">relays</a> in the network.", + "function": "bandwidth", + "parameters": [ + "start", + "end" + ] + }, + { "id": "bandwidth-flags", "title": "Advertised and consumed bandwidth by relay flags", "type": "Graph", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index aee4d5c..d6e7d0e 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -431,6 +431,7 @@ Relays self-report their advertised bandwidth in their server descriptors which The following description applies to the following graphs: <ul> +<li>Total relay bandwidth (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Advertised and consumed bandwidth by relay flags (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Advertised bandwidth by IP version <a href="/advbw-ipv6.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Advertised bandwidth distribution <a href="/advbwdist-perc.html" class="btn btn-primary btn-xs"> graph</a></li> @@ -468,7 +469,7 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with <h4>Step 3: Compute daily averages</h4> -The first two graphs described here, namely <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth. +The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth. In order to compute these averages, first match consensus entries with server descriptors by SHA-1 digest. Every consensus entry references exactly one server descriptor, and a server descriptor may be referenced from an arbitrary number of consensus entries. @@ -513,6 +514,7 @@ Relays self-report bandwidth histories as part of their extra-info descriptors, The following description applies to the following graphs: <ul> +<li>Total relay bandwidth (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Advertised and consumed bandwidth by relay flags (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"> graph</a></li> </ul> @@ -549,13 +551,13 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with <h4>Step 3: Compute daily totals</h4> -The first graph described here, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a>, shows daily totals of all bytes written or read by relays. -For this graph, we sum up all read and written bytes on a given day and divide the result by 2. +The first two graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a> and <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a> show daily totals of all bytes written or read by relays. +For both graphs we sum up all read and written bytes on a given day and divide the result by 2. However, we only include bandwidth histories for a given day if a relay was listed as running in a consensus at least once on that day. We attribute bandwidth to guards and/or exits if a relay was a guard and/or exit at least in one consensus on a day. -The second graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. -As opposed to the first graph, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. +The third graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. +As opposed to the first two graphs, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. Also, we compute total read directory and total written directory bytes for this fourth graph, not an average of the two. <h3 id="connbidirect" class="hover">Connection usage diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 5d653cb..eac75e3 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -52,6 +52,7 @@ https://metrics.torproject.org/identifier.csv <li>December 20, 2018: Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph to cover all data previously contained in the first two graphs.</li> <li>May 29, 2019: Extended <a href="#onionperf-latencies">Circuit round-trip latencies</a> graph to contain high/low values.</li> <li>June 2, 2019: Added <a href="#onionperf-throughput">Throughput</a> graph.</li> +<li>August 5, 2019: Re-added the <a href="#bandwidth">Total relay bandwidth</a> graph due to popular demand.</li> </ul> </div> @@ -334,6 +335,26 @@ Servers <a href="#servers" name="servers" class="anchor">#</a></h2> <h2> Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> +<h3>Total relay bandwidth +<a href="/bandwidth.html" class="btn btn-primary btn-xs"> graph</a> +<a href="/bandwidth.csv" class="btn btn-primary btn-xs"> data</a> +<a href="#bandwidth" name="bandwidth" class="anchor">#</a></h3> + +<h4>Parameters</h4> + +<ul> +<li>start: First UTC date (YYYY-MM-DD) to include in the file.</li> +<li>end: Last UTC date (YYYY-MM-DD) to include in the file.</li> +</ul> + +<h4>Columns</h4> + +<ul> +<li>date: UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> +<li>advbw: Total advertised bandwidth in Gbit/s that relays are capable to provide.</li> +<li>bwhist: Total consumed bandwidth in Gbit/s as the average of written and read traffic of all relays.</li> +</ul> + <h3>Advertised and consumed bandwidth by relay flags <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"> graph</a> <a href="/bandwidth-flags.csv" class="btn btn-primary btn-xs"> data</a>

1 0

[metrics-web/release] Include Torperf data in performance graphs again.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 896935d33cda602d9b5656a9deed392889b50ffc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jun 19 04:26:35 2019 -0700 Include Torperf data in performance graphs again. Fixes #30919. --- src/main/sql/onionperf/init-onionperf.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index b7c41d6..01bfef1 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -80,7 +80,7 @@ SELECT DATE(start) AS date, COUNT(*) AS requests FROM measurements WHERE DATE(start) < current_date - 1 -AND endpointremote NOT SIMILAR TO '_{56}.onion%' +AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%') GROUP BY date, filesize, source, server UNION SELECT DATE(start) AS date, @@ -98,7 +98,7 @@ SELECT DATE(start) AS date, COUNT(*) AS requests FROM measurements WHERE DATE(start) < current_date - 1 -AND endpointremote NOT SIMILAR TO '_{56}.onion%' +AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%') GROUP BY date, filesize, 3, server) sub ORDER BY date, filesize, source, server; @@ -140,7 +140,7 @@ WITH filtered_measurements AS ( WHERE DATE(start) < current_date - 1 AND datarequest > 0 AND dataresponse > 0 - AND endpointremote NOT SIMILAR TO '_{56}.onion%' + AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%') ), quartiles AS ( SELECT date, source, @@ -185,7 +185,7 @@ WITH filtered_measurements AS ( ELSE NULL END AS kbps FROM measurements WHERE DATE(start) < current_date - 1 - AND endpointremote NOT SIMILAR TO '_{56}.onion%' + AND (endpointremote IS NULL OR endpointremote NOT SIMILAR TO '_{56}.onion%') ), quartiles AS ( SELECT date, source,

1 0

[metrics-web/release] Fix unknown issue with spread() function.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit b3e80fce17f04cf7c83ef3843d2de0cc3bfef389 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri May 3 09:28:06 2019 +0200 Fix unknown issue with spread() function. This is a hotfix to work around the issue described in #30351. Hopefully, we'll come up with a better fix that doesn't go backwards from tidyr to reshape2. --- src/main/R/rserver/rserve-init.R | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 5a47550..3d6dad7 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -1,3 +1,4 @@ +require(reshape2) require(ggplot2) require(RColorBrewer) require(scales) @@ -436,7 +437,7 @@ prepare_platforms <- function(start_p = NULL, end_p = NULL) { filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% mutate(platform = tolower(platform)) %>% - spread(platform, relays) + dcast(date ~ platform, value.var = "relays") } plot_platforms <- function(start_p, end_p, path_p) { @@ -700,7 +701,7 @@ prepare_connbidirect <- function(start_p = NULL, end_p = NULL) { filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% mutate(quantile = paste("X", quantile, sep = ""), fraction = fraction / 100) %>% - spread(quantile, fraction) %>% + dcast(date + direction ~ quantile, value.var = "fraction") %>% rename(q1 = X0.25, md = X0.5, q3 = X0.75) } @@ -752,7 +753,8 @@ prepare_bandwidth_flags <- function(start_p = NULL, end_p = NULL) { filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% filter(!is.na(have_exit_flag)) %>% filter(!is.na(have_guard_flag)) %>% - spread(variable, value) + dcast(date + have_guard_flag + have_exit_flag ~ variable, + value.var = "value") } plot_bandwidth_flags <- function(start_p, end_p, path_p) { @@ -1058,7 +1060,7 @@ prepare_advbwdist_perc <- function(start_p = NULL, end_p = NULL, p_p = NULL) { transmute(date, percentile = as.factor(percentile), variable = ifelse(is.na(isexit), "all", "exits"), advbw = advbw * 8 / 1e9) %>% - spread(variable, advbw) %>% + dcast(date + percentile ~ variable, value.var = "advbw") %>% rename(p = percentile) } @@ -1096,7 +1098,7 @@ prepare_advbwdist_relay <- function(start_p = NULL, end_p = NULL, n_p = NULL) { transmute(date, relay = as.factor(relay), variable = ifelse(is.na(isexit), "all", "exits"), advbw = advbw * 8 / 1e9) %>% - spread(variable, advbw) %>% + dcast(date + relay ~ variable, value.var = "advbw") %>% rename(n = relay) } @@ -1194,7 +1196,7 @@ prepare_webstats_tb <- function(start_p = NULL, end_p = NULL) { filter(request_type %in% c("tbid", "tbsd", "tbup", "tbur")) %>% group_by(log_date, request_type) %>% summarize(count = sum(count)) %>% - spread(request_type, count) %>% + dcast(log_date ~ request_type, value.var = "count") %>% rename(date = log_date, initial_downloads = tbid, signature_downloads = tbsd, update_pings = tbup, update_requests = tbur) @@ -1239,7 +1241,7 @@ prepare_webstats_tb_platform <- function(start_p = NULL, end_p = NULL) { filter(request_type %in% c("tbid", "tbup")) %>% group_by(log_date, platform, request_type) %>% summarize(count = sum(count)) %>% - spread(request_type, count, fill = 0) %>% + dcast(log_date + platform ~ request_type, value.var = "count") %>% rename(date = log_date, initial_downloads = tbid, update_pings = tbup) } @@ -1287,7 +1289,7 @@ prepare_webstats_tb_locale <- function(start_p = NULL, end_p = NULL) { group_by(date, locale, request_type) %>% summarize(count = sum(count)) %>% mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>% - spread(request_type, count, fill = 0) %>% + dcast(date + locale ~ request_type, value.var = "count") %>% rename(initial_downloads = tbid, update_pings = tbup) } @@ -1342,7 +1344,8 @@ prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) { group_by(log_date, request_type) %>% summarize(count = sum(count)) %>% mutate(request_type = factor(request_type, levels = c("tmid", "tmup"))) %>% - spread(request_type, count, drop = FALSE, fill = 0) %>% + dcast(log_date ~ request_type, value.var = "count", drop = FALSE, + fill = 0) %>% rename(date = log_date, initial_downloads = tmid, update_pings = tmup) }

1 0

[metrics-web/release] Fix two issues found using metrics-test.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 5d98d4806e01169f226bace3fc095298844d4427 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 16 16:29:10 2019 +0200 Fix two issues found using metrics-test. - Make unit test locale-independent, similar to Onionoo's 860228c. - Replace deprecated method Class.newInstance(). --- .../java/org/torproject/metrics/stats/hidserv/Aggregator.java | 5 +++-- .../metrics/stats/hidserv/ComputedNetworkFractions.java | 5 +++-- .../org/torproject/metrics/stats/hidserv/DocumentStore.java | 6 ++++-- .../metrics/stats/hidserv/ExtrapolatedHidServStats.java | 10 ++++++---- .../java/org/torproject/metrics/stats/hidserv/Simulate.java | 9 +++++---- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java index 36e7967..12a89a3 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -176,8 +177,8 @@ public class Aggregator { /* Put together all aggregated values in a single line. */ String date = e.getKey(); int numStats = weightedValues.size(); - sb.append(String.format("%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n", date, - type, weightedMean, weightedMedian, weightedInterquartileMean, + sb.append(String.format(Locale.US, "%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n", + date, type, weightedMean, weightedMedian, weightedInterquartileMean, sumFraction, numStats)); } } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java index d5fd279..835ac6f 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java @@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; /** Computed fraction of hidden-service activity that a single relay is @@ -116,9 +117,9 @@ public class ComputedNetworkFractions implements Document { validAfterDate); String second = validAfterHour + (this.fractionRendRelayedCells == 0.0 ? "," - : String.format(",%f", this.fractionRendRelayedCells)) + : String.format(Locale.US, ",%f", this.fractionRendRelayedCells)) + (this.fractionDirOnionsSeen == 0.0 ? "," - : String.format(",%f", this.fractionDirOnionsSeen)); + : String.format(Locale.US, ",%f", this.fractionDirOnionsSeen)); return new String[] { first, second }; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java index f3163e1..9061aa3 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java @@ -13,6 +13,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.LineNumberReader; +import java.lang.reflect.InvocationTargetException; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -136,7 +137,7 @@ public class DocumentStore<T extends Document> { && !formattedString0.startsWith(prefix)) { /* Skip line not starting with prefix. */ } else { - T document = this.clazz.newInstance(); + T document = this.clazz.getDeclaredConstructor().newInstance(); if (!document.parse(new String[] { formattedString0, line.substring(1) })) { log.warn("Unable to read line {} from {}. Not retrieving any " @@ -151,7 +152,8 @@ public class DocumentStore<T extends Document> { log.warn("Unable to read {}. Not retrieving any previously stored " + "documents.", documentFile.getAbsolutePath(), e); return null; - } catch (InstantiationException | IllegalAccessException e) { + } catch (InstantiationException | IllegalAccessException + | NoSuchMethodException | InvocationTargetException e) { log.warn("Unable to read {}. Cannot instantiate document object.", documentFile.getAbsolutePath(), e); return null; diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java index ec09630..eb66c7a 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java @@ -6,6 +6,8 @@ package org.torproject.metrics.stats.hidserv; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Locale; + /** Extrapolated network totals of hidden-service statistics reported by a * single relay. Extrapolated values are based on reported statistics and * computed network fractions in the statistics interval. */ @@ -116,11 +118,11 @@ public class ExtrapolatedHidServStats implements Document { DateTimeHelper.ISO_DATE_FORMAT); String second = this.fingerprint + (this.fractionRendRelayedCells == 0.0 ? ",," - : String.format(",%.0f,%f", this.extrapolatedRendRelayedCells, - this.fractionRendRelayedCells)) + : String.format(Locale.US, ",%.0f,%f", + this.extrapolatedRendRelayedCells, this.fractionRendRelayedCells)) + (this.fractionDirOnionsSeen == 0.0 ? ",," - : String.format(",%.0f,%f", this.extrapolatedDirOnionsSeen, - this.fractionDirOnionsSeen)); + : String.format(Locale.US, ",%.0f,%f", + this.extrapolatedDirOnionsSeen, this.fractionDirOnionsSeen)); return new String[] { first, second }; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java b/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java index ac8a82e..445d931 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.SortedMap; @@ -196,8 +197,8 @@ public class Simulate { totalInterquartileProbability += extrapolation[2]; } } - sb.append(String.format("%d,%.2f,%.0f,%.0f,%.0f%n", run, fraction, - totalValues / totalProbability, weightedMedian, + sb.append(String.format(Locale.US, "%d,%.2f,%.0f,%.0f,%.0f%n", run, + fraction, totalValues / totalProbability, weightedMedian, totalInterquartileValues / totalInterquartileProbability)); } return sb.toString(); @@ -351,8 +352,8 @@ public class Simulate { totalInterquartileProbability += extrapolation[2]; } } - sb.append(String.format("%d,%.2f,%.0f,%.0f,%.0f%n", run, fraction, - totalValues / totalProbability, weightedMedian, + sb.append(String.format(Locale.US, "%d,%.2f,%.0f,%.0f,%.0f%n", run, + fraction, totalValues / totalProbability, weightedMedian, totalInterquartileValues / totalInterquartileProbability)); } return sb.toString();

1 0

[metrics-web/release] Adds explanation for first-hop build times overhead
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 6a11d7ac3c372a99156be1ecf10c69f5afe3b6a3 Author: Ana Custura <ana(a)netstat.org.uk> Date: Sat Jul 6 11:41:50 2019 +0100 Adds explanation for first-hop build times overhead --- src/main/resources/web/json/metrics.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index bfcde22..08e8c19 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -289,7 +289,7 @@ "id": "onionperf-buildtimes", "title": "Circuit build times", "type": "Graph", - "description": "This graph shows build times of circuits used for downloading static files of different sizes over Tor. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", + "description": "This graph shows build times of circuits used for downloading static files of different sizes over Tor. We disable Entry Guards in the Tor client configuration used by OnionPerf, to ensure a new guard is selected for each circuit we measure. In comparison, standard clients will reuse connections to a small number of long-term entry guards, and do not see the latency introduced by establishing new guard connections. As a result, first hop build times appear higher for these measurements compared to those commonly observed in Tor clients. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", "function": "onionperf_buildtimes", "parameters": [ "start",

1 0