May 2019 - tor-commits - lists.torproject.org

[tor/master] rng_test_helpers: add a needless lock/unlock pair to please coverity
by asn＠torproject.org 30 May '19

30 May '19

commit 1bf451cffba5da84166dda48ec957e0b9cf45bee Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 16 15:03:54 2019 -0400 rng_test_helpers: add a needless lock/unlock pair to please coverity Fix for CID 1444908 --- src/test/rng_test_helpers.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index 262d380bd..04b713847 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -183,11 +183,14 @@ void testing_enable_prefilled_rng(const void *buffer, size_t buflen) { tor_assert(buflen > 0); + tor_assert(!rng_mutex); rng_mutex = tor_mutex_new(); + tor_mutex_acquire(rng_mutex); prefilled_rng_buffer = tor_memdup(buffer, buflen); prefilled_rng_buflen = buflen; prefilled_rng_idx = 0; + tor_mutex_release(rng_mutex); MOCK(crypto_rand, crypto_rand_prefilled); MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);

1 0

[tor/master] changes file for coverity fixes in tests (30150)
by asn＠torproject.org 30 May '19

30 May '19

commit 6246f4539ed317e05b773b0e373f1aa369d3759d Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 16 15:38:08 2019 -0400 changes file for coverity fixes in tests (30150) --- changes/ticket30150 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changes/ticket30150 b/changes/ticket30150 new file mode 100644 index 000000000..70808fa5d --- /dev/null +++ b/changes/ticket30150 @@ -0,0 +1,4 @@ + o Minor bugfixes (static analysis): + - Fix several spurious Coverity warnings about the unit tests, to lower our + chances of missing any real warnings in the future. Fixes bug 30150; + bugfix on 0.3.5.1-alpha and various other Tor versions.

1 0

[stem/master] Travis: remove shellcheck
by atagar＠torproject.org 30 May '19

30 May '19

commit 026dfa4808cf5f9b56dbfba328919e6cfc3b666e Author: teor <teor(a)torproject.org> Date: Thu May 30 18:52:01 2019 +1000 Travis: remove shellcheck The last shell script has been removed from stem master, so there's no need for shellcheck in CI any more. Implements 30650. --- .travis.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b6ba37e..fdce9332 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,6 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.2.9.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - - shellcheck - tor env: TOR="0.2.9-nightly" - addons: @@ -48,7 +47,6 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.3.4.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - - shellcheck - tor env: TOR="0.3.4-nightly" ## The current stable release is 0.3.5, so we don't have a separate @@ -59,7 +57,6 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - - shellcheck - tor env: TOR="stable-release" - addons: @@ -68,7 +65,6 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-0.3.5.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - - shellcheck - tor env: TOR="0.3.5-nightly" ## The experimental deb line needs to be updated when 0.4.0 becomes stable @@ -78,7 +74,6 @@ matrix: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-experimental-0.4.0.x-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - - shellcheck - tor env: TOR="0.4.0-nightly" @@ -151,12 +146,10 @@ addons: - sourceline: 'deb https://deb.torproject.org/torproject.org tor-nightly-master-xenial main' key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - - shellcheck - tor ## (macOS only) homebrew: packages: - - shellcheck - tor install: @@ -168,12 +161,9 @@ install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --versions; fi - pip list - python --version - - shellcheck --version - tor --version script: - ## Check the only shell script in the repository - - shellcheck docs/republish ## Run stem's tests - ./run_tests.py --unit - ./run_tests.py --integ

1 0

[stem/master] Travis: add --target ONLINE to the latest python and tor versions
by atagar＠torproject.org 30 May '19

30 May '19

commit 36dc1ff55af20a20ad2a9836af0c535159e92cc8 Author: teor <teor(a)torproject.org> Date: Thu May 30 19:05:08 2019 +1000 Travis: add --target ONLINE to the latest python and tor versions Add --target ONLINE on: * macOS, tor stable, python 2.7 * Linux, tor master, python 2.7 * Linux, tor master, python master Closes 30563. --- .travis.yml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index fdce9332..91029687 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: - os: osx language: c python: - env: TOR="stable-release" + env: TOR="stable-release" TARGET="--target ONLINE" ## Test all supported and available tor versions on Linux ## If the deb.torproject.org repositories are removed, we will fall back to @@ -40,7 +40,7 @@ matrix: key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - tor - env: TOR="0.2.9-nightly" + env: TOR="0.2.9-nightly" TARGET="" - addons: apt: sources: @@ -48,7 +48,7 @@ matrix: key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - tor - env: TOR="0.3.4-nightly" + env: TOR="0.3.4-nightly" TARGET="" ## The current stable release is 0.3.5, so we don't have a separate ## 0.3.5-stable job - addons: @@ -58,7 +58,7 @@ matrix: key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - tor - env: TOR="stable-release" + env: TOR="stable-release" TARGET="" - addons: apt: sources: @@ -66,7 +66,7 @@ matrix: key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - tor - env: TOR="0.3.5-nightly" + env: TOR="0.3.5-nightly" TARGET="" ## The experimental deb line needs to be updated when 0.4.0 becomes stable - addons: apt: @@ -75,7 +75,7 @@ matrix: key_url: 'https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E8…' packages: - tor - env: TOR="0.4.0-nightly" + env: TOR="0.4.0-nightly" TARGET="" ## Test all supported python releases ## Pre-installed in Travis xenial: @@ -83,29 +83,37 @@ matrix: ## End of Life: 1 January 2020 ## https://www.python.org/dev/peps/pep-0373/#id2 - python: "2.7" + env: TOR="master-nightly" TARGET="--target ONLINE" ## End of Life: December 2021 ## https://www.python.org/dev/peps/pep-0494/#lifespan - python: "3.6" + env: TOR="master-nightly" TARGET="" ## End of Life: June 2023 ## https://www.python.org/dev/peps/pep-0537/#lifespan - python: "3.7" + env: TOR="master-nightly" TARGET="" ## Stable: 20 October 2019 ## (Switch from 3.8-dev to 3.8, and check for 3.9-dev) ## End of Life: October2024 ## https://www.python.org/dev/peps/pep-0569/#lifespan - python: "3.8-dev" + env: TOR="master-nightly" TARGET="" - python: "nightly" + env: TOR="master-nightly" TARGET="--target ONLINE" # PyPy versions ## End of Life: "forever" ## http://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2 ## But stem may decide not to support python 2 after 1 Jan 2020. ## Travis Xenial does not support pypy2.7 #- python: "pypy2.7" + # env: TOR="master-nightly" TARGET="" ## PyPy does not have documented end of life dates - python: "pypy3.5" + env: TOR="master-nightly" TARGET="" ## PyPy 3.6 is currently in alpha ## Travis Xenial does not support pypy3.6-dev yet? #- python: "pypy3.6-dev" + # env: TOR="master-nightly" TARGET="" ## Uncomment to allow the build to report success (with non-required ## sub-builds continuing to run) if all required sub-builds have @@ -117,7 +125,7 @@ matrix: ## These builds fail in Travis at the moment #allow_failures: - # - env: TOR="master-nightly" + # - env: TOR="master-nightly" TARGET="--target ONLINE" ## We don't need sudo. (The "apt:" stanza after this allows us to not need ## sudo; otherwise, we would need it for getting dependencies.) @@ -133,10 +141,9 @@ dist: xenial #osx_image: xcode9.4 ## The TOR env var should be kept in sync with the Linux tor version below -## We use the basic-min network by default, to reduce load and increase -## reliability +## This line doesn't actually create any Travis jobs env: - - TOR="master-nightly" + - TOR="master-nightly" TARGET="--target ONLINE" ## Download our dependencies addons: @@ -166,7 +173,7 @@ install: script: ## Run stem's tests - ./run_tests.py --unit - - ./run_tests.py --integ + - ./run_tests.py --integ $TARGET after_failure: ## Show the integ tor log

1 0

[tor/master] changes file for 30629
by dgoulet＠torproject.org 30 May '19

30 May '19

commit 8f3430fc28be9ac16909fd02e0d48aad4ccc128e Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu May 30 12:58:12 2019 -0400 changes file for 30629 --- changes/bug30629 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changes/bug30629 b/changes/bug30629 new file mode 100644 index 000000000..59fa96ee6 --- /dev/null +++ b/changes/bug30629 @@ -0,0 +1,6 @@ + o Minor bugfixes (shutdown, libevent, memory safety): + - Avoid use-after-free bugs when shutting down, by making sure that we + shut down libevent only after shutting down all of its users. We + believe these are harmless in practice, since they only occur on the + shutdown path, and do not involve any attacker-controlled data. Fixes + bug 30629; bugfix on 0.4.1.1-alpha.

1 0

[tor/master] Merge branch 'tor-github/pr/1057'
by dgoulet＠torproject.org 30 May '19

30 May '19

commit 8dfc8d7063bf4b317bd47b7a2e8cccb0a428e861 Merge: a462ca7cc 8f3430fc2 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu May 30 12:58:35 2019 -0400 Merge branch 'tor-github/pr/1057' changes/bug30629 | 6 ++++++ src/app/main/shutdown.c | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-)

1 0

[tor/master] Shutdown libevent _after_ the subsystems.
by dgoulet＠torproject.org 30 May '19

30 May '19

commit ba9b0319b00692038e5d4aa2eff7cf5d6947e659 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 29 11:25:47 2019 -0400 Shutdown libevent _after_ the subsystems. This is necessary since shutting down libevent frees some pointer that the subsystems want to free themselves. A longer term solution will be to turn the evloop module into a subsystem itself, but for now it is best to do the minimal fix. Fixes bug 30629; bugfix on 0.4.1.1-alpha. --- src/app/main/shutdown.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index e4dcaa132..cc0091a9a 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -157,10 +157,11 @@ tor_free_all(int postfork) if (!postfork) { release_lockfile(); } - tor_libevent_free_all(); subsystems_shutdown(); + tor_libevent_free_all(); + /* Stuff in util.c and address.c*/ if (!postfork) { esc_router_info(NULL);

1 0

[tor/master] Remove want_cmddata from HSFETCH, which does not in fact want data
by dgoulet＠torproject.org 30 May '19

30 May '19

commit 2d66250d8acfc61210442df94fe04df09c2dbab6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed May 29 10:10:57 2019 -0400 Remove want_cmddata from HSFETCH, which does not in fact want data This looks a copy-and-paste error to me. Fixes bug 30646; bugfix on 0.4.1.1-alpha. --- changes/bug30646 | 4 ++++ src/feature/control/control_cmd.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changes/bug30646 b/changes/bug30646 new file mode 100644 index 000000000..e95a54e3e --- /dev/null +++ b/changes/bug30646 @@ -0,0 +1,4 @@ + o Minor bugfixes (controller): + - Repair the HSFETCH command so that it works again. Previously, it + expected a body when it shouldn't have. Fixes bug 30646; bugfix on + 0.4.1.1-alpha. diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 17d5b0c7f..abb579bd4 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1385,7 +1385,6 @@ static const control_cmd_syntax_t hsfetch_syntax = { .min_args = 1, .max_args = 1, .accept_keywords = true, .allowed_keywords = hsfetch_keywords, - .want_cmddata = true, }; /** Implementation for the HSFETCH command. */

1 0

[tor/master] Merge branch 'tor-github/pr/1055'
by dgoulet＠torproject.org 30 May '19

30 May '19

commit a462ca7cce3699f488b5e2f2921738c944cb29c7 Merge: ef9170db4 2d66250d8 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu May 30 12:53:52 2019 -0400 Merge branch 'tor-github/pr/1055' changes/bug30646 | 4 ++++ src/feature/control/control_cmd.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-)

1 0

[metrics-web/master] Extend latency graph to contain high/low values.
by karsten＠torproject.org 30 May '19

30 May '19

commit fd251d6c69efeb3d548f915cc40e001e89c1f41f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 29 16:10:40 2019 +0200 Extend latency graph to contain high/low values. This patch adds two new lines to the existing circuit round-trip latencies graph: lowest and highest measurements that are not outliers. Implements #29773. --- src/main/R/rserver/rserve-init.R | 15 ++++-- .../torproject/metrics/stats/onionperf/Main.java | 10 ++-- src/main/resources/web/json/metrics.json | 2 +- .../resources/web/jsps/reproducible-metrics.jsp | 3 +- src/main/resources/web/jsps/stats.jsp | 2 + src/main/sql/onionperf/init-onionperf.sql | 58 +++++++++++----------- 6 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index 3d6dad7..88aa5b9 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -663,9 +663,11 @@ prepare_onionperf_latencies <- function(start_p = NULL, end_p = NULL, date = col_date(format = ""), source = col_character(), server = col_character(), + low = col_double(), q1 = col_double(), md = col_double(), - q3 = col_double())) %>% + q3 = col_double(), + high = col_double())) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% filter(if (!is.null(server_p)) server == server_p else TRUE) @@ -675,18 +677,23 @@ plot_onionperf_latencies <- function(start_p, end_p, server_p, path_p) { prepare_onionperf_latencies(start_p, end_p, server_p) %>% filter(source != "") %>% complete(date = full_seq(date, period = 1), nesting(source)) %>% - ggplot(aes(x = date, y = md, ymin = q1, ymax = q3, fill = source)) + + ggplot(aes(x = date, ymin = q1, ymax = q3, fill = source)) + geom_ribbon(alpha = 0.5) + - geom_line(aes(colour = source), size = 0.75) + + geom_line(aes(y = md, colour = source), size = 0.75) + + geom_line(aes(y = high, colour = source), size = 0.375) + + geom_line(aes(y = low, colour = source), size = 0.375) + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) + scale_y_continuous(name = "", labels = unit_format(unit = "ms"), limits = c(0, NA)) + scale_fill_hue(name = "Source") + scale_colour_hue(name = "Source") + + facet_grid(source ~ ., scales = "free", space = "free") + ggtitle(paste("Circuit round-trip latencies to", server_p, "server")) + labs(caption = copyright_notice) + - theme(legend.position = "top") + theme(legend.position = "none", + strip.text.y = element_text(angle = 0, hjust = 0), + strip.background = element_rect(fill = NA)) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index a970434..a75cd1b 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -298,22 +298,24 @@ public class Main { throws SQLException { log.info("Querying latency statistics from database."); List<String> statistics = new ArrayList<>(); - statistics.add("date,source,server,q1,md,q3"); + statistics.add("date,source,server,low,q1,md,q3,high"); Statement st = connection.createStatement(); - String queryString = "SELECT date, source, server, q1, md, q3 " + String queryString = "SELECT date, source, server, low, q1, md, q3, high " + "FROM latencies_stats"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { - statistics.add(String.format("%s,%s,%s,%d,%d,%d", + statistics.add(String.format("%s,%s,%s,%d,%d,%d,%d,%d", dateFormat.format(rs.getDate("date", calendar)), getStringFromResultSet(rs, "source"), rs.getString("server"), + rs.getInt("low"), rs.getInt("q1"), rs.getInt("md"), - rs.getInt("q3"))); + rs.getInt("q3"), + rs.getInt("high"))); } } return statistics; diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 5c99546..006de8b 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -300,7 +300,7 @@ "id": "onionperf-latencies", "title": "Circuit round-trip latencies", "type": "Graph", - "description": "This graph shows round-trip latencies of circuits used for downloading static files of different sizes over Tor, either from a server on the public internet or from a version 2 onion server. Round-trip latencies are measured as the time between sending the HTTP request and receiving the HTTP response header. The graph shows the range of measurements from first to third quartile, and highlights the median. The slowest and fastest quarter of measurements are omitted from the graph.", + "description": "This graph shows round-trip latencies of circuits used for downloading static files of different sizes over Tor, either from a server on the public internet or from a version 2 onion server. Round-trip latencies are measured as the time between sending the HTTP request and receiving the HTTP response header. The graph shows the median of measurements as thick line, the range of measurements from first to third quartile as ribbon, and the highest and lowest non-outlier measurements as thin lines.", "function": "onionperf_latencies", "parameters": [ "start", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 3838972..a833b31 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -650,7 +650,8 @@ We then compute the 25th, 50th, and 75th percentile of download times by sorting We're also interested in circuit round-trip latencies for the <a href="/onionperf-latencies.html">Circuit round-trip latencies</a> graph. We measure circuit latency as the time between sending the HTTP request and receiving the HTTP response header. We calculate latencies as <code>DATARESPONSE - DATAREQUEST</code> for measurements with non-zero values for both timestamps. -We then compute 25th, 50th, and 75th percentiles in the same way as for download times above. +We then compute 25th, 50th, and 75th percentiles in the same way as for download times above. +We also compute the lowest latency within 1.5 IQR of the lower quartile and the highest latency within 1.5 IQR of the upper quartile. Ideally, all measurements would succeed. But it's also possible that some measurements did not complete within a pre-defined timeout or failed for some other reason. diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 6c1955a..443c292 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -563,9 +563,11 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <li>date: UTC date (YYYY-MM-DD) when download performance was measured.</li> <li>source: Name of the OnionPerf or Torperf service performing measurements.</li> <li>server: Either "public" if the request was made to a server on the public internet, or "onion" if the request was made to a version 2 onion server.</li> +<li>low: Lowest latency within 1.5 IQR of lower quartile (lower whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>q1: First quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>md: Median of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> <li>q3: Third quartile of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> +<li>high: Highest latency within 1.5 IQR of upper quartile (upper whisker in a boxplot) of time in milliseconds between sending the HTTP request and receiving the HTTP response header.</li> </ul> </div> diff --git a/src/main/sql/onionperf/init-onionperf.sql b/src/main/sql/onionperf/init-onionperf.sql index 7c0176e..536cd45 100644 --- a/src/main/sql/onionperf/init-onionperf.sql +++ b/src/main/sql/onionperf/init-onionperf.sql @@ -130,37 +130,37 @@ GROUP BY date, 2, position) sub ORDER BY date, source, position; CREATE OR REPLACE VIEW latencies_stats AS +WITH filtered_measurements AS ( + SELECT DATE(start) AS date, + source, + CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' + ELSE 'public' END AS server, + dataresponse - datarequest AS latency + FROM measurements + WHERE DATE(start) < current_date - 1 + AND datarequest > 0 + AND dataresponse > 0 + AND endpointremote NOT SIMILAR TO '_{56}.onion%' +), quartiles AS ( + SELECT date, + source, + server, + PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) + WITHIN GROUP(ORDER BY latency) AS q + FROM filtered_measurements + GROUP BY date, source, server +) SELECT date, source, server, - TRUNC(q[1]) AS q1, - TRUNC(q[2]) AS md, - TRUNC(q[3]) AS q3 -FROM ( -SELECT DATE(start) AS date, - source, - CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' - ELSE 'public' END AS server, - PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) - WITHIN GROUP(ORDER BY dataresponse - datarequest) AS q -FROM measurements -WHERE DATE(start) < current_date - 1 -AND datarequest > 0 -AND dataresponse > 0 -AND endpointremote NOT SIMILAR TO '_{56}.onion%' -GROUP BY date, source, server -UNION -SELECT DATE(start) AS date, - '' AS source, - CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion' - ELSE 'public' END AS server, - PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) - WITHIN GROUP(ORDER BY dataresponse - datarequest) AS q -FROM measurements -WHERE DATE(start) < current_date - 1 -AND datarequest > 0 -AND dataresponse > 0 -AND endpointremote NOT SIMILAR TO '_{56}.onion%' -GROUP BY date, 2, server) sub + MIN(CASE WHEN latency >= q[1] - ((q[3] - q[1]) * 1.5) + THEN latency ELSE NULL END) AS low, + TRUNC(AVG(q[1])) AS q1, + TRUNC(AVG(q[2])) AS md, + TRUNC(AVG(q[3])) AS q3, + MAX(CASE WHEN latency <= q[3] + ((q[3] - q[1]) * 1.5) + THEN latency ELSE NULL END) AS high +FROM filtered_measurements NATURAL JOIN quartiles +GROUP BY 1, 2, 3 ORDER BY date, source, server;

1 0