November 2019 - tor-commits - lists.torproject.org

[metrics-web/release] Compute percentiles in advbwdist module using linear interpolation.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit dda2b51eaa6efdd871d5190f4b86fad008beee42 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat May 19 21:08:42 2018 +0200 Compute percentiles in advbwdist module using linear interpolation. So far we computed percentiles in the advbwdist module using our own formula that picked existing values for the various percentiles we provided. Now we're using Apache Commons Math with method R_7 which uses linear interpolation when a percentile lies between two data points. Implements another part of #26035. --- build.xml | 1 + .../torproject/metrics/stats/advbwdist/Main.java | 49 ++++++++++++--- .../metrics/stats/advbwdist/MainTest.java | 72 ++++++++++++++++++++++ 3 files changed, 112 insertions(+), 10 deletions(-) diff --git a/build.xml b/build.xml index 3614d78..b498ea8 100644 --- a/build.xml +++ b/build.xml @@ -60,6 +60,7 @@ <patternset refid="common" /> <include name="metrics-lib-${metricslibversion}.jar"/> <include name="commons-compress-1.13.jar"/> + <include name="commons-math3-3.6.1.jar"/> <include name="postgresql-9.4.1212.jar"/> <include name="servlet-api-3.1.jar"/> <include name="xz-1.6.jar"/> diff --git a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java index a16e7b0..7216581 100644 --- a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java @@ -10,6 +10,8 @@ import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor; +import org.apache.commons.math3.stat.descriptive.rank.Percentile; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -20,7 +22,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.SortedMap; import java.util.TimeZone; +import java.util.TreeMap; public class Main { @@ -115,29 +119,54 @@ public class Main { } /* Write advertised bandwidth percentiles of relays/exits. */ - Collections.sort(advertisedBandwidthsAllRelays); - Collections.sort(advertisedBandwidthsExitsOnly); int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 }; if (!advertisedBandwidthsAllRelays.isEmpty()) { - for (int percentile : percentiles) { + for (Map.Entry<Integer, Long> e : computePercentiles( + advertisedBandwidthsAllRelays, percentiles).entrySet()) { bw.write(String.format("%s,,,%d,%d%n", validAfter, - percentile, advertisedBandwidthsAllRelays.get( - ((advertisedBandwidthsAllRelays.size() - 1) - * percentile) / 100))); + e.getKey(), e.getValue())); } } if (!advertisedBandwidthsExitsOnly.isEmpty()) { - for (int percentile : percentiles) { + for (Map.Entry<Integer, Long> e : computePercentiles( + advertisedBandwidthsExitsOnly, percentiles).entrySet()) { bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter, - percentile, advertisedBandwidthsExitsOnly.get( - ((advertisedBandwidthsExitsOnly.size() - 1) - * percentile) / 100))); + e.getKey(), e.getValue())); } } } descriptorReader.saveHistoryFile(historyFile); bw.close(); } + + /** Compute percentiles (between 0 and 100) for the given list of values, and + * return a map with percentiles as keys and computed values as values. If the + * list of values is empty, the returned map contains all zeros. */ + static SortedMap<Integer, Long> computePercentiles( + List<Long> valueList, int ... percentiles) { + SortedMap<Integer, Long> computedPercentiles = new TreeMap<>(); + double[] valueArray = new double[valueList.size()]; + long minValue = Long.MAX_VALUE; + for (int i = 0; i < valueList.size(); i++) { + valueArray[i] = valueList.get(i).doubleValue(); + minValue = Math.min(minValue, valueList.get(i)); + } + if (valueList.isEmpty()) { + minValue = 0L; + } + Percentile percentile = new Percentile() + .withEstimationType(Percentile.EstimationType.R_7); + percentile.setData(valueArray); + for (int p : percentiles) { + if (0 == p) { + computedPercentiles.put(p, minValue); + } else { + computedPercentiles.put(p, + (long) Math.floor(percentile.evaluate((double) p))); + } + } + return computedPercentiles; + } } diff --git a/src/test/java/org/torproject/metrics/stats/advbwdist/MainTest.java b/src/test/java/org/torproject/metrics/stats/advbwdist/MainTest.java new file mode 100644 index 0000000..92752ef --- /dev/null +++ b/src/test/java/org/torproject/metrics/stats/advbwdist/MainTest.java @@ -0,0 +1,72 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.advbwdist; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.SortedMap; + +public class MainTest { + + @Test + public void testComputePercentilesZeroValues() { + List<Long> valueList = new ArrayList<>(); + SortedMap<Integer, Long> computedPercentiles = Main.computePercentiles( + valueList, 0, 25, 50, 75, 100); + assertEquals(0L, (long) computedPercentiles.get(0)); + assertEquals(0L, (long) computedPercentiles.get(25)); + assertEquals(0L, (long) computedPercentiles.get(50)); + assertEquals(0L, (long) computedPercentiles.get(75)); + assertEquals(0L, (long) computedPercentiles.get(100)); + } + + @Test + public void testComputePercentilesTenValues() { + List<Long> valueList = new ArrayList<>(); + valueList.add(3L); + valueList.add(6L); + valueList.add(7L); + valueList.add(8L); + valueList.add(8L); + valueList.add(10L); + valueList.add(13L); + valueList.add(15L); + valueList.add(16L); + valueList.add(20L); + SortedMap<Integer, Long> computedPercentiles = Main.computePercentiles( + valueList, 0, 25, 50, 75, 100); + assertEquals(3L, (long) computedPercentiles.get(0)); + assertEquals(7L, (long) computedPercentiles.get(25)); + assertEquals(9L, (long) computedPercentiles.get(50)); + assertEquals(14L, (long) computedPercentiles.get(75)); + assertEquals(20L, (long) computedPercentiles.get(100)); + } + + @Test + public void testComputePercentilesElevenValues() { + List<Long> valueList = new ArrayList<>(); + valueList.add(3L); + valueList.add(6L); + valueList.add(7L); + valueList.add(8L); + valueList.add(8L); + valueList.add(9L); + valueList.add(10L); + valueList.add(13L); + valueList.add(15L); + valueList.add(16L); + valueList.add(20L); + SortedMap<Integer, Long> computedPercentiles = Main.computePercentiles( + valueList, 0, 25, 50, 75, 100); + assertEquals(3L, (long) computedPercentiles.get(0)); + assertEquals(7L, (long) computedPercentiles.get(25)); + assertEquals(9L, (long) computedPercentiles.get(50)); + assertEquals(14L, (long) computedPercentiles.get(75)); + assertEquals(20L, (long) computedPercentiles.get(100)); + } +}

1 0

[metrics-web/release] Resolve a few checkstyle issues.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit a0e9033ed36024d90b0971afa2e5e197939c7876 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Oct 25 11:30:38 2018 +0200 Resolve a few checkstyle issues. --- src/main/java/org/torproject/metrics/stats/totalcw/Parser.java | 3 ++- .../metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java index 155b6b0..4367200 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java @@ -3,10 +3,11 @@ package org.torproject.metrics.stats.totalcw; -import org.apache.commons.math3.stat.descriptive.rank.Percentile; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusVote; +import org.apache.commons.math3.stat.descriptive.rank.Percentile; + import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; diff --git a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java index 085a976..dc9df7f 100644 --- a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java +++ b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java @@ -28,6 +28,7 @@ import java.util.Collection; @RunWith(Parameterized.class) public class TotalcwRelayNetworkStatusVoteTest { + /** Provide test data. */ @Parameters public static Collection<Object[]> data() { return Arrays.asList(new Object[][] {

1 0

[metrics-web/release] Compute percentiles in connbidirect module using linear interpolation.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit cda16c857e942a822df6b28f23518fc087bbc0f4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat May 19 21:11:49 2018 +0200 Compute percentiles in connbidirect module using linear interpolation. So far we computed percentiles in the connbidirect module using our own formula that picked existing values for median and quartiles. Now we're using Apache Commons Math with method R_7 which uses linear interpolation when a percentile lies between two data points. Implements the last part of #26035. --- .../metrics/stats/connbidirect/Main.java | 31 ++++++++-- .../metrics/stats/connbidirect/MainTest.java | 71 +++++++++++++++++++--- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java index b2dc1b6..468ffba 100644 --- a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java +++ b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java @@ -8,6 +8,8 @@ import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor; +import org.apache.commons.math3.stat.descriptive.rank.Percentile; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +24,6 @@ import java.io.StringReader; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -455,19 +456,39 @@ public class Main { } } final String[] quantiles = new String[] { "0.25", "0.5", "0.75" }; - final int[] centiles = new int[] { 25, 50, 75 }; + final double[] centiles = new double[] { 25.0, 50.0, 75.0 }; for (Map.Entry<String, List<Short>> e : fractionsByDateAndDirection.entrySet()) { String dateAndDirection = e.getKey(); List<Short> fractions = e.getValue(); - Collections.sort(fractions); + SortedMap<Double, Short> computedPercentiles + = computePercentiles(fractions, centiles); for (int i = 0; i < quantiles.length; i++) { String dateDirectionAndQuantile = dateAndDirection + "," + quantiles[i]; - short fraction = fractions.get((centiles[i] * fractions.size()) - / 100); + short fraction = computedPercentiles.get(centiles[i]); aggregateStats.put(dateDirectionAndQuantile, fraction); } } } + + /** Compute percentiles (greater than 0.0 and smaller than or equal 100.0) for + * the given list of values, and return a map with percentiles as keys and + * computed values as values. If the list of values is empty, the returned map + * contains all zeros. */ + static SortedMap<Double, Short> computePercentiles(List<Short> valueList, + double ... percentiles) { + SortedMap<Double, Short> computedPercentiles = new TreeMap<>(); + double[] valueArray = new double[valueList.size()]; + for (int i = 0; i < valueList.size(); i++) { + valueArray[i] = valueList.get(i).doubleValue(); + } + Percentile percentile = new Percentile() + .withEstimationType(Percentile.EstimationType.R_7); + percentile.setData(valueArray); + for (double p : percentiles) { + computedPercentiles.put(p, (short) Math.floor(percentile.evaluate(p))); + } + return computedPercentiles; + } } diff --git a/src/test/java/org/torproject/metrics/stats/connbidirect/MainTest.java b/src/test/java/org/torproject/metrics/stats/connbidirect/MainTest.java index 255cccf..69001da 100644 --- a/src/test/java/org/torproject/metrics/stats/connbidirect/MainTest.java +++ b/src/test/java/org/torproject/metrics/stats/connbidirect/MainTest.java @@ -11,6 +11,8 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; @@ -242,15 +244,15 @@ public class MainTest { @Test public void testUpdateAggregateStatsTwoRawStat() { SortedMap<String, Short> expectedAggregateStats = new TreeMap<>(); - expectedAggregateStats.put("2015-08-18,read,0.25", (short) 32); - expectedAggregateStats.put("2015-08-18,read,0.5", (short) 42); - expectedAggregateStats.put("2015-08-18,read,0.75", (short) 42); - expectedAggregateStats.put("2015-08-18,write,0.25", (short) 22); - expectedAggregateStats.put("2015-08-18,write,0.5", (short) 32); - expectedAggregateStats.put("2015-08-18,write,0.75", (short) 32); - expectedAggregateStats.put("2015-08-18,both,0.25", (short) 12); - expectedAggregateStats.put("2015-08-18,both,0.5", (short) 22); - expectedAggregateStats.put("2015-08-18,both,0.75", (short) 22); + expectedAggregateStats.put("2015-08-18,read,0.25", (short) 34); + expectedAggregateStats.put("2015-08-18,read,0.5", (short) 37); + expectedAggregateStats.put("2015-08-18,read,0.75", (short) 39); + expectedAggregateStats.put("2015-08-18,write,0.25", (short) 24); + expectedAggregateStats.put("2015-08-18,write,0.5", (short) 27); + expectedAggregateStats.put("2015-08-18,write,0.75", (short) 29); + expectedAggregateStats.put("2015-08-18,both,0.25", (short) 14); + expectedAggregateStats.put("2015-08-18,both,0.5", (short) 17); + expectedAggregateStats.put("2015-08-18,both,0.75", (short) 19); SortedSet<Main.RawStat> rawStats = new TreeSet<>(); rawStats.add(new Main.RawStat(DATE_A, FPR_A, (short) 32, (short) 22, (short) 12)); @@ -258,4 +260,55 @@ public class MainTest { (short) 22)); assertStatsCanBeAggregated(expectedAggregateStats, rawStats); } + + @Test + public void testComputePercentilesZeroValues() { + List<Short> valueList = new ArrayList<>(); + SortedMap<Double, Short> computedPercentiles = Main.computePercentiles( + valueList, 25.0, 50.0, 75.0); + assertEquals(0, computedPercentiles.get(25.0).shortValue()); + assertEquals(0, computedPercentiles.get(50.0).shortValue()); + assertEquals(0, computedPercentiles.get(75.0).shortValue()); + } + + @Test + public void testComputePercentilesTenValues() { + List<Short> valueList = new ArrayList<>(); + valueList.add((short) 3); + valueList.add((short) 6); + valueList.add((short) 7); + valueList.add((short) 8); + valueList.add((short) 8); + valueList.add((short) 10); + valueList.add((short) 13); + valueList.add((short) 15); + valueList.add((short) 16); + valueList.add((short) 20); + SortedMap<Double, Short> computedPercentiles = Main.computePercentiles( + valueList, 25.0, 50.0, 75.0); + assertEquals(7, computedPercentiles.get(25.0).shortValue()); + assertEquals(9, computedPercentiles.get(50.0).shortValue()); + assertEquals(14, computedPercentiles.get(75.0).shortValue()); + } + + @Test + public void testComputePercentilesElevenValues() { + List<Short> valueList = new ArrayList<>(); + valueList.add((short) 3); + valueList.add((short) 6); + valueList.add((short) 7); + valueList.add((short) 8); + valueList.add((short) 8); + valueList.add((short) 9); + valueList.add((short) 10); + valueList.add((short) 13); + valueList.add((short) 15); + valueList.add((short) 16); + valueList.add((short) 20); + SortedMap<Double, Short> computedPercentiles = Main.computePercentiles( + valueList, 25.0, 50.0, 75.0); + assertEquals(7, computedPercentiles.get(25.0).shortValue()); + assertEquals(9, computedPercentiles.get(50.0).shortValue()); + assertEquals(14, computedPercentiles.get(75.0).shortValue()); + } }

1 0

[metrics-web/release] Temporarily add "We Are Hiring!" link to navbar.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit f7f2f7f8ffcc02fccf8ceae66f1677e99b2746af Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Nov 5 21:32:01 2018 +0100 Temporarily add "We Are Hiring!" link to navbar. --- src/main/resources/web/jsps/top.jsp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/web/jsps/top.jsp b/src/main/resources/web/jsps/top.jsp index 440b2ee..96b3480 100644 --- a/src/main/resources/web/jsps/top.jsp +++ b/src/main/resources/web/jsps/top.jsp @@ -89,6 +89,7 @@ document.write('<div class="topButton" style="display:none;"><a href="#top"> <li class="visible-xs section-header">More</li> + <li><a href="https://www.torproject.org/about/jobs-metrics-data-architect.html.en"> We Are Hiring!</a></li> <li <c:if test="${'News'.equals(param.navActive)}"> class="active"</c:if>><a href="/news.html"> News</a></li> <li <c:if test="${'Sources'.equals(param.navActive)}"> class="active"</c:if>><a href="/sources.html"> Sources</a></li> <li <c:if test="${'Services'.equals(param.navActive)}"> class="active"</c:if>><a href="/services.html"> Services</a></li>

1 0

[metrics-web/release] Use ipv6servers' CSV files instead of servers.csv.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 28d42211d333d954a846f0885ebe1e4933bef366 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Nov 11 10:15:57 2018 +0100 Use ipv6servers' CSV files instead of servers.csv. --- build.xml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/build.xml b/build.xml index b498ea8..b95550d 100644 --- a/build.xml +++ b/build.xml @@ -369,14 +369,6 @@ <exec executable="psql" dir="${localmoddir}" failonerror="true" > - <arg value="-c COPY (SELECT * FROM stats_servers) TO STDOUT WITH CSV HEADER;" /> - <arg value="--dbname=tordir"/> - <arg value="--output=${statsdir}/servers.csv" /> - </exec> - - <exec executable="psql" - dir="${localmoddir}" - failonerror="true" > <arg value="-c COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;" /> <arg value="--dbname=tordir"/> <arg value="--output=${statsdir}/bandwidth.csv" /> @@ -515,7 +507,7 @@ <fileset dir="${modulebase}/hidserv/stats" includes="hidserv.csv" /> <fileset dir="${modulebase}/clients/stats" includes="clients*.csv userstats-combined.csv" /> - <fileset dir="${modulebase}/ipv6servers/stats" includes="ipv6servers.csv" /> + <fileset dir="${modulebase}/ipv6servers/stats" includes="*.csv" /> <fileset dir="${modulebase}/webstats/stats" includes="webstats.csv" /> <fileset dir="${modulebase}/totalcw/stats" includes="totalcw.csv" /> </copy>

1 0

[metrics-web/release] Handle days without any successful measurements, second attempt.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit e7c0c84754a070c09ebef09b4b748dab899f8250 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Oct 27 10:01:48 2018 +0200 Handle days without any successful measurements, second attempt. The earlier fix in 0408b73 was correct and necessary, but not sufficient. We'll also have to distinguish 0.0 and null on the Java side. Who would have expected that? Fixes #28136, but this time for real. --- .../org/torproject/metrics/stats/onionperf/Main.java | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 5bd55ab..c54ee6f 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -250,9 +250,9 @@ public class Main { rs.getInt("filesize"), emptyNull(rs.getString("source")), emptyNull(rs.getString("server")), - rs.getDouble("q1"), - rs.getDouble("md"), - rs.getDouble("q3"), + getDoubleFromResultSet(rs, "q1"), + getDoubleFromResultSet(rs, "md"), + getDoubleFromResultSet(rs, "q3"), rs.getInt("timeouts"), rs.getInt("failures"), rs.getInt("requests"))); @@ -315,6 +315,20 @@ public class Main { return null == text ? "" : text; } + /** Retrieves the <code>double</code> value of the designated column in the + * current row of the given <code>ResultSet</code> object as a + * <code>Double</code> object, or <code>null</code> if the retrieved value was + * <code>NULL</code>. */ + private static Double getDoubleFromResultSet(ResultSet rs, String columnLabel) + throws SQLException { + double result = rs.getDouble(columnLabel); + if (rs.wasNull()) { + return null; + } else { + return result; + } + } + static void writeStatistics(Path webstatsPath, List<String> statistics) throws IOException { webstatsPath.toFile().getParentFile().mkdirs();

1 0

[metrics-web/release] Extend Tor Browser graphs to include update pings.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit af06d70a389b4f5748dbdeb87c75855fbabbc220 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Oct 11 12:26:47 2018 +0200 Extend Tor Browser graphs to include update pings. Implements the second half of #27931 by making the actual code changes. --- src/main/R/rserver/graphs.R | 42 ++++++++++++++-------- src/main/resources/web/json/metrics.json | 8 ++--- .../resources/web/jsps/reproducible-metrics.jsp | 4 +-- src/main/resources/web/jsps/stats.jsp | 17 ++++----- 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index 60d905f..b021791 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -1331,14 +1331,17 @@ prepare_webstats_tb_platform <- function(start_p, end_p) { colClasses = c("log_date" = "Date")) %>% filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type == "tbid") %>% - group_by(log_date, platform) %>% + filter(request_type %in% c("tbid", "tbup")) %>% + group_by(log_date, platform, request_type) %>% summarize(count = sum(count)) } plot_webstats_tb_platform <- function(start_p, end_p, path_p) { - prepare_webstats_tb_platform(start_p, end_p) %>% - ggplot(aes(x = log_date, y = count, colour = platform)) + + d <- prepare_webstats_tb_platform(start_p, end_p) + levels(d$request_type) <- list( + "Initial downloads" = "tbid", + "Update pings" = "tbup") + ggplot(d, aes(x = log_date, y = count, colour = platform)) + geom_point() + geom_line() + scale_x_date(name = "", breaks = custom_breaks, @@ -1347,9 +1350,11 @@ plot_webstats_tb_platform <- function(start_p, end_p, path_p) { scale_colour_hue(name = "Platform", breaks = c("w", "m", "l", "o", ""), labels = c("Windows", "macOS", "Linux", "Other", "Unknown")) + + facet_grid(request_type ~ ., scales = "free_y") + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA)) + - ggtitle("Tor Browser downloads by platform") + + strip.background = element_rect(fill = NA), + legend.position = "top") + + ggtitle("Tor Browser downloads and updates by platform") + labs(caption = copyright_notice) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } @@ -1357,8 +1362,8 @@ plot_webstats_tb_platform <- function(start_p, end_p, path_p) { write_webstats_tb_platform <- function(start_p = NULL, end_p = NULL, path_p) { prepare_webstats_tb_platform(start_p, end_p) %>% rename(date = log_date) %>% - spread(platform, count) %>% - rename(linux = l, macos = m, windows = w) %>% + spread(request_type, count, fill = 0) %>% + rename(initial_downloads = tbid, update_pings = tbup) %>% write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") } @@ -1366,12 +1371,16 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) { d <- read.csv(paste(stats_dir, "webstats.csv", sep = ""), colClasses = c("log_date" = "Date", "locale" = "character")) d <- d[d$log_date >= start_p & d$log_date <= end_p & - d$request_type == "tbid", ] + d$request_type %in% c("tbid", "tbup"), ] + levels(d$request_type) <- list( + "Initial downloads" = "tbid", + "Update pings" = "tbup") e <- d e <- aggregate(list(count = e$count), by = list(locale = e$locale), FUN = sum) e <- e[order(e$count, decreasing = TRUE), ] e <- e[1:5, ] d <- aggregate(list(count = d$count), by = list(log_date = d$log_date, + request_type = d$request_type, locale = ifelse(d$locale %in% e$locale, d$locale, "(other)")), FUN = sum) ggplot(d, aes(x = log_date, y = count, colour = locale)) + geom_point() + @@ -1382,9 +1391,11 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) { scale_colour_hue(name = "Locale", breaks = c(e$locale, "(other)"), labels = c(e$locale, "Other")) + + facet_grid(request_type ~ ., scales = "free_y") + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), - strip.background = element_rect(fill = NA)) + - ggtitle("Tor Browser downloads by locale") + + strip.background = element_rect(fill = NA), + legend.position = "top") + + ggtitle("Tor Browser downloads and updates by locale") + labs(caption = copyright_notice) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } @@ -1399,10 +1410,13 @@ write_webstats_tb_locale <- function(start_p = NULL, end_p = NULL, path_p) { colClasses = c("log_date" = "Date", "locale" = "character")) %>% filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% - filter(request_type == "tbid") %>% - group_by(log_date, locale) %>% - summarize(initial_downloads = sum(count)) %>% + filter(request_type %in% c("tbid", "tbup")) %>% rename(date = log_date) %>% + group_by(date, locale, request_type) %>% + summarize(count = sum(count)) %>% + mutate(request_type = factor(request_type, levels = c("tbid", "tbup"))) %>% + spread(request_type, count, fill = 0) %>% + rename(initial_downloads = tbid, update_pings = tbup) %>% write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") } diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index ed74959..502b0e6 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -397,9 +397,9 @@ }, { "id": "webstats-tb-platform", - "title": "Tor Browser downloads by platform", + "title": "Tor Browser downloads and updates by platform", "type": "Graph", - "description": "This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable, broken down by platform (Windows, macOS, Linux) of the requested executable. Note that this graph does not show the platform used to download Tor Browser but the platform that it was downloaded for.", + "description": "This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable and requests made by Tor Browser to check for an update, broken down by platform (Windows, macOS, Linux). Note that, at least for initial downloads, this graph does not show the platform used to download Tor Browser but the platform that it was downloaded for.", "function": "webstats_tb_platform", "parameters": [ "start", @@ -408,9 +408,9 @@ }, { "id": "webstats-tb-locale", - "title": "Tor Browser downloads by locale", + "title": "Tor Browser downloads and updates by locale", "type": "Graph", - "description": "This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable, broken down by requested locale.", + "description": "This graph shows absolute numbers of requests to Tor's web servers to download a Tor Browser executable and requests made by Tor Browser to check for an update, broken down by locale.", "function": "webstats_tb_locale", "parameters": [ "start", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 4bed5cc..939b42e 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -809,8 +809,8 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <ul> <li>Tor Browser downloads and updates <a href="/webstats-tb.html" class="btn btn-primary btn-xs"> graph</a></li> -<li>Tor Browser downloads by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"> graph</a></li> -<li>Tor Browser downloads by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"> graph</a></li> +<li>Tor Browser downloads and updates by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"> graph</a></li> +<li>Tor Browser downloads and updates by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"> graph</a></li> <li>Tor Messenger downloads and updates <a href="/webstats-tm.html" class="btn btn-primary btn-xs"> graph</a></li> </ul> diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index c6da19f..3482320 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -47,7 +47,7 @@ https://metrics.torproject.org/identifier.csv <li>July 31, 2018: Announced pending changes to per-graph CSV files to become effective on August 15 and pre-aggregated CSV files to be removed by September 15.</li> <li>August 15, 2018: Made the first batch of changes to per-graph CSV files.</li> <li>September 15, 2018: Removed all pre-aggregated CSV files.</li> -<li>October 28, 2018: Add and/or remove columns to <a href="#webstats-tb-platform">Tor Browser downloads by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads by locale</a> graphs.</li> +<li>October 28, 2018: Added and/or removed columns to <a href="#webstats-tb-platform">Tor Browser downloads and updates by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads and updates by locale</a> graphs.</li> </ul> </div> @@ -685,7 +685,7 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <li>update_requests: Number of Tor Browser update requests: GET requests to all sites with resource strings <code>'%/torbrowser/%.mar'</code> and response code 302.</li> </ul> -<h3>Tor Browser downloads by platform +<h3>Tor Browser downloads and updates by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"> graph</a> <a href="/webstats-tb-platform.csv" class="btn btn-primary btn-xs"> data</a> <a href="#webstats-tb-platform" name="webstats-tb-platform" class="anchor">#</a></h3> @@ -701,15 +701,12 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <ul> <li>date: UTC date (YYYY-MM-DD) when requests to <code>torproject.org</code> web servers have been logged.</li> -<li>linux: Number of Tor Browser initial downloads for Linux. This column is going to be removed after October 28, 2018.</li> -<li>macos: Number of Tor Browser initial downloads for macOS. This column is going to be removed after October 28, 2018.</li> -<li>windows: Number of Tor Browser initial downloads for Windows. This column is going to be removed after October 28, 2018.</li> -<li>platform: Platform, like "Linux", "macOS", or "Windows". This column is going to be added after October 28, 2018.</li> -<li>initial_downloads: Number of Tor Browser initial downloads. This column is going to be added after October 28, 2018.</li> -<li>update_pings: Number of Tor Browser update pings. This column is going to be added after October 28, 2018.</li> +<li>platform: Platform, like "Linux", "macOS", or "Windows".</li> +<li>initial_downloads: Number of Tor Browser initial downloads.</li> +<li>update_pings: Number of Tor Browser update pings.</li> </ul> -<h3>Tor Browser downloads by locale +<h3>Tor Browser downloads and updates by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"> graph</a> <a href="/webstats-tb-locale.csv" class="btn btn-primary btn-xs"> data</a> <a href="#webstats-tb-locale" name="webstats-tb-locale" class="anchor">#</a></h3> @@ -727,7 +724,7 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <li>date: UTC date (YYYY-MM-DD) when requests to <code>torproject.org</code> web servers have been logged.</li> <li>locale: Locale, like "en-US" for English (United States), "de" for German, etc., and "??" for unrecognized locales.</li> <li>initial_downloads: Number of Tor Browser initial downloads.</li> -<li>update_pings: Number of Tor Browser update pings. This column is going to be added after October 28, 2018.</li> +<li>update_pings: Number of Tor Browser update pings.</li> </ul> <h3>Tor Messenger downloads and updates

1 0

[metrics-web/release] Also update module name for init-ipv6servers.sql.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 1ce7b61353ff43e79e12e831bc1c5c3267f0c1b5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Nov 22 16:03:36 2018 +0100 Also update module name for init-ipv6servers.sql. Part of #28116. --- src/main/sql/{ipv6servers => servers}/init-ipv6servers.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/main/sql/ipv6servers/init-ipv6servers.sql b/src/main/sql/servers/init-ipv6servers.sql similarity index 100% rename from src/main/sql/ipv6servers/init-ipv6servers.sql rename to src/main/sql/servers/init-ipv6servers.sql

1 0

[metrics-web/release] Fix ipv6servers unit tests.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 88535e1f37ef47c2db61b2e6e0f0b0ab633d0a99 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Nov 14 17:05:18 2018 +0100 Fix ipv6servers unit tests. --- .../metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java b/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java index aaadcbf..2f3ca42 100644 --- a/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java +++ b/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java @@ -122,8 +122,11 @@ public class Ipv6NetworkStatusTest { for (Ipv6NetworkStatus.Entry parsedEntry : parsedNetworkStatus.entries) { if (this.digest.equals(parsedEntry.digest)) { - assertEquals(this.description, this.guard, parsedEntry.guard); - assertEquals(this.description, this.exit, parsedEntry.exit); + assertEquals(this.description, this.guard, + parsedEntry.flags.contains("Guard")); + assertEquals(this.description, this.exit, + parsedEntry.flags.contains("Exit") + && !parsedEntry.flags.contains("BadExit")); assertEquals(this.description, this.reachable, parsedEntry.reachable); foundEntry = true;

1 0

[metrics-web/release] Include client numbers with fractions up to 110%.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 3b5a2fb6c87a98508ab1633792f5c5176f6ffc31 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Nov 30 10:22:35 2018 +0100 Include client numbers with fractions up to 110%. Turns out that almost all relays report directory-request statistics these days, including a small number of relays that temporarily drop out of the consensus. We're now accepting up to 10% of those additional statistics. See #28305 for more details. --- src/main/sql/clients/init-userstats.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/sql/clients/init-userstats.sql b/src/main/sql/clients/init-userstats.sql index 38521f2..cf2b620 100644 --- a/src/main/sql/clients/init-userstats.sql +++ b/src/main/sql/clients/init-userstats.sql @@ -678,8 +678,13 @@ CREATE OR REPLACE VIEW estimated AS SELECT FROM aggregated WHERE hh * nn > 0.0) a -- Only include estimates with at least 10% of nodes reporting directory - -- request statistics. - WHERE a.frac BETWEEN 0.1 AND 1.0 + -- request statistics, and exclude estimates with fractions higher than 110%. + -- The upper bound is 110% and not 100%, because there can be relays reporting + -- statistics that temporarily didn't make it into the consensus, and we + -- accept up to 10% of those additional statistics. However, there needs to be + -- some upper bound to exclude obvious outliers with fractions of 120%, 150%, + -- or even 200%. See #28305 for more details. + WHERE a.frac BETWEEN 0.1 AND 1.1 -- Skip estimates that are as recent as yesterday or newer. AND a.date < current_date - 1

1 0