November 2019 - tor-commits - lists.torproject.org

[metrics-web/release] Rename ipv6servers module to servers.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit c95125a2b329c801db84eea2bdbc4e5118bde9b2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Nov 14 10:48:56 2018 +0100 Rename ipv6servers module to servers. Part of #28116. --- build.xml | 12 ++++++------ .../metrics/stats/ipv6servers/Configuration.java | 18 ------------------ .../metrics/stats/servers/Configuration.java | 18 ++++++++++++++++++ .../stats/{ipv6servers => servers}/Database.java | 2 +- .../{ipv6servers => servers}/Ipv6NetworkStatus.java | 2 +- .../{ipv6servers => servers}/Ipv6ServerDescriptor.java | 2 +- .../metrics/stats/{ipv6servers => servers}/Main.java | 8 ++++---- .../metrics/stats/{ipv6servers => servers}/Parser.java | 2 +- .../metrics/stats/{ipv6servers => servers}/Writer.java | 2 +- .../Ipv6NetworkStatusTest.java | 6 +++--- .../Ipv6ServerDescriptorTest.java | 14 +++++++------- .../000a7fe20a17bf5d9839a126b1dff43f998aac6f | 0 .../0018ab4f2f28af683d52f06407edbf7ce1bd3b7d | 0 .../0041dbf9fe846f9765882f7dc8332f94b709e35a | 0 .../01003df74972ce952ebfa390f468ef63c50efa25 | 0 .../018c1229d5f56eebfc1d709d4692673d098800e8 | 0 .../2017-12-04-20-00-00-consensus.part | 0 ...90507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part | 0 .../64dd486d89af14027c9a7b4347a94b74dddb5cdb | 0 .../sql/{ipv6servers => servers}/test-ipv6servers.sql | 0 20 files changed, 43 insertions(+), 43 deletions(-) diff --git a/build.xml b/build.xml index a391416..89c8b31 100644 --- a/build.xml +++ b/build.xml @@ -105,7 +105,7 @@ depends="init" description="Run all available database pgTAP tests." > <antcall target="test-db"> - <param name="db2test" value="ipv6servers" /> + <param name="db2test" value="servers" /> </antcall> <antcall target="test-db"> <param name="db2test" value="userstats" /> @@ -319,7 +319,7 @@ <antcall target="advbwdist" /> <antcall target="hidserv" /> <antcall target="clients" /> - <antcall target="ipv6servers" /> + <antcall target="servers" /> <antcall target="webstats" /> <antcall target="totalcw" /> <antcall target="make-data-available" /> @@ -416,11 +416,11 @@ </antcall> </target> - <target name="ipv6servers" > - <property name="module.name" value="ipv6servers" /> + <target name="servers" > + <property name="module.name" value="servers" /> <antcall target="run-java" > <param name="module.main" - value="org.torproject.metrics.stats.ipv6servers.Main" /> + value="org.torproject.metrics.stats.servers.Main" /> </antcall> </target> @@ -477,7 +477,7 @@ <fileset dir="${modulebase}/hidserv/stats" includes="hidserv.csv" /> <fileset dir="${modulebase}/clients/stats" includes="clients*.csv userstats-combined.csv" /> - <fileset dir="${modulebase}/ipv6servers/stats" includes="*.csv" /> + <fileset dir="${modulebase}/servers/stats" includes="*.csv" /> <fileset dir="${modulebase}/webstats/stats" includes="webstats.csv" /> <fileset dir="${modulebase}/totalcw/stats" includes="totalcw.csv" /> </copy> diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java deleted file mode 100644 index d849cb6..0000000 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.ipv6servers; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -class Configuration { - static String descriptors = System.getProperty("ipv6servers.descriptors", - "../../shared/in/"); - static String database = System.getProperty("ipv6servers.database", - "jdbc:postgresql:ipv6servers"); - static String history = System.getProperty("ipv6servers.history", - "status/read-descriptors"); - static String output = System.getProperty("ipv6servers.output", - "stats/"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java new file mode 100644 index 0000000..1b7c217 --- /dev/null +++ b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java @@ -0,0 +1,18 @@ +/* Copyright 2017--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.servers; + +/** Configuration options parsed from Java properties with reasonable hard-coded + * defaults. */ +class Configuration { + static String descriptors = System.getProperty("servers.descriptors", + "../../shared/in/"); + static String database = System.getProperty("servers.database", + "jdbc:postgresql:ipv6servers"); + static String history = System.getProperty("servers.history", + "status/read-descriptors"); + static String output = System.getProperty("servers.output", + "stats/"); +} + diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java b/src/main/java/org/torproject/metrics/stats/servers/Database.java similarity index 99% rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java rename to src/main/java/org/torproject/metrics/stats/servers/Database.java index b5efe3e..9c9bda3 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Database.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import java.sql.Connection; import java.sql.DriverManager; diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java b/src/main/java/org/torproject/metrics/stats/servers/Ipv6NetworkStatus.java similarity index 98% rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java rename to src/main/java/org/torproject/metrics/stats/servers/Ipv6NetworkStatus.java index 526adc7..2f40854 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Ipv6NetworkStatus.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import java.time.LocalDateTime; import java.util.ArrayList; diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java b/src/main/java/org/torproject/metrics/stats/servers/Ipv6ServerDescriptor.java similarity index 95% rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java rename to src/main/java/org/torproject/metrics/stats/servers/Ipv6ServerDescriptor.java index 387024b..4450a3c 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Ipv6ServerDescriptor.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; /** Data object holding all relevant parts parsed from a (relay or bridge) * server descriptor. */ diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java similarity index 95% rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java rename to src/main/java/org/torproject/metrics/stats/servers/Main.java index d322a2e..54f44d0 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.Descriptor; @@ -18,7 +18,7 @@ import java.nio.file.Paths; import java.sql.SQLException; import java.util.Arrays; -/** Main class of the ipv6servers module that imports relevant parts from server +/** Main class of the servers module that imports relevant parts from server * descriptors and network statuses into a database, and exports aggregate * statistics to CSV files. */ public class Main { @@ -38,7 +38,7 @@ public class Main { /** Run the module. */ public static void main(String[] args) throws Exception { - log.info("Starting ipv6servers module."); + log.info("Starting servers module."); log.info("Reading descriptors and inserting relevant parts into the " + "database."); @@ -99,7 +99,7 @@ public class Main { new Writer().write(Paths.get(Configuration.output, "platforms.csv"), database.queryPlatforms()); - log.info("Terminating ipv6servers module."); + log.info("Terminating servers module."); } catch (SQLException sqle) { log.error("Cannot recover from SQL exception while querying. Not writing " + "output file.", sqle); diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java b/src/main/java/org/torproject/metrics/stats/servers/Parser.java similarity index 99% rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java rename to src/main/java/org/torproject/metrics/stats/servers/Parser.java index 9d8b71a..65055f9 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Parser.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.NetworkStatusEntry; diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java b/src/main/java/org/torproject/metrics/stats/servers/Writer.java similarity index 96% rename from src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java rename to src/main/java/org/torproject/metrics/stats/servers/Writer.java index 4e2893c..5ff1858 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Writer.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import java.io.File; import java.io.IOException; diff --git a/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java b/src/test/java/org/torproject/metrics/stats/servers/Ipv6NetworkStatusTest.java similarity index 96% rename from src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java rename to src/test/java/org/torproject/metrics/stats/servers/Ipv6NetworkStatusTest.java index 2f3ca42..4ad0b9c 100644 --- a/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatusTest.java +++ b/src/test/java/org/torproject/metrics/stats/servers/Ipv6NetworkStatusTest.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.fail; @@ -33,8 +33,8 @@ public class Ipv6NetworkStatusTest { /** Provide test data. */ @Parameters public static Collection<Object[]> data() { - String relayFileName = "ipv6servers/2017-12-04-20-00-00-consensus.part"; - String bridgeFileName = "ipv6servers/" + String relayFileName = "servers/2017-12-04-20-00-00-consensus.part"; + String bridgeFileName = "servers/" + "20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part"; return Arrays.asList(new Object[][] { { "Relay status without Guard or Exit flag and without IPv6 address. ", diff --git a/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptorTest.java b/src/test/java/org/torproject/metrics/stats/servers/Ipv6ServerDescriptorTest.java similarity index 86% rename from src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptorTest.java rename to src/test/java/org/torproject/metrics/stats/servers/Ipv6ServerDescriptorTest.java index 7b63c1e..54a6941 100644 --- a/src/test/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptorTest.java +++ b/src/test/java/org/torproject/metrics/stats/servers/Ipv6ServerDescriptorTest.java @@ -1,7 +1,7 @@ /* Copyright 2017--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.ipv6servers; +package org.torproject.metrics.stats.servers; import static junit.framework.TestCase.assertEquals; import static org.junit.Assert.assertNotNull; @@ -32,22 +32,22 @@ public class Ipv6ServerDescriptorTest { public static Collection<Object[]> data() { return Arrays.asList(new Object[][] { { "Relay server descriptor without or-address or ipv6-policy line.", - "ipv6servers/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d", + "servers/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d", 819200, false, false }, { "Relay server descriptor with or-address and ipv6-policy line.", - "ipv6servers/01003df74972ce952ebfa390f468ef63c50efa25", + "servers/01003df74972ce952ebfa390f468ef63c50efa25", 6576128, true, true }, { "Relay server descriptor with or-address line only.", - "ipv6servers/018c1229d5f56eebfc1d709d4692673d098800e8", + "servers/018c1229d5f56eebfc1d709d4692673d098800e8", 0, true, false }, { "Bridge server descriptor without or-address or ipv6-policy line.", - "ipv6servers/000a7fe20a17bf5d9839a126b1dff43f998aac6f", + "servers/000a7fe20a17bf5d9839a126b1dff43f998aac6f", 0, false, false }, { "Bridge server descriptor with or-address line.", - "ipv6servers/0041dbf9fe846f9765882f7dc8332f94b709e35a", + "servers/0041dbf9fe846f9765882f7dc8332f94b709e35a", 0, true, false }, { "Bridge server descriptor with (ignored) ipv6-policy accept line.", - "ipv6servers/64dd486d89af14027c9a7b4347a94b74dddb5cdb", + "servers/64dd486d89af14027c9a7b4347a94b74dddb5cdb", 0, false, false } }); } diff --git a/src/test/resources/ipv6servers/000a7fe20a17bf5d9839a126b1dff43f998aac6f b/src/test/resources/servers/000a7fe20a17bf5d9839a126b1dff43f998aac6f similarity index 100% rename from src/test/resources/ipv6servers/000a7fe20a17bf5d9839a126b1dff43f998aac6f rename to src/test/resources/servers/000a7fe20a17bf5d9839a126b1dff43f998aac6f diff --git a/src/test/resources/ipv6servers/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d b/src/test/resources/servers/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d similarity index 100% rename from src/test/resources/ipv6servers/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d rename to src/test/resources/servers/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d diff --git a/src/test/resources/ipv6servers/0041dbf9fe846f9765882f7dc8332f94b709e35a b/src/test/resources/servers/0041dbf9fe846f9765882f7dc8332f94b709e35a similarity index 100% rename from src/test/resources/ipv6servers/0041dbf9fe846f9765882f7dc8332f94b709e35a rename to src/test/resources/servers/0041dbf9fe846f9765882f7dc8332f94b709e35a diff --git a/src/test/resources/ipv6servers/01003df74972ce952ebfa390f468ef63c50efa25 b/src/test/resources/servers/01003df74972ce952ebfa390f468ef63c50efa25 similarity index 100% rename from src/test/resources/ipv6servers/01003df74972ce952ebfa390f468ef63c50efa25 rename to src/test/resources/servers/01003df74972ce952ebfa390f468ef63c50efa25 diff --git a/src/test/resources/ipv6servers/018c1229d5f56eebfc1d709d4692673d098800e8 b/src/test/resources/servers/018c1229d5f56eebfc1d709d4692673d098800e8 similarity index 100% rename from src/test/resources/ipv6servers/018c1229d5f56eebfc1d709d4692673d098800e8 rename to src/test/resources/servers/018c1229d5f56eebfc1d709d4692673d098800e8 diff --git a/src/test/resources/ipv6servers/2017-12-04-20-00-00-consensus.part b/src/test/resources/servers/2017-12-04-20-00-00-consensus.part similarity index 100% rename from src/test/resources/ipv6servers/2017-12-04-20-00-00-consensus.part rename to src/test/resources/servers/2017-12-04-20-00-00-consensus.part diff --git a/src/test/resources/ipv6servers/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part b/src/test/resources/servers/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part similarity index 100% rename from src/test/resources/ipv6servers/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part rename to src/test/resources/servers/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part diff --git a/src/test/resources/ipv6servers/64dd486d89af14027c9a7b4347a94b74dddb5cdb b/src/test/resources/servers/64dd486d89af14027c9a7b4347a94b74dddb5cdb similarity index 100% rename from src/test/resources/ipv6servers/64dd486d89af14027c9a7b4347a94b74dddb5cdb rename to src/test/resources/servers/64dd486d89af14027c9a7b4347a94b74dddb5cdb diff --git a/src/test/sql/ipv6servers/test-ipv6servers.sql b/src/test/sql/servers/test-ipv6servers.sql similarity index 100% rename from src/test/sql/ipv6servers/test-ipv6servers.sql rename to src/test/sql/servers/test-ipv6servers.sql

1 0

[metrics-web/release] Remove unused parts of totalcw module.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit e94ceeb9b6763374f6edb0105ae6da9fd5098d99 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Nov 29 08:07:11 2018 +0100 Remove unused parts of totalcw module. Requires updating the vote table of the database. Part of #28137, #28328, and #28352. --- .../torproject/metrics/stats/totalcw/Database.java | 15 ++------ .../torproject/metrics/stats/totalcw/Parser.java | 36 +++++-------------- .../totalcw/TotalcwRelayNetworkStatusVote.java | 25 ------------- src/main/sql/totalcw/init-totalcw.sql | 23 ------------ .../totalcw/TotalcwRelayNetworkStatusVoteTest.java | 42 +++------------------- 5 files changed, 17 insertions(+), 124 deletions(-) diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java index be4cad3..66b0366 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java @@ -66,10 +66,8 @@ class Database implements AutoCloseable { "SELECT EXISTS (SELECT 1 FROM vote " + "WHERE valid_after = ? AND authority_id = ?)"); this.psVoteInsert = this.connection.prepareStatement( - "INSERT INTO vote (valid_after, authority_id, measured_count, " - + "measured_sum, measured_mean, measured_min, measured_q1, " - + "measured_median, measured_q3, measured_max) " - + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO vote (valid_after, authority_id, measured_sum) " + + "VALUES (?, ?, ?)", Statement.RETURN_GENERATED_KEYS); } @@ -124,14 +122,7 @@ class Database implements AutoCloseable { Timestamp.from(ZonedDateTime.of(vote.validAfter, ZoneId.of("UTC")).toInstant()), calendar); this.psVoteInsert.setInt(2, authorityId); - this.psVoteInsert.setLong(3, vote.measuredCount); - this.psVoteInsert.setLong(4, vote.measuredSum); - this.psVoteInsert.setLong(5, vote.measuredMean); - this.psVoteInsert.setLong(6, vote.measuredMin); - this.psVoteInsert.setLong(7, vote.measuredQ1); - this.psVoteInsert.setLong(8, vote.measuredMedian); - this.psVoteInsert.setLong(9, vote.measuredQ3); - this.psVoteInsert.setLong(10, vote.measuredMax); + this.psVoteInsert.setLong(3, vote.measuredSum); this.psVoteInsert.execute(); try (ResultSet rs = this.psVoteInsert.getGeneratedKeys()) { if (rs.next()) { diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java index 4367200..893184c 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java @@ -6,13 +6,8 @@ package org.torproject.metrics.stats.totalcw; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusVote; -import org.apache.commons.math3.stat.descriptive.rank.Percentile; - import java.time.Instant; import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; /** Parser that extracts bandwidth measurement statistics from votes and creates * data objects for them. */ @@ -22,13 +17,17 @@ class Parser { * contain any bandwidth measurements. */ TotalcwRelayNetworkStatusVote parseRelayNetworkStatusVote( RelayNetworkStatusVote vote) { - List<Long> measuredBandwidths = new ArrayList<>(); + Long measuredSum = null; for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { - if (entry.getMeasured() >= 0L) { - measuredBandwidths.add(entry.getMeasured()); + if (entry.getMeasured() < 0L) { + continue; + } + if (null == measuredSum) { + measuredSum = 0L; } + measuredSum += entry.getMeasured(); } - if (measuredBandwidths.isEmpty()) { + if (null == measuredSum) { /* Return null, because we wouldn't want to add this vote to the database * anyway. */ return null; @@ -39,24 +38,7 @@ class Parser { .atZone(ZoneId.of("UTC")).toLocalDateTime(); parsedVote.identityHex = vote.getIdentity(); parsedVote.nickname = vote.getNickname(); - Collections.sort(measuredBandwidths); - long totalValue = 0L; - double[] values = new double[measuredBandwidths.size()]; - for (int i = 0; i < measuredBandwidths.size(); i++) { - values[i] = (double) measuredBandwidths.get(i); - totalValue += measuredBandwidths.get(i); - } - parsedVote.measuredCount = values.length; - parsedVote.measuredSum = totalValue; - parsedVote.measuredMean = totalValue / values.length; - parsedVote.measuredMin = (long) Math.floor(values[0]); - parsedVote.measuredMax = (long) Math.floor(values[values.length - 1]); - Percentile percentile = new Percentile().withEstimationType( - Percentile.EstimationType.R_7); - percentile.setData(values); - parsedVote.measuredQ1 = (long) Math.floor(percentile.evaluate(25.0)); - parsedVote.measuredMedian = (long) Math.floor(percentile.evaluate(50.0)); - parsedVote.measuredQ3 = (long) Math.floor(percentile.evaluate(75.0)); + parsedVote.measuredSum = measuredSum; return parsedVote; } } diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java index c139cdc..ff56d91 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java @@ -19,32 +19,7 @@ class TotalcwRelayNetworkStatusVote { * key. */ String identityHex; - /** Count of status entries containing bandwidth measurements. */ - long measuredCount; - /** Sum of bandwidth measurements of all contained status entries. */ long measuredSum; - - /** Mean value of bandwidth measurements of all contained status entries. */ - long measuredMean; - - /** Minimum value of bandwidth measurements of all contained status - * entries. */ - long measuredMin; - - /** First quartile value of bandwidth measurements of all contained status - * entries. */ - long measuredQ1; - - /** Median value of bandwidth measurements of all contained status entries. */ - long measuredMedian; - - /** Third quartile value of bandwidth measurements of all contained status - * entries. */ - long measuredQ3; - - /** Maximum value of bandwidth measurements of all contained status - * entries. */ - long measuredMax; } diff --git a/src/main/sql/totalcw/init-totalcw.sql b/src/main/sql/totalcw/init-totalcw.sql index bbb6cac..d723adb 100644 --- a/src/main/sql/totalcw/init-totalcw.sql +++ b/src/main/sql/totalcw/init-totalcw.sql @@ -31,32 +31,9 @@ CREATE TABLE vote ( -- Numeric identifier uniquely identifying the authority generating this vote. authority_id INTEGER REFERENCES authority (authority_id), - -- Count of status entries containing bandwidth measurements. - measured_count BIGINT NOT NULL, - -- Sum of bandwidth measurements of all contained status entries. measured_sum BIGINT NOT NULL, - -- Mean value of bandwidth measurements of all contained status entries. - measured_mean BIGINT NOT NULL, - - -- Minimum value of bandwidth measurements of all contained status entries. - measured_min BIGINT NOT NULL, - - -- First quartile value of bandwidth measurements of all contained status - -- entries. - measured_q1 BIGINT NOT NULL, - - -- Median value of bandwidth measurements of all contained status entries. - measured_median BIGINT NOT NULL, - - -- Third quartile value of bandwidth measurements of all contained status - -- entries. - measured_q3 BIGINT NOT NULL, - - -- Maximum value of bandwidth measurements of all contained status entries. - measured_max BIGINT NOT NULL, - UNIQUE (valid_after, authority_id) ); diff --git a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java index dc9df7f..11f931d 100644 --- a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java +++ b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java @@ -35,23 +35,19 @@ public class TotalcwRelayNetworkStatusVoteTest { { "2018-10-15-00-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-" + "55A38ED50848BE1F13C6A35C3CA637B0D962C2EF.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L }, + "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", -1L }, { "2018-10-15-00-00-00-vote-27102BC123E7AF1D4741AE047E160C91ADC76B21-" + "049AB3179B12DACC391F06A10C2A8904E4339D33.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", - 20L, 138803L, 6940L, 5L, 76L, 2490L, 9732L, 34800L }, + "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", 138803L }, { "2018-10-15-00-00-00-vote-ED03BB616EB2F60BEC80151114BB25CEF515B226-" + "2669AD153408F88E416CE6206D1A75EC3324A2F4.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", - 19, 133441L, 7023L, 2L, 153L, 3920L, 11030L, 31600L }, + "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", 133441L }, { "2018-10-15-00-00-00-vote-EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97-" + "38C6A19F78948B689345EE41D7119D76246C4D3E.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", - 20, 158534L, 7926L, 6L, 109L, 3215L, 9582L, 40700L } + "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", 158534L } }); } @@ -68,29 +64,8 @@ public class TotalcwRelayNetworkStatusVoteTest { public String expectedIdentityHex; @Parameter(4) - public long expectedMeasuredCount; - - @Parameter(5) public long expectedMeasuredSum; - @Parameter(6) - public long expectedMeasuredMean; - - @Parameter(7) - public long expectedMeasuredMin; - - @Parameter(8) - public long expectedMeasuredQ1; - - @Parameter(9) - public long expectedMeasuredMedian; - - @Parameter(10) - public long expectedMeasuredQ3; - - @Parameter(11) - public long expectedMeasuredMax; - @Test public void testParseVote() throws Exception { InputStream is = getClass().getClassLoader().getResourceAsStream( @@ -107,20 +82,13 @@ public class TotalcwRelayNetworkStatusVoteTest { sb.toString().getBytes(), new File(this.fileName), this.fileName)) { TotalcwRelayNetworkStatusVote parsedVote = new Parser() .parseRelayNetworkStatusVote((RelayNetworkStatusVote) descriptor); - if (0L == expectedMeasuredCount) { + if (this.expectedMeasuredSum < 0L) { assertNull(parsedVote); } else { assertEquals(this.expectedValidAfter, parsedVote.validAfter); assertEquals(this.expectedNickname, parsedVote.nickname); assertEquals(this.expectedIdentityHex, parsedVote.identityHex); - assertEquals(this.expectedMeasuredCount, parsedVote.measuredCount); assertEquals(this.expectedMeasuredSum, parsedVote.measuredSum); - assertEquals(this.expectedMeasuredMean, parsedVote.measuredMean); - assertEquals(this.expectedMeasuredMin, parsedVote.measuredMin); - assertEquals(this.expectedMeasuredQ1, parsedVote.measuredQ1); - assertEquals(this.expectedMeasuredMedian, parsedVote.measuredMedian); - assertEquals(this.expectedMeasuredQ3, parsedVote.measuredQ3); - assertEquals(this.expectedMeasuredMax, parsedVote.measuredMax); } } }

1 0

[metrics-web/release] Schedule changes related to #28353.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit ab9acb2f7b92fa70b33c0f9c6980a604a84e346d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 5 21:19:10 2018 +0100 Schedule changes related to #28353. In roughly two weeks from now we're going to update one bandwidth graph and remove two obsolete ones. Similar to commit b14dbc3, announce this change now, so that any folks using our CSV files get the chance to update their tools. --- src/main/resources/web/json/metrics.json | 8 ++++---- src/main/resources/web/jsps/stats.jsp | 25 ++++++++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 527f13f..9cb50ad 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -68,9 +68,9 @@ }, { "id": "bandwidth", - "title": "Total relay bandwidth", + "title": "Total relay bandwidth (deprecated)", "type": "Graph", - "description": "<p>This graph shows the total <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of all <a href=\"glossary.html#relay\">relays</a> in the network.</p>", + "description": "<p>This graph shows the total <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of all <a href=\"glossary.html#relay\">relays</a> in the network. <strong>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href=\"/bandwidth-flags.html\">Advertised and consumed bandwidth by relay flags</a> graph.</strong></p>", "function": "bandwidth", "parameters": [ "start", @@ -79,9 +79,9 @@ }, { "id": "bwhist-flags", - "title": "Consumed bandwidth by Exit/Guard flag combination", + "title": "Consumed bandwidth by Exit/Guard flag combination (deprecated)", "type": "Graph", - "description": "<p>This graph shows the <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> reported by relays, subdivided into four distinct subsets by assigned \"Exit\" and/or \"Guard\" <a href=\"glossary.html#relay-flag\">flags</a>.</p>", + "description": "<p>This graph shows the <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> reported by relays, subdivided into four distinct subsets by assigned \"Exit\" and/or \"Guard\" <a href=\"glossary.html#relay-flag\">flags</a>. <strong>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href=\"/bandwidth-flags.html\">Advertised and consumed bandwidth by relay flags</a> graph.</strong></p>", "function": "bwhist_flags", "parameters": [ "start", diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 17a49ac..002a3af 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -48,7 +48,8 @@ https://metrics.torproject.org/identifier.csv <li><b>August 15, 2018:</b> Made the first batch of changes to per-graph CSV files.</li> <li><b>September 15, 2018:</b> Removed all pre-aggregated CSV files.</li> <li><b>October 28, 2018:</b> Added and/or removed columns to <a href="#webstats-tb-platform">Tor Browser downloads and updates by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads and updates by locale</a> graphs.</li> -<li><b>December 20, 2018:</b> Remove source parameters and output rows with aggregates over all sources from <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> +<li><b>December 20, 2018 (scheduled):</b> Remove source parameters and output rows with aggregates over all sources from <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> +<li><b>December 20, 2018 (scheduled):</b> Remove two graphs <a href="#bandwidth">Total relay bandwidth</a> and <a href="#bwhist-flags">Consumed bandwidth by Exit/Guard flag combination</a>, and update the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flag</a> graph to cover all data previously contained in the first two graphs.</li> </ul> </div> @@ -336,6 +337,11 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> <a href="/bandwidth.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> <a href="#bandwidth" name="bandwidth" class="anchor">#</a></h3> +<div class="bs-callout bs-callout-warning"> +<h3>Deprecated</h3> +<p>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph.</p> +</div> + <h4>Parameters</h4> <ul> @@ -367,10 +373,14 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> <ul> <li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>guard_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Guard"</b> relay flag are capable to provide.</li> -<li><b>guard_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Guard"</b> relay flag.</li> -<li><b>exit_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Exit"</b> relay flag are capable to provide.</li> -<li><b>exit_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Exit"</b> relay flag.</li> +<li><b>guard_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Guard"</b> relay flag are capable to provide. <span class="red">This column is going to be removed after December 20, 2018.</span></li> +<li><b>guard_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Guard"</b> relay flag. <span class="red">This column is going to be removed after December 20, 2018.</span></li> +<li><b>exit_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Exit"</b> relay flag are capable to provide. <span class="red">This column is going to be removed after December 20, 2018.</span></li> +<li><b>exit_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Exit"</b> relay flag. <span class="red">This column is going to be removed after December 20, 2018.</span></li> +<li><b>have_guard_flag:</b> Whether relays included in this row had the <code>"Guard"</code> relay flag assigned (<code>"t"</code>) or not (<code>"f"</code>). <span class="blue">This column is going to be added after December 20, 2018.</span></li> +<li><b>have_exit_flag:</b> Whether relays included in this row had the <code>"Exit"</code> relay flag assigned and at the same time the <code>"BadExit"</code> not assigned (<code>"t"</code>) or not (<code>"f"</code>). <span class="blue">This column is going to be added after December 20, 2018.</span></li> +<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide. <span class="blue">This column is going to be added after December 20, 2018.</span></li> +<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic. <span class="blue">This column is going to be added after December 20, 2018.</span></li> </ul> <h3>Advertised bandwidth by IP version @@ -446,6 +456,11 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> <a href="/bwhist-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> <a href="#bwhist-flags" name="bwhist-flags" class="anchor">#</a></h3> +<div class="bs-callout bs-callout-warning"> +<h3>Deprecated</h3> +<p>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph.</p> +</div> + <h4>Parameters</h4> <ul>

1 0

[metrics-web/release] Extend ipv6servers to replace servers.csv.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 54976512ff47cc3b8a2c6b5ab2cecc98b1ba8088 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 29 19:41:12 2018 +0100 Extend ipv6servers to replace servers.csv. This change extends the ipv6servers module to generate all relevant data about servers (relays and bridges) that the legacy module currently generates. There are several reasons why this is useful: - This change is a step towards implementing all statistics in Java, without needing to call psql or other external tools from within Ant. In fact, it's a step towards getting rid of Ant for executing modules. - The ipv6servers module already supports other statistics than absolute server counts, namely advertised bandwidths. It's easy to extend statistics to consensus weights and guard/middle/exit probabilities. This prepares future graphs which can be added quite easily. - With these new statistics we can finally provide graphs on bridges by version or platform. Or we can extend current graphs to display both relays and bridges, if we want to avoid adding more graphs. This commit does not yet remove any code from the legacy module. That will be the next logical step. It will even be fun. Implements #28116. --- CHANGELOG.md | 2 + src/main/R/rserver/graphs.R | 45 +-- .../metrics/stats/ipv6servers/Configuration.java | 2 +- .../metrics/stats/ipv6servers/Database.java | 416 ++++++++++++++++++--- .../stats/ipv6servers/Ipv6NetworkStatus.java | 48 ++- .../stats/ipv6servers/Ipv6ServerDescriptor.java | 8 + .../torproject/metrics/stats/ipv6servers/Main.java | 24 +- .../metrics/stats/ipv6servers/OutputLine.java | 72 ---- .../metrics/stats/ipv6servers/Parser.java | 158 ++++++-- .../metrics/stats/ipv6servers/Writer.java | 14 +- src/main/sql/ipv6servers/init-ipv6servers.sql | 410 ++++++++++++++++++-- 11 files changed, 972 insertions(+), 227 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f604ff9..028df01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ * Medium changes - Start downloading and processing votes. - Add Apache Commons Math 3.6.1 as dependency. + - Extend ipv6servers module to generate servers part of legacy + module. # Changes in version 1.2.0 - 2018-08-25 diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index b021791..9dc8c2d 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -349,16 +349,10 @@ robust_call <- function(wrappee, filename) { } prepare_networksize <- function(start_p, end_p) { - read.csv(paste(stats_dir, "servers.csv", sep = ""), + read.csv(paste(stats_dir, "networksize.csv", sep = ""), colClasses = c("date" = "Date")) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(flag == "") %>% - filter(country == "") %>% - filter(version == "") %>% - filter(platform == "") %>% - filter(ec2bridge == "") %>% - select(date, relays, bridges) + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) } plot_networksize <- function(start_p, end_p, path_p) { @@ -384,16 +378,10 @@ write_networksize <- function(start_p = NULL, end_p = NULL, path_p) { } prepare_versions <- function(start_p, end_p) { - read.csv(paste(stats_dir, "servers.csv", sep = ""), + read.csv(paste(stats_dir, "versions.csv", sep = ""), colClasses = c("date" = "Date")) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(flag == "") %>% - filter(country == "") %>% - filter(version != "") %>% - filter(platform == "") %>% - filter(ec2bridge == "") %>% - select(date, version, relays) + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) } plot_versions <- function(start_p, end_p, path_p) { @@ -428,18 +416,10 @@ write_versions <- function(start_p = NULL, end_p = NULL, path_p) { } prepare_platforms <- function(start_p, end_p) { - read.csv(paste(stats_dir, "servers.csv", sep = ""), + read.csv(paste(stats_dir, "platforms.csv", sep = ""), colClasses = c("date" = "Date")) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(flag == "") %>% - filter(country == "") %>% - filter(version == "") %>% - filter(platform != "") %>% - filter(ec2bridge == "") %>% - select(date, platform, relays) %>% - mutate(platform = ifelse(platform == "Darwin", "macOS", - as.character(platform))) + filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) } plot_platforms <- function(start_p, end_p, path_p) { @@ -451,7 +431,8 @@ plot_platforms <- function(start_p, end_p, path_p) { scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + scale_colour_manual(name = "Platform", breaks = c("Linux", "macOS", "BSD", "Windows", "Other"), - values = c("#E69F00", "#56B4E9", "#009E73", "#0072B2", "#333333")) + + values = c("Linux" = "#56B4E9", "macOS" = "#333333", "BSD" = "#E69F00", + "Windows" = "#0072B2", "Other" = "#009E73")) + ggtitle("Relay platforms") + labs(caption = copyright_notice) ggsave(filename = path_p, width = 8, height = 5, dpi = 150) @@ -576,17 +557,11 @@ write_dirbytes <- function(start_p = NULL, end_p = NULL, path_p) { } prepare_relayflags <- function(start_p, end_p, flag_p) { - read.csv(paste(stats_dir, "servers.csv", sep = ""), + read.csv(paste(stats_dir, "relayflags.csv", sep = ""), colClasses = c("date" = "Date")) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(country == "") %>% - filter(version == "") %>% - filter(platform == "") %>% - filter(ec2bridge == "") %>% - mutate(flag = ifelse(flag == "", "Running", as.character(flag))) %>% - filter(if (!is.null(flag_p)) flag %in% flag_p else TRUE) %>% - select(date, flag, relays) + filter(if (!is.null(flag_p)) flag %in% flag_p else TRUE) } plot_relayflags <- function(start_p, end_p, flag_p, path_p) { diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java index c940882..d849cb6 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java @@ -13,6 +13,6 @@ class Configuration { static String history = System.getProperty("ipv6servers.history", "status/read-descriptors"); static String output = System.getProperty("ipv6servers.output", - "stats/ipv6servers.csv"); + "stats/"); } diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java index 67afa92..c3a1fec 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java @@ -10,12 +10,18 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; +import java.sql.Types; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Calendar; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; import java.util.TimeZone; /** Database wrapper to connect to the database, insert data, run the stored @@ -28,6 +34,37 @@ class Database implements AutoCloseable { /** Connection object for all interactions with the database. */ private Connection connection; + /** Cache for the mapping of platform strings to identifiers in the database. + * Initialized at startup and kept in sync with the database. */ + private Map<String, Integer> platformsCache = new HashMap<>(); + + /** Cache for the mapping of version strings to identifiers in the database. + * Initialized at startup and kept in sync with the database. */ + private Map<String, Integer> versionsCache = new HashMap<>(); + + /** Cache of version strings that have been recommended as server version at + * least once in a consensus. Initialized at startup and kept in sync with the + * database. */ + private Set<String> recommendedVersions = new HashSet<>(); + + /** Cache for the mapping of relay flags to identifiers in the database. + * Initialized at startup and kept in sync with the database. */ + private Map<String, Integer> flagsCache = new HashMap<>(); + + /** Prepared statement for inserting a platform string into the platforms + * table. */ + private PreparedStatement psPlatformsInsert; + + /** Prepared statement for inserting a version string into the versions + * table. */ + private PreparedStatement psVersionsInsert; + + /** Prepared statement for updating a version in the versions table. */ + private PreparedStatement psVersionsUpdate; + + /** Prepared statement for inserting a flag into the flags table. */ + private PreparedStatement psFlagsInsert; + /** Prepared statement for finding out whether a given server descriptor is * already contained in the server_descriptors table. */ private PreparedStatement psServerDescriptorsSelect; @@ -54,6 +91,7 @@ class Database implements AutoCloseable { this.jdbcString = jdbcString; this.connect(); this.prepareStatements(); + this.initializeCaches(); } private void connect() throws SQLException { @@ -62,24 +100,69 @@ class Database implements AutoCloseable { } private void prepareStatements() throws SQLException { + this.psPlatformsInsert = this.connection.prepareStatement( + "INSERT INTO platforms (platform_string) VALUES (?)", + Statement.RETURN_GENERATED_KEYS); + this.psVersionsInsert = this.connection.prepareStatement( + "INSERT INTO versions (version_string, recommended) VALUES (?, ?)", + Statement.RETURN_GENERATED_KEYS); + this.psVersionsUpdate = this.connection.prepareStatement( + "UPDATE versions SET recommended = TRUE WHERE version_id = ?"); + this.psFlagsInsert = this.connection.prepareStatement( + "INSERT INTO flags (flag_id, flag_string) VALUES (?, ?)"); this.psServerDescriptorsSelect = this.connection.prepareStatement( "SELECT EXISTS (SELECT 1 FROM server_descriptors " - + "WHERE descriptor_digest_sha1 = decode(?, 'hex'))"); + + "WHERE descriptor_digest_sha1 = decode(?, 'hex'))"); this.psServerDescriptorsInsert = this.connection.prepareStatement( - "INSERT INTO server_descriptors (descriptor_digest_sha1, " - + "advertised_bandwidth_bytes, announced_ipv6, exiting_ipv6_relay) " - + "VALUES (decode(?, 'hex'), ?, ?, ?)"); + "INSERT INTO server_descriptors (descriptor_digest_sha1, platform_id, " + + "version_id, advertised_bandwidth_bytes, announced_ipv6, " + + "exiting_ipv6_relay) VALUES (decode(?, 'hex'), ?, ?, ?, ?, ?)"); this.psStatusesSelect = this.connection.prepareStatement( "SELECT EXISTS (SELECT 1 FROM statuses " - + "WHERE server = CAST(? AS server_enum) AND valid_after = ?)"); + + "WHERE server = CAST(? AS server_enum) AND valid_after = ?)"); this.psStatusesInsert = this.connection.prepareStatement( - "INSERT INTO statuses (server, valid_after, running_count) " - + "VALUES (CAST(? AS server_enum), ?, ?)", + "INSERT INTO statuses (server, valid_after, running_count, " + + "consensus_weight_sum, guard_weight_sum, middle_weight_sum, " + + "exit_weight_sum) VALUES (CAST(? AS server_enum), ?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS); this.psStatusEntriesInsert = this.connection.prepareStatement( "INSERT INTO status_entries (status_id, descriptor_digest_sha1, " - + "guard_relay, exit_relay, reachable_ipv6_relay) " - + "VALUES (?, decode(?, 'hex'), ?, ?, ?)"); + + "flags, reachable_ipv6_relay, consensus_weight, guard_weight, " + + "middle_weight, exit_weight) " + + "VALUES (?, decode(?, 'hex'), ?, ?, ?, ?, ?, ?)"); + } + + private void initializeCaches() throws SQLException { + Statement st = this.connection.createStatement(); + String queryString = "SELECT platform_id, platform_string FROM platforms"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + this.platformsCache.put( + rs.getString("platform_string"), rs.getInt("platform_id")); + } + } + st = this.connection.createStatement(); + queryString = "SELECT version_id, version_string, recommended " + + "FROM versions"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + String version = rs.getString("version_string"); + int versionId = rs.getInt("version_id"); + boolean recommended = rs.getBoolean("recommended"); + this.versionsCache.put(version, versionId); + if (recommended) { + this.recommendedVersions.add(version); + } + } + } + st = this.connection.createStatement(); + queryString = "SELECT flag_id, flag_string FROM flags"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + this.flagsCache.put( + rs.getString("flag_string"), rs.getInt("flag_id")); + } + } } /** Insert a server descriptor into the server_descriptors table. */ @@ -98,17 +181,87 @@ class Database implements AutoCloseable { } this.psServerDescriptorsInsert.clearParameters(); this.psServerDescriptorsInsert.setString(1, serverDescriptor.digest); - this.psServerDescriptorsInsert.setInt(2, + if (null != serverDescriptor.platform) { + this.psServerDescriptorsInsert.setInt(2, + this.selectOrInsertPlatform(serverDescriptor.platform)); + } else { + this.psServerDescriptorsInsert.setNull(2, Types.INTEGER); + } + if (null != serverDescriptor.version) { + this.psServerDescriptorsInsert.setInt(3, + this.selectOrInsertVersion(serverDescriptor.version, false)); + } else { + this.psServerDescriptorsInsert.setNull(3, Types.INTEGER); + } + this.psServerDescriptorsInsert.setInt(4, serverDescriptor.advertisedBandwidth); - this.psServerDescriptorsInsert.setBoolean(3, serverDescriptor.announced); - this.psServerDescriptorsInsert.setBoolean(4, serverDescriptor.exiting); + this.psServerDescriptorsInsert.setBoolean(5, serverDescriptor.announced); + this.psServerDescriptorsInsert.setBoolean(6, serverDescriptor.exiting); this.psServerDescriptorsInsert.execute(); } + /** Return the platform identifier for a given platform string, either from + * our local cache, from a database query, or after inserting it into the + * platforms table. */ + int selectOrInsertPlatform(String platform) throws SQLException { + if (!this.platformsCache.containsKey(platform)) { + int platformId = -1; + this.psPlatformsInsert.clearParameters(); + this.psPlatformsInsert.setString(1, platform); + this.psPlatformsInsert.execute(); + try (ResultSet rs = this.psPlatformsInsert.getGeneratedKeys()) { + if (rs.next()) { + platformId = rs.getInt(1); + } + } + if (platformId < 0) { + throw new SQLException("Could not retrieve auto-generated key for " + + "new platforms entry."); + } + this.platformsCache.put(platform, platformId); + } + return this.platformsCache.get(platform); + } + + /** Return the version identifier for a given version string, either from our + * local cache, from a database query, or after inserting it into the versions + * table. */ + int selectOrInsertVersion(String version, boolean recommended) + throws SQLException { + if (!this.versionsCache.containsKey(version)) { + int versionId = -1; + this.psVersionsInsert.clearParameters(); + this.psVersionsInsert.setString(1, version); + this.psVersionsInsert.setBoolean(2, recommended); + this.psVersionsInsert.execute(); + try (ResultSet rs = this.psVersionsInsert.getGeneratedKeys()) { + if (rs.next()) { + versionId = rs.getInt(1); + } + } + if (versionId < 0) { + throw new SQLException("Could not retrieve auto-generated key for " + + "new versions entry."); + } + this.versionsCache.put(version, versionId); + if (recommended) { + this.recommendedVersions.add(version); + } + } + if (recommended && !this.recommendedVersions.contains(version)) { + int versionId = this.versionsCache.get(version); + this.psVersionsUpdate.clearParameters(); + this.psVersionsUpdate.setInt(1, versionId); + this.psVersionsUpdate.execute(); + this.recommendedVersions.add(version); + } + return this.versionsCache.get(version); + } + /** Insert a status and all contained entries into the statuses and * status_entries table. */ - void insertStatus(Ipv6NetworkStatus networkStatus) - throws SQLException { + void insertStatus(Ipv6NetworkStatus networkStatus) throws SQLException { + this.insertRecommendedVersions(networkStatus.recommendedVersions); this.psStatusesSelect.clearParameters(); this.psStatusesSelect.setString(1, networkStatus.isRelay ? "relay" : "bridge"); @@ -133,6 +286,26 @@ class Database implements AutoCloseable { Timestamp.from(ZonedDateTime.of(networkStatus.timestamp, ZoneId.of("UTC")).toInstant()), calendar); this.psStatusesInsert.setInt(3, networkStatus.running); + if (null != networkStatus.totalConsensusWeight) { + this.psStatusesInsert.setFloat(4, networkStatus.totalConsensusWeight); + } else { + this.psStatusesInsert.setNull(4, Types.FLOAT); + } + if (null != networkStatus.totalGuardWeight) { + this.psStatusesInsert.setFloat(5, networkStatus.totalGuardWeight); + } else { + this.psStatusesInsert.setNull(5, Types.FLOAT); + } + if (null != networkStatus.totalMiddleWeight) { + this.psStatusesInsert.setFloat(6, networkStatus.totalMiddleWeight); + } else { + this.psStatusesInsert.setNull(6, Types.FLOAT); + } + if (null != networkStatus.totalExitWeight) { + this.psStatusesInsert.setFloat(7, networkStatus.totalExitWeight); + } else { + this.psStatusesInsert.setNull(7, Types.FLOAT); + } this.psStatusesInsert.execute(); try (ResultSet rs = this.psStatusesInsert.getGeneratedKeys()) { if (rs.next()) { @@ -144,22 +317,81 @@ class Database implements AutoCloseable { + "statuses entry."); } for (Ipv6NetworkStatus.Entry entry : networkStatus.entries) { - this.psStatusEntriesInsert.clearParameters(); - this.psStatusEntriesInsert.setInt(1, statusId); - this.psStatusEntriesInsert.setString(2, entry.digest); - this.psStatusEntriesInsert.setBoolean(3, entry.guard); - this.psStatusEntriesInsert.setBoolean(4, entry.exit); - this.psStatusEntriesInsert.setBoolean(5, entry.reachable); - this.psStatusEntriesInsert.addBatch(); + this.insertStatusEntry(statusId, entry); } this.psStatusEntriesInsert.executeBatch(); } + + void insertRecommendedVersions(List<String> versions) + throws SQLException { + if (null != versions + && !this.versionsCache.keySet().containsAll(versions)) { + for (String version : versions) { + this.selectOrInsertVersion(version, true); + } + } + } + + /** Insert a status entry into the status_entries table. */ + void insertStatusEntry(int statusId, Ipv6NetworkStatus.Entry entry) + throws SQLException { + this.insertFlags(entry.flags); + int flags = 0; + if (null != entry.flags) { + for (String flag : entry.flags) { + int flagId = this.flagsCache.get(flag); + flags |= 1 << flagId; + } + } + this.psStatusEntriesInsert.clearParameters(); + this.psStatusEntriesInsert.setInt(1, statusId); + this.psStatusEntriesInsert.setString(2, entry.digest); + this.psStatusEntriesInsert.setInt(3, flags); + this.psStatusEntriesInsert.setBoolean(4, entry.reachable); + if (null != entry.consensusWeight) { + this.psStatusEntriesInsert.setFloat(5, entry.consensusWeight); + } else { + this.psStatusEntriesInsert.setNull(5, Types.FLOAT); + } + if (null != entry.guardWeight) { + this.psStatusEntriesInsert.setFloat(6, entry.guardWeight); + } else { + this.psStatusEntriesInsert.setNull(6, Types.FLOAT); + } + if (null != entry.middleWeight) { + this.psStatusEntriesInsert.setFloat(7, entry.middleWeight); + } else { + this.psStatusEntriesInsert.setNull(7, Types.FLOAT); + } + if (null != entry.exitWeight) { + this.psStatusEntriesInsert.setFloat(8, entry.exitWeight); + } else { + this.psStatusEntriesInsert.setNull(8, Types.FLOAT); + } + this.psStatusEntriesInsert.addBatch(); + } + + void insertFlags(SortedSet<String> flags) throws SQLException { + if (null != flags && !this.flagsCache.keySet().containsAll(flags)) { + for (String flag : flags) { + if (!this.flagsCache.containsKey(flag)) { + int flagId = this.flagsCache.size(); + this.psFlagsInsert.clearParameters(); + this.psFlagsInsert.setInt(1, flagId); + this.psFlagsInsert.setString(2, flag); + this.psFlagsInsert.execute(); + this.flagsCache.put(flag, flagId); + } + } + } + } + /** Call the aggregate() function to aggregate rows from the status_entries - * and server_descriptors tables into the aggregated table. */ + * and server_descriptors tables into the aggregated_* tables. */ void aggregate() throws SQLException { Statement st = this.connection.createStatement(); - st.executeQuery("SELECT aggregate_ipv6()"); + st.executeQuery("SELECT aggregate()"); } /** Roll back any changes made in this execution. */ @@ -172,41 +404,131 @@ class Database implements AutoCloseable { this.connection.commit(); } - /** Query the servers_ipv6 view to obtain aggregated statistics. */ - Iterable<OutputLine> queryServersIpv6() throws SQLException { - List<OutputLine> statistics = new ArrayList<>(); + /** Query the servers_ipv6 view. */ + List<String[]> queryServersIpv6() throws SQLException { + List<String[]> statistics = new ArrayList<>(); + String columns = "valid_after_date, server, guard_relay, exit_relay, " + + "announced_ipv6, exiting_ipv6_relay, reachable_ipv6_relay, " + + "server_count_sum_avg, advertised_bandwidth_bytes_sum_avg"; + statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.US); - String queryString = "SELECT " + OutputLine.columnHeadersDelimitedBy(", ") - + " FROM ipv6servers"; + String queryString = "SELECT " + columns + " FROM ipv6servers"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { - OutputLine outputLine = new OutputLine(); - outputLine.date = rs.getDate(OutputLine.Column.VALID_AFTER_DATE.name(), - calendar).toLocalDate(); - outputLine.server = rs.getString(OutputLine.Column.SERVER.name()); - outputLine.guard = rs.getString(OutputLine.Column.GUARD_RELAY.name()); - outputLine.exit = rs.getString(OutputLine.Column.EXIT_RELAY.name()); - outputLine.announced = rs.getString( - OutputLine.Column.ANNOUNCED_IPV6.name()); - outputLine.exiting = rs.getString( - OutputLine.Column.EXITING_IPV6_RELAY.name()); - outputLine.reachable = rs.getString( - OutputLine.Column.REACHABLE_IPV6_RELAY.name()); - outputLine.count = rs.getLong( - OutputLine.Column.SERVER_COUNT_SUM_AVG.name()); - outputLine.advertisedBandwidth = rs.getLong( - OutputLine.Column.ADVERTISED_BANDWIDTH_BYTES_SUM_AVG.name()); - if (rs.wasNull()) { - outputLine.advertisedBandwidth = null; - } + String[] outputLine = new String[9]; + outputLine[0] = rs.getDate("valid_after_date", calendar) + .toLocalDate().toString(); + outputLine[1] = rs.getString("server"); + outputLine[2] = rs.getString("guard_relay"); + outputLine[3] = rs.getString("exit_relay"); + outputLine[4] = rs.getString("announced_ipv6"); + outputLine[5] = rs.getString("exiting_ipv6_relay"); + outputLine[6] = rs.getString("reachable_ipv6_relay"); + outputLine[7] = getLongFromResultSet(rs, "server_count_sum_avg"); + outputLine[8] = getLongFromResultSet(rs, + "advertised_bandwidth_bytes_sum_avg"); + statistics.add(outputLine); + } + } + return statistics; + } + + /** Query the servers_networksize view. */ + List<String[]> queryNetworksize() throws SQLException { + List<String[]> statistics = new ArrayList<>(); + String columns = "date, relays, bridges"; + statistics.add(columns.split(", ")); + Statement st = this.connection.createStatement(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + String queryString = "SELECT " + columns + " FROM servers_networksize"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + String[] outputLine = new String[3]; + outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[1] = getLongFromResultSet(rs, "relays"); + outputLine[2] = getLongFromResultSet(rs, "bridges"); statistics.add(outputLine); } } return statistics; } + /** Query the servers_relayflags view. */ + List<String[]> queryRelayflags() throws SQLException { + List<String[]> statistics = new ArrayList<>(); + String columns = "date, flag, relays"; + statistics.add(columns.split(", ")); + Statement st = this.connection.createStatement(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + String queryString = "SELECT " + columns + " FROM servers_relayflags"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + String[] outputLine = new String[3]; + outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[1] = rs.getString("flag"); + outputLine[2] = getLongFromResultSet(rs, "relays"); + statistics.add(outputLine); + } + } + return statistics; + } + + /** Query the servers_versions view. */ + List<String[]> queryVersions() throws SQLException { + List<String[]> statistics = new ArrayList<>(); + String columns = "date, version, relays"; + statistics.add(columns.split(", ")); + Statement st = this.connection.createStatement(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + String queryString = "SELECT " + columns + " FROM servers_versions"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + String[] outputLine = new String[3]; + outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[1] = rs.getString("version"); + outputLine[2] = getLongFromResultSet(rs, "relays"); + statistics.add(outputLine); + } + } + return statistics; + } + + /** Query the servers_platforms view. */ + List<String[]> queryPlatforms() throws SQLException { + List<String[]> statistics = new ArrayList<>(); + String columns = "date, platform, relays"; + statistics.add(columns.split(", ")); + Statement st = this.connection.createStatement(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + String queryString = "SELECT " + columns + " FROM servers_platforms"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + String[] outputLine = new String[3]; + outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[1] = rs.getString("platform"); + outputLine[2] = getLongFromResultSet(rs, "relays"); + statistics.add(outputLine); + } + } + return statistics; + } + + /** Retrieve the <code>long</code> value of the designated column in the + * current row of the given <code>ResultSet</code> object and format it as a + * <code>String</code> object, or return <code>null</code> if the retrieved + * value was <code>NULL</code>. */ + private static String getLongFromResultSet(ResultSet rs, String columnLabel) + throws SQLException { + long result = rs.getLong(columnLabel); + return rs.wasNull() ? null : String.valueOf(result); + } + /** Release database connection. */ public void close() throws SQLException { this.connection.close(); diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java index 611ca4e..526adc7 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6NetworkStatus.java @@ -6,6 +6,7 @@ package org.torproject.metrics.stats.ipv6servers; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.SortedSet; /** Data object holding all relevant parts parsed from a (relay or bridge) * network status. */ @@ -19,9 +20,32 @@ class Ipv6NetworkStatus { * case of bridge network status. */ LocalDateTime timestamp; + /** List of recommended (server) versions, or null in case of bridge network + * status. */ + List<String> recommendedVersions; + /** Number of relays or bridges with the Running flag. */ int running = 0; + /** Total consensus weight of all status entries, or null if the status does + * not contain entries with consensus weights. */ + Float totalConsensusWeight; + + /** Total guard-weighted consensus weight of all status entries, or null if + * the status either does not contain entries with consensus weight or no Wxx + * values. */ + Float totalGuardWeight; + + /** Total middle-weighted consensus weight of all status entries, or null if + * the status either does not contain entries with consensus weight or no Wxx + * values. */ + Float totalMiddleWeight; + + /** Total exit-weighted consensus weight of all status entries, or null if + * the status either does not contain entries with consensus weight or no Wxx + * values. */ + Float totalExitWeight; + /** Contained status entries. */ List<Entry> entries = new ArrayList<>(); @@ -31,17 +55,29 @@ class Ipv6NetworkStatus { /** Hex-encoded SHA-1 server descriptor digest. */ String digest; - /** Whether this relay has the Guard flag; false for bridges. */ - boolean guard; - - /** Whether this relay has the Exit flag (and not the BadExit flag at the - * same time); false for bridges. */ - boolean exit; + /** Relay flags assigned to this relay or bridge. */ + SortedSet<String> flags; /** Whether the directory authorities include an IPv6 address in this * entry's "a" line, confirming the relay's reachability via IPv6; false for * bridges. */ boolean reachable; + + /** Consensus weight of this entry, or null if the entry does not have a "w" + * line. */ + Float consensusWeight; + + /** Guard-weighted consensus weight of this entry, or null if either the + * entry does not have a "w" line or the consensus has no Wxx values. */ + Float guardWeight; + + /** Middle-weighted consensus weight of this entry, or null if either the + * entry does not have a "w" line or the consensus has no Wxx values. */ + Float middleWeight; + + /** Exit-weighted consensus weight of this entry, or null if either the + * entry does not have a "w" line or the consensus has no Wxx values. */ + Float exitWeight; } } diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java index cb725b0..387024b 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Ipv6ServerDescriptor.java @@ -10,6 +10,14 @@ class Ipv6ServerDescriptor { /** Hex-encoded SHA-1 server descriptor digest. */ String digest; + /** Platform obtained from the platform line without the Tor software + * version. */ + String platform; + + /** Tor software version obtained from the platform line without the + * platform. */ + String version; + /** Advertised bandwidth bytes of this relay as the minimum of bandwidth rate, * bandwidth burst, and observed bandwidth (if reported); 0 for bridges. */ int advertisedBandwidth; diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java index 5a20b58..a91a74f 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java @@ -20,7 +20,7 @@ import java.util.Arrays; /** Main class of the ipv6servers module that imports relevant parts from server * descriptors and network statuses into a database, and exports aggregate - * statistics on IPv6 support to a CSV file. */ + * statistics to CSV files. */ public class Main { private static Logger log = LoggerFactory.getLogger(Main.class); @@ -61,8 +61,13 @@ public class Main { } else if (descriptor instanceof BridgeNetworkStatus) { database.insertStatus(parser.parseBridgeNetworkStatus( (BridgeNetworkStatus) descriptor)); + } else if (null != descriptor.getRawDescriptorBytes()) { + log.debug("Skipping unknown descriptor of type {} starting with " + + "'{}'.", descriptor.getClass(), + new String(descriptor.getRawDescriptorBytes(), 0, + Math.min(descriptor.getRawDescriptorLength(), 100))); } else { - log.debug("Skipping unknown descriptor of type {}.", + log.debug("Skipping unknown, empty descriptor of type {}.", descriptor.getClass()); } } @@ -81,11 +86,16 @@ public class Main { reader.saveHistoryFile(historyFile); log.info("Querying aggregated statistics from the database."); - Iterable<OutputLine> output = database.queryServersIpv6(); - log.info("Writing aggregated statistics to {}.", Configuration.output); - if (null != output) { - new Writer().write(Paths.get(Configuration.output), output); - } + new Writer().write(Paths.get(Configuration.output, "ipv6servers.csv"), + database.queryServersIpv6()); + new Writer().write(Paths.get(Configuration.output, "networksize.csv"), + database.queryNetworksize()); + new Writer().write(Paths.get(Configuration.output, "relayflags.csv"), + database.queryRelayflags()); + new Writer().write(Paths.get(Configuration.output, "versions.csv"), + database.queryVersions()); + new Writer().write(Paths.get(Configuration.output, "platforms.csv"), + database.queryPlatforms()); log.info("Terminating ipv6servers module."); } catch (SQLException sqle) { diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java deleted file mode 100644 index a5acfa0..0000000 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.ipv6servers; - -import java.time.LocalDate; -import java.util.Arrays; -import java.util.stream.Collectors; - -/** Data object holding all parts of an output line. */ -class OutputLine { - - /** Column names used in the database and in the first line of the output - * file. */ - enum Column { - VALID_AFTER_DATE, SERVER, GUARD_RELAY, EXIT_RELAY, ANNOUNCED_IPV6, - EXITING_IPV6_RELAY, REACHABLE_IPV6_RELAY, SERVER_COUNT_SUM_AVG, - ADVERTISED_BANDWIDTH_BYTES_SUM_AVG - } - - /** Column headers joined together with the given delimiter. */ - static String columnHeadersDelimitedBy(String delimiter) { - return Arrays.stream(Column.values()).map(c -> c.toString().toLowerCase()) - .collect(Collectors.joining(delimiter)); - } - - /** Date. */ - LocalDate date; - - /** Server type, which can be "relay" or "bridge". */ - String server; - - /** Whether relays had the Guard flag ("t") or not ("f"). */ - String guard; - - /** Whether relays had the Exit flag ("t") or not ("f"). */ - String exit; - - /** Whether relays or bridges have announced an IPv6 address in their server - * descriptor ("t") or not ("f"). */ - String announced; - - /** Whether relays have announced a non-reject-all IPv6 exit policy in their - * server descriptor ("t") or not ("f"). */ - String exiting; - - /** Whether the directory authorities have confirmed IPv6 OR reachability by - * including an "a" line for a relay containing an IPv6 address. */ - String reachable; - - /** Number of relays or bridges matching the previous criteria. */ - long count; - - /** Total advertised bandwidth of all relays matching the previous - * criteria. */ - Long advertisedBandwidth; - - /** Format all fields in a single output line for inclusion in a CSV - * file. */ - @Override - public String toString() { - return String.format("%s,%s,%s,%s,%s,%s,%s,%s,%s", - date, server, emptyNull(guard), emptyNull(exit), emptyNull(announced), - emptyNull(exiting), emptyNull(reachable), emptyNull(count), - emptyNull(advertisedBandwidth)); - } - - private static String emptyNull(Object text) { - return null == text ? "" : text.toString(); - } -} - diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java index 95c495e..9d8b71a 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java @@ -13,17 +13,31 @@ import org.apache.commons.lang3.StringUtils; import java.time.Instant; import java.time.ZoneId; +import java.util.Arrays; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** Parser that extracts all relevant parts from (relay and bridge) server * descriptors and (relay and bridge) statuses and creates data objects for * them. */ class Parser { + private Pattern platformPattern = Pattern.compile("^Tor (.+) on (.+)$"); + /** Parse a (relay or bridge) server descriptor. */ Ipv6ServerDescriptor parseServerDescriptor( ServerDescriptor serverDescriptor) { Ipv6ServerDescriptor parsedDescriptor = new Ipv6ServerDescriptor(); parsedDescriptor.digest = serverDescriptor.getDigestSha1Hex(); + if (null != serverDescriptor.getPlatform()) { + Matcher platformMatcher = platformPattern.matcher( + serverDescriptor.getPlatform()); + if (platformMatcher.matches() && platformMatcher.groupCount() == 2) { + parsedDescriptor.version = platformMatcher.group(1); + parsedDescriptor.platform = platformMatcher.group(2); + } + } for (String orAddress : serverDescriptor.getOrAddresses()) { /* Check whether the additional OR address is an IPv6 address containing * at least two colons as opposed to an IPv4 address and TCP port @@ -52,48 +66,132 @@ class Parser { Ipv6NetworkStatus parseRelayNetworkStatusConsensus( RelayNetworkStatusConsensus consensus) { - return this.parseStatus(true, consensus.getValidAfterMillis(), - consensus.getStatusEntries().values()); - } - - Ipv6NetworkStatus parseBridgeNetworkStatus(BridgeNetworkStatus status) { - return this.parseStatus(false, status.getPublishedMillis(), - status.getStatusEntries().values()); - } - - private Ipv6NetworkStatus parseStatus(boolean isRelay, long timestampMillis, - Iterable<NetworkStatusEntry> entries) { Ipv6NetworkStatus parsedStatus = new Ipv6NetworkStatus(); - parsedStatus.isRelay = isRelay; - parsedStatus.timestamp = Instant.ofEpochMilli(timestampMillis) + parsedStatus.isRelay = true; + parsedStatus.timestamp = Instant.ofEpochMilli( + consensus.getValidAfterMillis()) .atZone(ZoneId.of("UTC")).toLocalDateTime(); - for (NetworkStatusEntry entry : entries) { - if (!entry.getFlags().contains("Running")) { - continue; + parsedStatus.recommendedVersions = consensus.getRecommendedServerVersions(); + boolean consensusContainsBandwidthWeights = + null != consensus.getBandwidthWeights() + && consensus.getBandwidthWeights().keySet().containsAll(Arrays.asList( + "Wgg", "Wgd", "Wmg", "Wmm", "Wme", "Wmd", "Wee", "Wed")); + float wgg = 0.0f; + float wgd = 0.0f; + float wmg = 0.0f; + float wmm = 0.0f; + float wme = 0.0f; + float wmd = 0.0f; + float wee = 0.0f; + float wed = 0.0f; + if (consensusContainsBandwidthWeights) { + for (Map.Entry<String, Integer> e + : consensus.getBandwidthWeights().entrySet()) { + float weight = e.getValue().floatValue() / 10000.0f; + switch (e.getKey()) { + case "Wgg": + wgg = weight; + break; + case "Wgd": + wgd = weight; + break; + case "Wmg": + wmg = weight; + break; + case "Wmm": + wmm = weight; + break; + case "Wme": + wme = weight; + break; + case "Wmd": + wmd = weight; + break; + case "Wee": + wee = weight; + break; + case "Wed": + wed = weight; + break; + default: + /* Ignore other weights. */ + } } - parsedStatus.running++; } - for (NetworkStatusEntry entry : entries) { + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { if (!entry.getFlags().contains("Running")) { continue; } Ipv6NetworkStatus.Entry parsedEntry = new Ipv6NetworkStatus.Entry(); parsedEntry.digest = entry.getDescriptor().toLowerCase(); - if (isRelay) { - parsedEntry.guard = entry.getFlags().contains("Guard"); - parsedEntry.exit = entry.getFlags().contains("Exit") - && !entry.getFlags().contains("BadExit"); - parsedEntry.reachable = false; - for (String orAddress : entry.getOrAddresses()) { - /* Check whether the additional OR address is an IPv6 address - * containing at least two colons as opposed to an IPv4 address and - * TCP port containing only one colon as separator. */ - if (StringUtils.countMatches(orAddress, ":") >= 2) { - parsedEntry.reachable = true; - break; + parsedEntry.flags = entry.getFlags(); + parsedEntry.reachable = false; + for (String orAddress : entry.getOrAddresses()) { + /* Check whether the additional OR address is an IPv6 address + * containing at least two colons as opposed to an IPv4 address and + * TCP port containing only one colon as separator. */ + if (StringUtils.countMatches(orAddress, ":") >= 2) { + parsedEntry.reachable = true; + break; + } + } + parsedStatus.running++; + boolean isExit = entry.getFlags().contains("Exit") + && !entry.getFlags().contains("BadExit"); + boolean isGuard = entry.getFlags().contains("Guard"); + long consensusWeight = entry.getBandwidth(); + if (consensusWeight >= 0L) { + parsedEntry.consensusWeight = (float) consensusWeight; + if (consensusContainsBandwidthWeights) { + if (isGuard && isExit) { + parsedEntry.guardWeight = parsedEntry.consensusWeight * wgd; + parsedEntry.middleWeight = parsedEntry.consensusWeight * wmd; + parsedEntry.exitWeight = parsedEntry.consensusWeight * wed; + } else if (isGuard) { + parsedEntry.guardWeight = parsedEntry.consensusWeight * wgg; + parsedEntry.middleWeight = parsedEntry.consensusWeight * wmg; + parsedEntry.exitWeight = 0.0f; + } else if (isExit) { + parsedEntry.guardWeight = 0.0f; + parsedEntry.middleWeight = parsedEntry.consensusWeight * wme; + parsedEntry.exitWeight = parsedEntry.consensusWeight * wee; + } else { + parsedEntry.guardWeight = 0.0f; + parsedEntry.middleWeight = parsedEntry.consensusWeight * wmm; + parsedEntry.exitWeight = 0.0f; + } + if (null == parsedStatus.totalGuardWeight) { + parsedStatus.totalGuardWeight = 0.0f; + parsedStatus.totalMiddleWeight = 0.0f; + parsedStatus.totalExitWeight = 0.0f; } + parsedStatus.totalGuardWeight += parsedEntry.guardWeight; + parsedStatus.totalMiddleWeight += parsedEntry.middleWeight; + parsedStatus.totalExitWeight += parsedEntry.exitWeight; } + if (null == parsedStatus.totalConsensusWeight) { + parsedStatus.totalConsensusWeight = 0.0f; + } + parsedStatus.totalConsensusWeight += parsedEntry.consensusWeight; + } + parsedStatus.entries.add(parsedEntry); + } + return parsedStatus; + } + + Ipv6NetworkStatus parseBridgeNetworkStatus(BridgeNetworkStatus status) { + Ipv6NetworkStatus parsedStatus = new Ipv6NetworkStatus(); + parsedStatus.isRelay = false; + parsedStatus.timestamp = Instant.ofEpochMilli(status.getPublishedMillis()) + .atZone(ZoneId.of("UTC")).toLocalDateTime(); + for (NetworkStatusEntry entry : status.getStatusEntries().values()) { + if (!entry.getFlags().contains("Running")) { + continue; } + parsedStatus.running++; + Ipv6NetworkStatus.Entry parsedEntry = new Ipv6NetworkStatus.Entry(); + parsedEntry.digest = entry.getDescriptor().toLowerCase(); + parsedEntry.flags = entry.getFlags(); parsedStatus.entries.add(parsedEntry); } return parsedStatus; diff --git a/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java b/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java index 9edb347..4e2893c 100644 --- a/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java +++ b/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java @@ -16,7 +16,7 @@ import java.util.List; class Writer { /** Write output lines to the given file. */ - void write(Path filePath, Iterable<OutputLine> outputLines) + void write(Path filePath, Iterable<String[]> outputLines) throws IOException { File parentFile = filePath.toFile().getParentFile(); if (null != parentFile && !parentFile.exists()) { @@ -26,9 +26,15 @@ class Writer { } } List<String> formattedOutputLines = new ArrayList<>(); - formattedOutputLines.add(OutputLine.columnHeadersDelimitedBy(",")); - for (OutputLine line : outputLines) { - formattedOutputLines.add(line.toString()); + for (String[] outputLine : outputLines) { + StringBuilder formattedOutputLine = new StringBuilder(); + for (String outputLinePart : outputLine) { + formattedOutputLine.append(','); + if (null != outputLinePart) { + formattedOutputLine.append(outputLinePart); + } + } + formattedOutputLines.add(formattedOutputLine.substring(1)); } Files.write(filePath, formattedOutputLines, StandardCharsets.UTF_8); } diff --git a/src/main/sql/ipv6servers/init-ipv6servers.sql b/src/main/sql/ipv6servers/init-ipv6servers.sql index 8ed9372..b478a49 100644 --- a/src/main/sql/ipv6servers/init-ipv6servers.sql +++ b/src/main/sql/ipv6servers/init-ipv6servers.sql @@ -1,12 +1,54 @@ -- Copyright 2017--2018 The Tor Project -- See LICENSE for licensing information +-- Table of all known flags, to match flag strings to bit positions in the flags +-- column in status_entries. If the number of known flags ever grows larger than +-- 31, we'll have to extend flag_id to accept values between 0 and 63 and alter +-- the flags column in status_entries from INTEGER to BIGINT. And if it grows +-- even larger, we'll have to do something even smarter. +CREATE TABLE flags ( + flag_id SMALLINT PRIMARY KEY CHECK (flag_id BETWEEN 0 AND 30), + flag_string TEXT UNIQUE NOT NULL +); + +-- Hard-wire the Guard and (Bad)Exit flags, so that we can implement +-- aggregate_ipv6() below without having to look up their IDs in the flags +-- table. +INSERT INTO flags (flag_id, flag_string) VALUES (0, 'Guard'); +INSERT INTO flags (flag_id, flag_string) VALUES (1, 'Exit'); +INSERT INTO flags (flag_id, flag_string) VALUES (2, 'BadExit'); + +-- Table of all known versions, either from platform lines in server descriptors +-- or recommended-server-versions lines in consensuses. Versions found in the +-- latter are marked as recommended, which enables us to only include major +-- versions in results that have been recommended at least once in a consensus. +CREATE TABLE versions ( + version_id SERIAL PRIMARY KEY, + version_string TEXT UNIQUE NOT NULL, + recommended BOOLEAN NOT NULL +); + +-- Hard-wire a 0.1.0 version and a 0.1.1 version, because these major versions +-- were never recommended in a consensus, yet they are supposed to go into the +-- versions statistics. +INSERT INTO versions (version_string, recommended) VALUES ('0.1.0.17', TRUE); +INSERT INTO versions (version_string, recommended) VALUES ('0.1.1.26', TRUE); + +-- Table of all known platforms from platform lines in server descriptors, that +-- is, without Tor software version information. +CREATE TABLE platforms ( + platform_id SERIAL PRIMARY KEY, + platform_string TEXT UNIQUE NOT NULL +); + -- Table of all relevant parts contained in relay or bridge server descriptors. -- We're not deleting from this table, because we can never be sure that we -- won't import a previously missing status that we'll want to match against -- existing server descriptors. CREATE TABLE server_descriptors ( descriptor_digest_sha1 BYTEA PRIMARY KEY, + platform_id INTEGER REFERENCES platforms (platform_id), + version_id INTEGER REFERENCES versions (version_id), advertised_bandwidth_bytes INTEGER NOT NULL, announced_ipv6 BOOLEAN NOT NULL, exiting_ipv6_relay BOOLEAN NOT NULL @@ -21,6 +63,10 @@ CREATE TABLE statuses ( server server_enum NOT NULL, valid_after TIMESTAMP WITHOUT TIME ZONE NOT NULL, running_count INTEGER NOT NULL, + consensus_weight_sum REAL, + guard_weight_sum REAL, + middle_weight_sum REAL, + exit_weight_sum REAL, UNIQUE (server, valid_after) ); @@ -30,9 +76,12 @@ CREATE TABLE statuses ( CREATE TABLE status_entries ( status_id INTEGER REFERENCES statuses (status_id) NOT NULL, descriptor_digest_sha1 BYTEA NOT NULL, - guard_relay BOOLEAN NOT NULL, - exit_relay BOOLEAN NOT NULL, + flags INTEGER NOT NULL, reachable_ipv6_relay BOOLEAN NOT NULL, + consensus_weight REAL, + guard_weight REAL, + middle_weight REAL, + exit_weight REAL, UNIQUE (status_id, descriptor_digest_sha1) ); @@ -47,33 +96,158 @@ CREATE TABLE aggregated_ipv6 ( exiting_ipv6_relay BOOLEAN NOT NULL, reachable_ipv6_relay BOOLEAN NOT NULL, server_count_sum INTEGER NOT NULL, + consensus_weight REAL, + guard_weight REAL, + middle_weight REAL, + exit_weight REAL, advertised_bandwidth_bytes_sum BIGINT NOT NULL, CONSTRAINT aggregated_ipv6_unique UNIQUE (status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, reachable_ipv6_relay) ); +-- Table of joined and aggregated server_descriptors and status_entries rows by +-- relay flag. +CREATE TABLE aggregated_flags ( + status_id INTEGER REFERENCES statuses (status_id) NOT NULL, + flag_id INTEGER REFERENCES flags (flag_id) NOT NULL, + server_count_sum INTEGER NOT NULL, + consensus_weight REAL, + guard_weight REAL, + middle_weight REAL, + exit_weight REAL, + advertised_bandwidth_bytes_sum BIGINT NOT NULL, + CONSTRAINT aggregated_flags_unique UNIQUE (status_id, flag_id) +); + +-- Table of joined and aggregated server_descriptors and status_entries rows by +-- version. +CREATE TABLE aggregated_versions ( + status_id INTEGER REFERENCES statuses (status_id) NOT NULL, + version_id INTEGER REFERENCES versions (version_id), + server_count_sum INTEGER NOT NULL, + consensus_weight REAL, + guard_weight REAL, + middle_weight REAL, + exit_weight REAL, + advertised_bandwidth_bytes_sum BIGINT NOT NULL, + CONSTRAINT aggregated_versions_unique UNIQUE (status_id, version_id) +); + +-- Table of joined and aggregated server_descriptors and status_entries rows by +-- platform. +CREATE TABLE aggregated_platforms ( + status_id INTEGER REFERENCES statuses (status_id) NOT NULL, + platform_id INTEGER REFERENCES platforms (platform_id), + server_count_sum INTEGER NOT NULL, + consensus_weight REAL, + guard_weight REAL, + middle_weight REAL, + exit_weight REAL, + advertised_bandwidth_bytes_sum BIGINT NOT NULL, + CONSTRAINT aggregated_platforms_unique UNIQUE (status_id, platform_id) +); + -- Function to aggregate server_descriptors and status_entries rows into the --- aggregated table and delete rows from status_entries that are then contained --- in the aggregated table. This function is supposed to be called once after --- inserting new rows into server_descriptors and/or status_entries. Subsequent --- calls won't have any effect. -CREATE OR REPLACE FUNCTION aggregate_ipv6() RETURNS VOID AS $$ +-- aggregated_* tables and delete rows from status_entries that are then +-- contained in the aggregated_* tables. This function is supposed to be called +-- once after inserting new rows into server_descriptors and/or status_entries. +-- Subsequent calls won't have any effect. +CREATE OR REPLACE FUNCTION aggregate() RETURNS VOID AS $$ +-- Aggregate by IPv6 support. INSERT INTO aggregated_ipv6 -SELECT status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, +SELECT status_id, flags & 1 > 0 AS guard_relay, + flags & 2 > 0 AND flags & 4 = 0 AS exit_relay, + announced_ipv6, exiting_ipv6_relay, reachable_ipv6_relay, COUNT(*) AS server_count_sum, - SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes + SUM(consensus_weight) AS consensus_weight, + SUM(guard_weight) AS guard_weight, + SUM(middle_weight) AS middle_weight, + SUM(exit_weight) AS exit_weight, + SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes_sum FROM status_entries NATURAL JOIN server_descriptors -NATURAL JOIN statuses GROUP BY status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, reachable_ipv6_relay ON CONFLICT ON CONSTRAINT aggregated_ipv6_unique DO UPDATE SET server_count_sum = aggregated_ipv6.server_count_sum - + EXCLUDED.server_count_sum, + + EXCLUDED.server_count_sum, + consensus_weight = aggregated_ipv6.consensus_weight + + EXCLUDED.consensus_weight, + guard_weight = aggregated_ipv6.guard_weight + EXCLUDED.guard_weight, + middle_weight = aggregated_ipv6.middle_weight + EXCLUDED.middle_weight, + exit_weight = aggregated_ipv6.exit_weight + EXCLUDED.exit_weight, + advertised_bandwidth_bytes_sum + = aggregated_ipv6.advertised_bandwidth_bytes_sum + + EXCLUDED.advertised_bandwidth_bytes_sum; +-- Aggregate by assigned relay flags. +INSERT INTO aggregated_flags +SELECT status_id, flag_id, COUNT(*) AS server_count_sum, + SUM(consensus_weight) AS consensus_weight, + SUM(guard_weight) AS guard_weight, + SUM(middle_weight) AS middle_weight, + SUM(exit_weight) AS exit_weight, + SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes_sum +FROM status_entries +NATURAL JOIN server_descriptors +JOIN flags ON flags & (1 << flag_id) > 0 +GROUP BY status_id, flag_id +ON CONFLICT ON CONSTRAINT aggregated_flags_unique +DO UPDATE SET server_count_sum = aggregated_flags.server_count_sum + + EXCLUDED.server_count_sum, + consensus_weight = aggregated_flags.consensus_weight + + EXCLUDED.consensus_weight, + guard_weight = aggregated_flags.guard_weight + EXCLUDED.guard_weight, + middle_weight = aggregated_flags.middle_weight + EXCLUDED.middle_weight, + exit_weight = aggregated_flags.exit_weight + EXCLUDED.exit_weight, advertised_bandwidth_bytes_sum - = aggregated_ipv6.advertised_bandwidth_bytes_sum - + EXCLUDED.advertised_bandwidth_bytes_sum; + = aggregated_flags.advertised_bandwidth_bytes_sum + + EXCLUDED.advertised_bandwidth_bytes_sum; +-- Aggregate by version. +INSERT INTO aggregated_versions +SELECT status_id, version_id, COUNT(*) AS server_count_sum, + SUM(consensus_weight) AS consensus_weight, + SUM(guard_weight) AS guard_weight, + SUM(middle_weight) AS middle_weight, + SUM(exit_weight) AS exit_weight, + SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes_sum +FROM status_entries +NATURAL JOIN server_descriptors +GROUP BY status_id, version_id +ON CONFLICT ON CONSTRAINT aggregated_versions_unique +DO UPDATE SET server_count_sum = aggregated_versions.server_count_sum + + EXCLUDED.server_count_sum, + consensus_weight = aggregated_versions.consensus_weight + + EXCLUDED.consensus_weight, + guard_weight = aggregated_versions.guard_weight + EXCLUDED.guard_weight, + middle_weight = aggregated_versions.middle_weight + EXCLUDED.middle_weight, + exit_weight = aggregated_versions.exit_weight + EXCLUDED.exit_weight, + advertised_bandwidth_bytes_sum + = aggregated_versions.advertised_bandwidth_bytes_sum + + EXCLUDED.advertised_bandwidth_bytes_sum; +-- Aggregate by platform. +INSERT INTO aggregated_platforms +SELECT status_id, platform_id, COUNT(*) AS server_count_sum, + SUM(consensus_weight) AS consensus_weight, + SUM(guard_weight) AS guard_weight, + SUM(middle_weight) AS middle_weight, + SUM(exit_weight) AS exit_weight, + SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes_sum +FROM status_entries +NATURAL JOIN server_descriptors +GROUP BY status_id, platform_id +ON CONFLICT ON CONSTRAINT aggregated_platforms_unique +DO UPDATE SET server_count_sum = aggregated_platforms.server_count_sum + + EXCLUDED.server_count_sum, + consensus_weight = aggregated_platforms.consensus_weight + + EXCLUDED.consensus_weight, + guard_weight = aggregated_platforms.guard_weight + EXCLUDED.guard_weight, + middle_weight = aggregated_platforms.middle_weight + EXCLUDED.middle_weight, + exit_weight = aggregated_platforms.exit_weight + EXCLUDED.exit_weight, + advertised_bandwidth_bytes_sum + = aggregated_platforms.advertised_bandwidth_bytes_sum + + EXCLUDED.advertised_bandwidth_bytes_sum; +-- Delete obsolete rows from the status_entries table. DELETE FROM status_entries WHERE EXISTS ( SELECT 1 FROM server_descriptors WHERE descriptor_digest_sha1 = status_entries.descriptor_digest_sha1); @@ -100,24 +274,210 @@ WITH included_statuses AS ( HAVING COUNT(status_id) >= 12 AND DATE(valid_after) < (SELECT MAX(DATE(valid_after)) FROM included_statuses) +), grouped_by_status AS ( + SELECT valid_after, server, + CASE WHEN server = 'relay' THEN guard_relay ELSE NULL END + AS guard_relay_or_null, + CASE WHEN server = 'relay' THEN exit_relay ELSE NULL END + AS exit_relay_or_null, + announced_ipv6, + CASE WHEN server = 'relay' THEN exiting_ipv6_relay ELSE NULL END + AS exiting_ipv6_relay_or_null, + CASE WHEN server = 'relay' THEN reachable_ipv6_relay ELSE NULL END + AS reachable_ipv6_relay_or_null, + SUM(server_count_sum) AS server_count_sum, + CASE WHEN server = 'relay' THEN SUM(advertised_bandwidth_bytes_sum) + ELSE NULL END AS advertised_bandwidth_bytes_sum + FROM statuses NATURAL JOIN aggregated_ipv6 + WHERE status_id IN (SELECT status_id FROM included_statuses) + AND DATE(valid_after) IN ( + SELECT valid_after_date FROM included_dates + WHERE included_dates.server = statuses.server) + GROUP BY status_id, valid_after, server, guard_relay_or_null, + exit_relay_or_null, announced_ipv6, exiting_ipv6_relay_or_null, + reachable_ipv6_relay_or_null ) SELECT DATE(valid_after) AS valid_after_date, server, - CASE WHEN server = 'relay' THEN guard_relay ELSE NULL END AS guard_relay, - CASE WHEN server = 'relay' THEN exit_relay ELSE NULL END AS exit_relay, + guard_relay_or_null AS guard_relay, + exit_relay_or_null AS exit_relay, announced_ipv6, - CASE WHEN server = 'relay' THEN exiting_ipv6_relay ELSE NULL END - AS exiting_ipv6_relay, - CASE WHEN server = 'relay' THEN reachable_ipv6_relay ELSE NULL END - AS reachable_ipv6_relay, + exiting_ipv6_relay_or_null AS exiting_ipv6_relay, + reachable_ipv6_relay_or_null AS reachable_ipv6_relay, FLOOR(AVG(server_count_sum)) AS server_count_sum_avg, - CASE WHEN server = 'relay' THEN FLOOR(AVG(advertised_bandwidth_bytes_sum)) - ELSE NULL END AS advertised_bandwidth_bytes_sum_avg -FROM statuses NATURAL JOIN aggregated_ipv6 -WHERE status_id IN (SELECT status_id FROM included_statuses) -AND DATE(valid_after) IN ( - SELECT valid_after_date FROM included_dates WHERE server = statuses.server) + FLOOR(AVG(advertised_bandwidth_bytes_sum)) + AS advertised_bandwidth_bytes_sum_avg +FROM grouped_by_status GROUP BY DATE(valid_after), server, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, reachable_ipv6_relay ORDER BY valid_after_date, server, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, reachable_ipv6_relay; +-- View on the number of running servers by relay flag. +CREATE OR REPLACE VIEW servers_flags_complete AS +WITH included_statuses AS ( + SELECT status_id, server, valid_after + FROM statuses NATURAL JOIN aggregated_flags + GROUP BY status_id + HAVING running_count > 0 + AND 1000 * SUM(server_count_sum) > 999 * running_count +), included_dates AS ( + SELECT DATE(valid_after) AS valid_after_date, server + FROM included_statuses NATURAL JOIN aggregated_flags + GROUP BY DATE(valid_after), server + HAVING COUNT(status_id) >= 12 + AND DATE(valid_after) + < (SELECT MAX(DATE(valid_after)) FROM included_statuses) +) +SELECT DATE(valid_after) AS valid_after_date, server, flag_string AS flag, + FLOOR(AVG(server_count_sum)) AS server_count_sum_avg, + AVG(consensus_weight / consensus_weight_sum) AS consensus_weight_fraction, + AVG(guard_weight / guard_weight_sum) AS guard_weight_fraction, + AVG(middle_weight / middle_weight_sum) AS middle_weight_fraction, + AVG(exit_weight / exit_weight_sum) AS exit_weight_fraction +FROM statuses NATURAL JOIN aggregated_flags NATURAL JOIN flags +WHERE status_id IN (SELECT status_id FROM included_statuses) +AND DATE(valid_after) IN ( + SELECT valid_after_date FROM included_dates + WHERE included_dates.server = statuses.server) +GROUP BY DATE(valid_after), server, flag +ORDER BY valid_after_date, server, flag; + +-- View on the number of running relays and bridges. +CREATE OR REPLACE VIEW servers_networksize AS +SELECT valid_after_date AS date, + FLOOR(AVG(CASE WHEN server = 'relay' THEN server_count_sum_avg + ELSE NULL END)) AS relays, + FLOOR(AVG(CASE WHEN server = 'bridge' THEN server_count_sum_avg + ELSE NULL END)) AS bridges +FROM servers_flags_complete +WHERE flag = 'Running' +GROUP BY date +ORDER BY date; + +-- View on the number of running relays by relay flag. +CREATE OR REPLACE VIEW servers_relayflags AS +SELECT valid_after_date AS date, flag, server_count_sum_avg AS relays +FROM servers_flags_complete +WHERE server = 'relay' +AND flag IN ('Running', 'Exit', 'Fast', 'Guard', 'Stable', 'HSDir') +ORDER BY date, flag; + +-- View on the number of running servers by version. +CREATE OR REPLACE VIEW servers_versions_complete AS +WITH included_statuses AS ( + SELECT status_id, server, valid_after + FROM statuses NATURAL JOIN aggregated_versions + GROUP BY status_id, server, valid_after + HAVING running_count > 0 + AND 1000 * SUM(server_count_sum) > 999 * running_count +), included_dates AS ( + SELECT DATE(valid_after) AS valid_after_date, server + FROM included_statuses + GROUP BY DATE(valid_after), server + HAVING COUNT(status_id) >= 12 + AND DATE(valid_after) + < (SELECT MAX(DATE(valid_after)) FROM included_statuses) +), included_versions AS ( + SELECT SUBSTRING(version_string FROM '^([^\.]+\.[^\.]+\.[^\.]+)') AS version + FROM versions + WHERE recommended IS TRUE + GROUP BY version +), grouped_by_version AS ( + SELECT server, valid_after, + CASE WHEN SUBSTRING(version_string FROM '^([^\.]+\.[^\.]+\.[^\.]+)') + IN (SELECT version FROM included_versions) + THEN SUBSTRING(version_string FROM '^([^\.]+\.[^\.]+\.[^\.]+)') + ELSE 'Other' END AS version, + SUM(server_count_sum) AS server_count_sum, + SUM(consensus_weight) AS consensus_weight, + SUM(guard_weight) AS guard_weight, + SUM(middle_weight) AS middle_weight, + SUM(exit_weight) AS exit_weight, + consensus_weight_sum, + guard_weight_sum, + middle_weight_sum, + exit_weight_sum + FROM statuses NATURAL JOIN aggregated_versions LEFT JOIN versions + ON aggregated_versions.version_id = versions.version_id + WHERE status_id IN (SELECT status_id FROM included_statuses) + AND DATE(valid_after) IN ( + SELECT valid_after_date FROM included_dates + WHERE included_dates.server = statuses.server) + GROUP BY status_id, server, valid_after, version +) +SELECT DATE(valid_after) AS valid_after_date, server, version, + FLOOR(AVG(server_count_sum)) AS server_count_sum_avg, + AVG(consensus_weight / consensus_weight_sum) AS consensus_weight_fraction, + AVG(guard_weight / guard_weight_sum) AS guard_weight_fraction, + AVG(middle_weight / middle_weight_sum) AS middle_weight_fraction, + AVG(exit_weight / exit_weight_sum) AS exit_weight_fraction +FROM grouped_by_version +GROUP BY DATE(valid_after), server, version +ORDER BY valid_after_date, server, version; + +-- View on the number of running relays by version. +CREATE OR REPLACE VIEW servers_versions AS +SELECT valid_after_date AS date, version, server_count_sum_avg AS relays +FROM servers_versions_complete +WHERE server = 'relay' +ORDER BY date, version; + +-- View on the number of running servers by platform. +CREATE OR REPLACE VIEW servers_platforms_complete AS +WITH included_statuses AS ( + SELECT status_id, server, valid_after + FROM statuses NATURAL JOIN aggregated_platforms + GROUP BY status_id, server, valid_after + HAVING running_count > 0 + AND 1000 * SUM(server_count_sum) > 999 * running_count +), included_dates AS ( + SELECT DATE(valid_after) AS valid_after_date, server + FROM included_statuses + GROUP BY DATE(valid_after), server + HAVING COUNT(status_id) >= 12 + AND DATE(valid_after) + < (SELECT MAX(DATE(valid_after)) FROM included_statuses) +), grouped_by_platform AS ( + SELECT server, valid_after, + CASE WHEN platform_string LIKE 'Linux%' THEN 'Linux' + WHEN platform_string LIKE 'Windows%' THEN 'Windows' + WHEN platform_string LIKE 'Darwin%' THEN 'macOS' + WHEN platform_string LIKE '%BSD%' + -- Uncomment, if we ever want to count DragonFly as BSD + -- OR platform_string = 'DragonFly%' + THEN 'BSD' + ELSE 'Other' END AS platform, + SUM(server_count_sum) AS server_count_sum, + SUM(consensus_weight) AS consensus_weight, + SUM(guard_weight) AS guard_weight, + SUM(middle_weight) AS middle_weight, + SUM(exit_weight) AS exit_weight, + consensus_weight_sum, + guard_weight_sum, + middle_weight_sum, + exit_weight_sum + FROM statuses NATURAL JOIN aggregated_platforms LEFT JOIN platforms + ON aggregated_platforms.platform_id = platforms.platform_id + WHERE status_id IN (SELECT status_id FROM included_statuses) + AND DATE(valid_after) IN ( + SELECT valid_after_date FROM included_dates + WHERE included_dates.server = statuses.server) + GROUP BY status_id, server, valid_after, platform +) +SELECT DATE(valid_after) AS valid_after_date, server, platform, + FLOOR(AVG(server_count_sum)) AS server_count_sum_avg, + AVG(consensus_weight / consensus_weight_sum) AS consensus_weight_fraction, + AVG(guard_weight / guard_weight_sum) AS guard_weight_fraction, + AVG(middle_weight / middle_weight_sum) AS middle_weight_fraction, + AVG(exit_weight / exit_weight_sum) AS exit_weight_fraction +FROM grouped_by_platform +GROUP BY DATE(valid_after), server, platform +ORDER BY valid_after_date, server, platform; + +-- View on the number of running relays by platform. +CREATE OR REPLACE VIEW servers_platforms AS +SELECT valid_after_date AS date, platform, server_count_sum_avg AS relays +FROM servers_platforms_complete +WHERE server = 'relay' +ORDER BY date, platform; +

1 0

[metrics-web/release] Modernize legacy module and rename it to bwhist.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit f8fa108d183968540eca529250cb142f8216ce8c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Nov 14 10:39:24 2018 +0100 Modernize legacy module and rename it to bwhist. Changes include using similar mechanisms for configuration, calling the database aggregation function, querying the database, and writing results as we're using in the ipv6servers and other modules. Configuration options can now be changed via the following Java properties: bwhist.descriptors bwhist.database bwhist.history bwhist.output The legacy.config file, if one exists, will be ignored. Part of #28116. --- build.xml | 38 +----- .../metrics/stats/bwhist/Configuration.java | 18 +++ .../org/torproject/metrics/stats/bwhist/Main.java | 56 +++++++++ .../RelayDescriptorDatabaseImporter.java | 131 +++++++++++++++------ .../torproject/metrics/stats/bwhist/Writer.java | 42 +++++++ .../metrics/stats/servers/Configuration.java | 87 -------------- .../org/torproject/metrics/stats/servers/Main.java | 40 ------- src/main/resources/legacy.config.template | 8 -- 8 files changed, 212 insertions(+), 208 deletions(-) diff --git a/build.xml b/build.xml index b95550d..a391416 100644 --- a/build.xml +++ b/build.xml @@ -315,7 +315,7 @@ <antcall target="collectdescs" /> <antcall target="connbidirect" /> <antcall target="onionperf" /> - <antcall target="legacy" /> + <antcall target="bwhist" /> <antcall target="advbwdist" /> <antcall target="hidserv" /> <antcall target="clients" /> @@ -340,39 +340,9 @@ <antcall target="run-java" /> </target> -  - <target name="legacy-create-config" > - <copy file="${resources}/legacy.config.template" - tofile="${basedir}/legacy.config"/> - </target> - -  - <target name="legacy" > - <property name="module.name" value="servers" /> - <property name="localmoddir" value="${modulebase}/${module.name}" /> - <property name="statsdir" - value="${localmoddir}/stats" /> - <mkdir dir="${statsdir}" /> - - <copy file="${basedir}/legacy.config" - tofile="${localmoddir}/config"/> - + <target name="bwhist" > + <property name="module.name" value="bwhist" /> <antcall target="run-java" /> - - <exec executable="psql" - dir="${localmoddir}" - failonerror="true" > - <arg value="--dbname=tordir"/> - <arg value="-c SELECT * FROM refresh_all();" /> - </exec> - - <exec executable="psql" - dir="${localmoddir}" - failonerror="true" > - <arg value="-c COPY (SELECT * FROM stats_bandwidth) TO STDOUT WITH CSV HEADER;" /> - <arg value="--dbname=tordir"/> - <arg value="--output=${statsdir}/bandwidth.csv" /> - </exec> </target> <target name="advbwdist"> @@ -503,7 +473,7 @@ <fileset dir="${modulebase}/onionperf/stats" includes="*.csv" /> <fileset dir="${modulebase}/connbidirect/stats" includes="connbidirect2.csv" /> <fileset dir="${modulebase}/advbwdist/stats" includes="advbwdist.csv" /> - <fileset dir="${modulebase}/servers/stats" includes="*.csv" /> + <fileset dir="${modulebase}/bwhist/stats" includes="*.csv" /> <fileset dir="${modulebase}/hidserv/stats" includes="hidserv.csv" /> <fileset dir="${modulebase}/clients/stats" includes="clients*.csv userstats-combined.csv" /> diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java b/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java new file mode 100644 index 0000000..2a0fbc5 --- /dev/null +++ b/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java @@ -0,0 +1,18 @@ +/* Copyright 2011--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.bwhist; + +/** Configuration options parsed from Java properties with reasonable hard-coded + * defaults. */ +public class Configuration { + static String descriptors = System.getProperty("bwhist.descriptors", + "../../shared/in/"); + static String database = System.getProperty("bwhist.database", + "jdbc:postgresql:tordir"); + static String history = System.getProperty("bwhist.history", + "status/read-descriptors"); + static String output = System.getProperty("bwhist.output", + "stats/"); +} + diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java new file mode 100644 index 0000000..61c1435 --- /dev/null +++ b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java @@ -0,0 +1,56 @@ +/* Copyright 2011--2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.bwhist; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Arrays; + +/** + * Coordinate downloading and parsing of descriptors and extraction of + * statistically relevant data for later processing with R. + */ +public class Main { + + private static Logger log = LoggerFactory.getLogger(Main.class); + + private static String[][] paths = { + {"recent", "relay-descriptors", "consensuses"}, + {"recent", "relay-descriptors", "extra-infos"}, + {"archive", "relay-descriptors", "consensuses"}, + {"archive", "relay-descriptors", "extra-infos"}}; + + /** Executes this data-processing module. */ + public static void main(String[] args) throws Exception { + + log.info("Starting bwhist module."); + + log.info("Reading descriptors and inserting relevant parts into the " + + "database."); + File[] descriptorDirectories = Arrays.stream(paths).map((String[] path) + -> Paths.get(Configuration.descriptors, path).toFile()) + .toArray(File[]::new); + File historyFile = new File(Configuration.history); + RelayDescriptorDatabaseImporter database + = new RelayDescriptorDatabaseImporter(descriptorDirectories, + historyFile, Configuration.database); + database.importRelayDescriptors(); + + log.info("Aggregating database entries."); + database.aggregate(); + + log.info("Querying aggregated statistics from the database."); + new Writer().write(Paths.get(Configuration.output, "bandwidth.csv"), + database.queryBandwidth()); + + log.info("Closing database connection."); + database.closeConnection(); + + log.info("Terminating bwhist module."); + } +} + diff --git a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java similarity index 84% rename from src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java rename to src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java index d1ae43c..a6cf0cc 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java @@ -1,7 +1,7 @@ /* Copyright 2011--2018 The Tor Project * See LICENSE for licensing information */ -package org.torproject.metrics.stats.servers; +package org.torproject.metrics.stats.bwhist; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorReader; @@ -20,6 +20,7 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -27,6 +28,7 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -108,22 +110,19 @@ public final class RelayDescriptorDatabaseImporter { private boolean importIntoDatabase = true; - private List<File> archivesDirectories; + private File[] descriptorDirectories; - private File statsDirectory; + private File historyFile; /** * Initialize database importer by connecting to the database and * preparing statements. */ - public RelayDescriptorDatabaseImporter(String connectionUrl, - List<File> archivesDirectories, File statsDirectory) { + public RelayDescriptorDatabaseImporter(File[] descriptorDirectories, + File historyFile, String connectionUrl) { - if (archivesDirectories == null || statsDirectory == null) { - throw new IllegalArgumentException(); - } - this.archivesDirectories = archivesDirectories; - this.statsDirectory = statsDirectory; + this.descriptorDirectories = descriptorDirectories; + this.historyFile = historyFile; if (connectionUrl != null) { try { @@ -520,29 +519,20 @@ public final class RelayDescriptorDatabaseImporter { /** Imports relay descriptors into the database. */ public void importRelayDescriptors() { - log.info("Importing files in directories " + archivesDirectories - + "/..."); - if (!this.archivesDirectories.isEmpty()) { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.setMaxDescriptorsInQueue(10); - File historyFile = new File(statsDirectory, - "database-importer-relay-descriptor-history"); - reader.setHistoryFile(historyFile); - for (Descriptor descriptor : reader.readDescriptors( - this.archivesDirectories.toArray( - new File[this.archivesDirectories.size()]))) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - this.addRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof ExtraInfoDescriptor) { - this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); - } + DescriptorReader reader = + DescriptorSourceFactory.createDescriptorReader(); + reader.setMaxDescriptorsInQueue(10); + reader.setHistoryFile(this.historyFile); + for (Descriptor descriptor : reader.readDescriptors( + this.descriptorDirectories)) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.addRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof ExtraInfoDescriptor) { + this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); } - reader.saveHistoryFile(historyFile); } - - log.info("Finished importing relay descriptors."); + reader.saveHistoryFile(this.historyFile); } private void addRelayNetworkStatusConsensus( @@ -583,9 +573,9 @@ public final class RelayDescriptorDatabaseImporter { } /** - * Close the relay descriptor database connection. + * Commit any non-commited parts. */ - public void closeConnection() { + public void commit() { /* Log stats about imported descriptors. */ log.info("Finished importing relay descriptors: {} network status entries " @@ -609,21 +599,84 @@ public final class RelayDescriptorDatabaseImporter { } } - /* Commit any stragglers before closing. */ + /* Commit any stragglers. */ if (this.conn != null) { try { this.csH.executeBatch(); this.conn.commit(); - } catch (SQLException e) { + } catch (SQLException e) { log.warn("Could not commit final records to database", e); } - try { - this.conn.close(); - } catch (SQLException e) { - log.warn("Could not close database connection.", e); + } + } + + /** Call the refresh_all() function to aggregate newly imported data. */ + void aggregate() throws SQLException { + Statement st = this.conn.createStatement(); + st.executeQuery("SELECT refresh_all()"); + } + + /** Query the servers_platforms view. */ + List<String[]> queryBandwidth() throws SQLException { + List<String[]> statistics = new ArrayList<>(); + String columns = "date, isexit, isguard, bwread, bwwrite, dirread, " + + "dirwrite"; + statistics.add(columns.split(", ")); + Statement st = this.conn.createStatement(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + String queryString = "SELECT " + columns + " FROM stats_bandwidth"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + String[] outputLine = new String[7]; + outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[1] = getBooleanFromResultSet(rs, "isexit"); + outputLine[2] = getBooleanFromResultSet(rs, "isguard"); + outputLine[3] = getLongFromResultSet(rs, "bwread"); + outputLine[4] = getLongFromResultSet(rs, "bwwrite"); + outputLine[5] = getLongFromResultSet(rs, "dirread"); + outputLine[6] = getLongFromResultSet(rs, "dirwrite"); + statistics.add(outputLine); } } + return statistics; + } + + /** Retrieve the <code>boolean</code> value of the designated column in the + * current row of the given <code>ResultSet</code> object and format it as a + * <code>String</code> object with <code>"t"</code> for <code>true</code> and + * <code>"f"</code> for <code>false</code>, or return <code>null</code> if the + * retrieved value was <code>NULL</code>. */ + private static String getBooleanFromResultSet(ResultSet rs, + String columnLabel) throws SQLException { + boolean result = rs.getBoolean(columnLabel); + if (rs.wasNull()) { + return null; + } else { + return result ? "t" : "f"; + } + } + + /** Retrieve the <code>long</code> value of the designated column in the + * current row of the given <code>ResultSet</code> object and format it as a + * <code>String</code> object, or return <code>null</code> if the retrieved + * value was <code>NULL</code>. */ + private static String getLongFromResultSet(ResultSet rs, String columnLabel) + throws SQLException { + long result = rs.getLong(columnLabel); + return rs.wasNull() ? null : String.valueOf(result); + } + + /** + * Close the relay descriptor database connection. + */ + public void closeConnection() { + try { + this.conn.close(); + } catch (SQLException e) { + log.warn("Could not close database connection.", e); + } } } diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Writer.java b/src/main/java/org/torproject/metrics/stats/bwhist/Writer.java new file mode 100644 index 0000000..1ac1fd9 --- /dev/null +++ b/src/main/java/org/torproject/metrics/stats/bwhist/Writer.java @@ -0,0 +1,42 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.bwhist; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** Writer that takes output line objects and writes them to a file, preceded + * by a column header line. */ +class Writer { + + /** Write output lines to the given file. */ + void write(Path filePath, Iterable<String[]> outputLines) + throws IOException { + File parentFile = filePath.toFile().getParentFile(); + if (null != parentFile && !parentFile.exists()) { + if (!parentFile.mkdirs()) { + throw new IOException("Unable to create parent directory of output " + + "file. Not writing this file."); + } + } + List<String> formattedOutputLines = new ArrayList<>(); + for (String[] outputLine : outputLines) { + StringBuilder formattedOutputLine = new StringBuilder(); + for (String outputLinePart : outputLine) { + formattedOutputLine.append(','); + if (null != outputLinePart) { + formattedOutputLine.append(outputLinePart); + } + } + formattedOutputLines.add(formattedOutputLine.substring(1)); + } + Files.write(filePath, formattedOutputLines, StandardCharsets.UTF_8); + } +} + diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java deleted file mode 100644 index b6ee397..0000000 --- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java +++ /dev/null @@ -1,87 +0,0 @@ -/* Copyright 2011--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.servers; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Initialize configuration with hard-coded defaults, overwrite with - * configuration in config file, if exists, and answer Main.java about our - * configuration. - */ -public class Configuration { - - private static Logger log = LoggerFactory.getLogger(Configuration.class); - - private List<File> directoryArchivesDirectories = new ArrayList<>(); - - private String relayDescriptorDatabaseJdbc = - "jdbc:postgresql://localhost/tordir?user=metrics&password=password"; - - /** Initializes this configuration class. */ - public Configuration() { - - /* Read config file, if present. */ - File configFile = new File("config"); - if (!configFile.exists()) { - log.warn("Could not find config file."); - return; - } - String line = null; - try (BufferedReader br = new BufferedReader(new FileReader(configFile))) { - while ((line = br.readLine()) != null) { - if (line.startsWith("DirectoryArchivesDirectory")) { - this.directoryArchivesDirectories.add(new File(line.split(" ")[1])); - } else if (line.startsWith("RelayDescriptorDatabaseJDBC")) { - this.relayDescriptorDatabaseJdbc = line.split(" ")[1]; - } else if (!line.startsWith("#") && line.length() > 0) { - log.error("Configuration file contains unrecognized " - + "configuration key in line '{}'! Exiting!", line); - System.exit(1); - } - } - } catch (ArrayIndexOutOfBoundsException e) { - log.warn("Configuration file contains configuration key without value in " - + "line '{}'. Exiting!", line); - System.exit(1); - } catch (MalformedURLException e) { - log.warn("Configuration file contains illegal URL or IP:port pair in " - + "line '{}'. Exiting!", line); - System.exit(1); - } catch (NumberFormatException e) { - log.warn("Configuration file contains illegal value in line '{}' with " - + "legal values being 0 or 1. Exiting!", line); - System.exit(1); - } catch (IOException e) { - log.error("Unknown problem while reading config file! Exiting!", e); - System.exit(1); - } - } - - /** Returns directories containing archived descriptors. */ - public List<File> getDirectoryArchivesDirectories() { - if (this.directoryArchivesDirectories.isEmpty()) { - String prefix = "../../shared/in/recent/relay-descriptors/"; - return Arrays.asList(new File(prefix + "consensuses/"), - new File(prefix + "extra-infos/")); - } else { - return this.directoryArchivesDirectories; - } - } - - public String getRelayDescriptorDatabaseJdbc() { - return this.relayDescriptorDatabaseJdbc; - } -} - diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java deleted file mode 100644 index 1454418..0000000 --- a/src/main/java/org/torproject/metrics/stats/servers/Main.java +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright 2011--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.servers; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; - -/** - * Coordinate downloading and parsing of descriptors and extraction of - * statistically relevant data for later processing with R. - */ -public class Main { - - private static Logger log = LoggerFactory.getLogger(Main.class); - - /** Executes this data-processing module. */ - public static void main(String[] args) { - - log.info("Starting ERNIE."); - - // Initialize configuration - Configuration config = new Configuration(); - - // Define stats directory for temporary files - File statsDirectory = new File("stats"); - - // Import relay descriptors - RelayDescriptorDatabaseImporter rddi = new RelayDescriptorDatabaseImporter( - config.getRelayDescriptorDatabaseJdbc(), - config.getDirectoryArchivesDirectories(), statsDirectory); - rddi.importRelayDescriptors(); - rddi.closeConnection(); - - log.info("Terminating ERNIE."); - } -} - diff --git a/src/main/resources/legacy.config.template b/src/main/resources/legacy.config.template deleted file mode 100644 index e2e0dac..0000000 --- a/src/main/resources/legacy.config.template +++ /dev/null @@ -1,8 +0,0 @@ -## Relative paths to directories to import directory archives from -#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/ -#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/ -#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/ -# -## JDBC string for relay descriptor database -#RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=metrics&password=password -#

1 0

[metrics-web/release] Take out "We're hiring" link from start page.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 54930cb95f685f7a42610edd4ae2a39f2add7116 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 19 21:05:01 2018 +0100 Take out "We're hiring" link from start page. --- src/main/resources/web/jsps/index.jsp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/resources/web/jsps/index.jsp b/src/main/resources/web/jsps/index.jsp index fbccd7d..3fa49b8 100644 --- a/src/main/resources/web/jsps/index.jsp +++ b/src/main/resources/web/jsps/index.jsp @@ -6,11 +6,6 @@ </jsp:include> <div class="container"> - - <div class="col-12 alert alert-success" role="alert"> - <strong>We're hiring</strong> – The Tor Project is seeking an experienced Data Architect to take our metrics work to the next level. <a href="https://www.torproject.org/about/jobs-metrics-data-architect.html.en" target="_blank">Read the job posting.</a> - </div> - <div class="jumbotron"> <h1>Welcome to Tor Metrics!</h1> <div class="row">

1 0

[metrics-web/release] Tweak Advertised and consumed bandwidth by relay flag graph.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 6b5f75996ad0d9ac5151da48f9693d478dd682de Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 5 21:32:55 2018 +0100 Tweak Advertised and consumed bandwidth by relay flag graph. This graph now contains everything that's contained in the Total relay bandwidth and the Consumed bandwidth by Exit/Guard flag combination graph. Removing those graphs will be done in a separate commit. Part of #28353. --- src/main/R/rserver/graphs.R | 40 ++++++++++++++------------------ src/main/resources/web/json/metrics.json | 4 ++-- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index 1f7309b..cb40d52 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -788,41 +788,37 @@ write_connbidirect <- function(start_p = NULL, end_p = NULL, path_p) { prepare_bandwidth_flags <- function(start_p, end_p) { advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""), colClasses = c("date" = "Date")) %>% - transmute(date, isguard, isexit, variable = "advbw", - value = advbw * 8 / 1e9) + transmute(date, have_guard_flag = isguard, have_exit_flag = isexit, + variable = "advbw", value = advbw * 8 / 1e9) bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) %>% - transmute(date, isguard, isexit, variable = "bwhist", - value = (bwread + bwwrite) * 8 / 2e9) + transmute(date, have_guard_flag = isguard, have_exit_flag = isexit, + variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9) rbind(advbw, bwhist) %>% filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - group_by(date, variable) %>% - summarize(exit = sum(value[isexit == "t"]), - guard = sum(value[isguard == "t"])) %>% - gather(flag, value, -date, -variable) %>% - unite(variable, flag, variable) %>% - mutate(variable = factor(variable, - levels = c("guard_advbw", "guard_bwhist", "exit_advbw", "exit_bwhist"))) + filter(have_exit_flag != "") %>% + filter(have_guard_flag != "") } plot_bandwidth_flags <- function(start_p, end_p, path_p) { prepare_bandwidth_flags(start_p, end_p) %>% - complete(date = full_seq(date, period = 1), - variable = unique(variable)) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + + unite(flags, have_guard_flag, have_exit_flag) %>% + mutate(flags = factor(flags, levels = c("f_t", "t_t", "t_f", "f_f"), + labels = c("Exit only", "Guard and Exit", "Guard only", + "Neither Guard nor Exit"))) %>% + mutate(variable = ifelse(variable == "advbw", + "Advertised bandwidth", "Consumed bandwidth")) %>% + ggplot(aes(x = date, y = value, fill = flags)) + + geom_area() + scale_x_date(name = "", breaks = custom_breaks, labels = custom_labels, minor_breaks = custom_minor_breaks) + scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), limits = c(0, NA)) + - scale_colour_manual(name = "", - breaks = c("guard_advbw", "guard_bwhist", "exit_advbw", "exit_bwhist"), - labels = c("Guard, advertised bandwidth", "Guard, bandwidth history", - "Exit, advertised bandwidth", "Exit, bandwidth history"), - values = c("#E69F00", "#D6C827", "#009E73", "#00C34F")) + - ggtitle(paste("Advertised bandwidth and bandwidth history by", - "relay flags")) + + scale_fill_manual(name = "", + values = c("#03B3FF", "#39FF02", "#FFFF00", "#AAAA99")) + + facet_grid(variable ~ .) + + ggtitle("Advertised and consumed bandwidth by relay flags") + labs(caption = copyright_notice) + theme(legend.position = "top") ggsave(filename = path_p, width = 8, height = 5, dpi = 150) diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index b351814..5173ae4 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -90,9 +90,9 @@ }, { "id": "bandwidth-flags", - "title": "Advertised and consumed bandwidth by relay flag", + "title": "Advertised and consumed bandwidth by relay flags", "type": "Graph", - "description": "<p>This graph shows <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of relays with either \"Exit\" or \"Guard\" <a href=\"glossary.html#relay-flag\">flags</a> assigned by the directory authorities. These sets are not distinct, because a relay that has both the \"Exit\" and \"Guard\" flags assigned will be included in both sets.</p>", + "description": "<p>This graph shows <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of relays with \"Exit\" and/or \"Guard\" <a href=\"glossary.html#relay-flag\">flags</a> assigned by the directory authorities.</p>", "function": "bandwidth_flags", "parameters": [ "start",

1 0

[metrics-web/release] Remove bandwidth and bwhist-flags graphs.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 59b800a491f93d08bcec6bda1b393a762156226d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 12 12:08:52 2018 +0100 Remove bandwidth and bwhist-flags graphs. The "Advertised and consumed bandwidth by relay flags" graph now contains everything that's contained in the "Total relay bandwidth" and the "Consumed bandwidth by Exit/Guard flag combination" graphs. Removing these two graphs as obsolete. Also update documentation for the newly deployed "Advertised and consumed bandwidth by relay flags" graph. Part of #28353. --- src/main/R/rserver/graphs.R | 79 ---------------------- src/main/resources/web.xml | 8 --- src/main/resources/web/json/categories.json | 2 - src/main/resources/web/json/metrics.json | 22 ------ .../resources/web/jsps/reproducible-metrics.jsp | 17 ++--- src/main/resources/web/jsps/stats.jsp | 68 ++----------------- 6 files changed, 13 insertions(+), 183 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index cb40d52..1ca9357 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -448,85 +448,6 @@ write_platforms <- function(start_p = NULL, end_p = NULL, path_p) { write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") } -prepare_bandwidth <- function(start_p, end_p) { - advbw <- read.csv(paste(stats_dir, "advbw.csv", sep = ""), - colClasses = c("date" = "Date")) %>% - transmute(date, variable = "advbw", value = advbw * 8 / 1e9) - bwhist <- read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), - colClasses = c("date" = "Date")) %>% - transmute(date, variable = "bwhist", value = (bwread + bwwrite) * 8 / 2e9) - rbind(advbw, bwhist) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(!is.na(value)) %>% - group_by(date, variable) %>% - summarize(value = sum(value)) %>% - spread(variable, value) -} - -plot_bandwidth <- function(start_p, end_p, path_p) { - prepare_bandwidth(start_p, end_p) %>% - gather(variable, value, -date) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_hue(name = "", h.start = 90, - breaks = c("advbw", "bwhist"), - labels = c("Advertised bandwidth", "Bandwidth history")) + - ggtitle("Total relay bandwidth") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -write_bandwidth <- function(start_p = NULL, end_p = NULL, path_p) { - prepare_bandwidth(start_p, end_p) %>% - write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") -} - -prepare_bwhist_flags <- function(start_p, end_p) { - read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), - colClasses = c("date" = "Date")) %>% - filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>% - filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>% - filter(isexit != "") %>% - filter(isguard != "") %>% - mutate(variable = ifelse(isexit == "t", - ifelse(isguard == "t", "guard_and_exit", "exit_only"), - ifelse(isguard == "t", "guard_only", "middle_only")), - value = (bwread + bwwrite) * 8 / 2e9) %>% - select(date, variable, value) -} - -plot_bwhist_flags <- function(start_p, end_p, path_p) { - prepare_bwhist_flags(start_p, end_p) %>% - complete(date = full_seq(date, period = 1), - variable = unique(variable)) %>% - ggplot(aes(x = date, y = value, colour = variable)) + - geom_line() + - scale_x_date(name = "", breaks = custom_breaks, - labels = custom_labels, minor_breaks = custom_minor_breaks) + - scale_y_continuous(name = "", labels = unit_format(unit = "Gbit/s"), - limits = c(0, NA)) + - scale_colour_manual(name = "", - breaks = c("exit_only", "guard_and_exit", "guard_only", "middle_only"), - labels = c("Exit only", "Guard & Exit", "Guard only", "Middle only"), - values = c("#E69F00", "#56B4E9", "#009E73", "#0072B2")) + - ggtitle("Bandwidth history by relay flags") + - labs(caption = copyright_notice) + - theme(legend.position = "top") - ggsave(filename = path_p, width = 8, height = 5, dpi = 150) -} - -write_bwhist_flags <- function(start_p = NULL, end_p = NULL, path_p) { - prepare_bwhist_flags(start_p, end_p) %>% - spread(variable, value) %>% - write.csv(path_p, quote = FALSE, row.names = FALSE, na = "") -} - prepare_dirbytes <- function(start_p, end_p, path_p) { read.csv(paste(stats_dir, "bandwidth.csv", sep = ""), colClasses = c("date" = "Date")) %>% diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index 6ff82ad..b643b89 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -30,8 +30,6 @@ <url-pattern>/relayflags.html</url-pattern> <url-pattern>/versions.html</url-pattern> <url-pattern>/platforms.html</url-pattern> - <url-pattern>/bandwidth.html</url-pattern> - <url-pattern>/bwhist-flags.html</url-pattern> <url-pattern>/bandwidth-flags.html</url-pattern> <url-pattern>/dirbytes.html</url-pattern> <url-pattern>/advbwdist-perc.html</url-pattern> @@ -119,12 +117,6 @@ <url-pattern>/platforms.png</url-pattern> <url-pattern>/platforms.pdf</url-pattern> <url-pattern>/platforms.csv</url-pattern> - <url-pattern>/bandwidth.png</url-pattern> - <url-pattern>/bandwidth.pdf</url-pattern> - <url-pattern>/bandwidth.csv</url-pattern> - <url-pattern>/bwhist-flags.png</url-pattern> - <url-pattern>/bwhist-flags.pdf</url-pattern> - <url-pattern>/bwhist-flags.csv</url-pattern> <url-pattern>/bandwidth-flags.png</url-pattern> <url-pattern>/bandwidth-flags.pdf</url-pattern> <url-pattern>/bandwidth-flags.csv</url-pattern> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index 73d2f01..d19aeca 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -43,12 +43,10 @@ "summary": "How much traffic the Tor network can handle and how much traffic there is.", "description": "We measure total available bandwidth and current capacity by aggregating what relays and bridges report to directory authorities.", "metrics": [ - "bandwidth", "bandwidth-flags", "advbw-ipv6", "advbwdist-perc", "advbwdist-relay", - "bwhist-flags", "dirbytes", "connbidirect", "uncharted-data-flow" diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index 5173ae4..5c99546 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -67,28 +67,6 @@ ] }, { - "id": "bandwidth", - "title": "Total relay bandwidth (deprecated)", - "type": "Graph", - "description": "<p>This graph shows the total <a href=\"glossary.html#advertised-bandwidth\">advertised</a> and <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> of all <a href=\"glossary.html#relay\">relays</a> in the network. <strong>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href=\"/bandwidth-flags.html\">Advertised and consumed bandwidth by relay flags</a> graph.</strong></p>", - "function": "bandwidth", - "parameters": [ - "start", - "end" - ] - }, - { - "id": "bwhist-flags", - "title": "Consumed bandwidth by Exit/Guard flag combination (deprecated)", - "type": "Graph", - "description": "<p>This graph shows the <a href=\"glossary.html#bandwidth-history\">consumed bandwidth</a> reported by relays, subdivided into four distinct subsets by assigned \"Exit\" and/or \"Guard\" <a href=\"glossary.html#relay-flag\">flags</a>. <strong>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href=\"/bandwidth-flags.html\">Advertised and consumed bandwidth by relay flags</a> graph.</strong></p>", - "function": "bwhist_flags", - "parameters": [ - "start", - "end" - ] - }, - { "id": "bandwidth-flags", "title": "Advertised and consumed bandwidth by relay flags", "type": "Graph", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index 24bdba0..3838972 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -431,8 +431,7 @@ Relays self-report their advertised bandwidth in their server descriptors which <p>The following description applies to the following graphs:</p> <ul> -<li>Total relay bandwidth (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Advertised and consumed bandwidth by relay flag (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Advertised and consumed bandwidth by relay flags (just the advertised bandwidth part; for the consumed bandwidth part <a href="#consumed-bandwidth">see below</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth by IP version <a href="/advbw-ipv6.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth distribution <a href="/advbwdist-perc.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Advertised bandwidth of n-th fastest relays <a href="/advbwdist-relay.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> @@ -469,7 +468,7 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with <h4>Step 3: Compute daily averages</h4> -<p>The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a>, and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p> +<p>The first two graphs described here, namely <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a> and <a href="/advbw-ipv6.html">Advertised bandwidth by IP version</a>, have in common that they show daily averages of advertised bandwidth.</p> <p>In order to compute these averages, first match consensus entries with server descriptors by SHA-1 digest. Every consensus entry references exactly one server descriptor, and a server descriptor may be referenced from an arbitrary number of consensus entries. @@ -514,9 +513,7 @@ Relays self-report bandwidth histories as part of their extra-info descriptors, <p>The following description applies to the following graphs:</p> <ul> -<li>Total relay bandwidth (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Advertised and consumed bandwidth by relay flag (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> -<li>Consumed bandwidth by Exit/Guard flag combination <a href="/bwhist-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Advertised and consumed bandwidth by relay flags (just the consumed bandwidth part; for the advertised bandwidth part <a href="#advertised-bandwidth">see above</a>) <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> </ul> @@ -552,13 +549,13 @@ We consider a relay with the <code>"Guard"</code> flag as guard and a relay with <h4>Step 3: Compute daily totals</h4> -<p>The first three graphs described here, namely <a href="/bandwidth.html">Total relay bandwidth</a>, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flag</a>, and <a href="/bwhist-flags.html">Consumed bandwidth by Exit/Guard flag combination</a>, show daily totals of all bytes written or read by relays. -For all three graphs, we sum up all read and written bytes on a given day and divide the result by 2. +<p>The first graph described here, <a href="/bandwidth-flags.html">Advertised and consumed bandwidth by relay flags</a>, shows daily totals of all bytes written or read by relays. +For this graph, we sum up all read and written bytes on a given day and divide the result by 2. However, we only include bandwidth histories for a given day if a relay was listed as running in a consensus at least once on that day. We attribute bandwidth to guards and/or exits if a relay was a guard and/or exit at least in one consensus on a day.</p> -<p>The fourth graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. -As opposed to the first three graphs, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. +<p>The second graph, <a href="/dirbytes.html">Bandwidth spent on answering directory requests</a>, shows bytes spent by <a href="/glossary.html#directory-mirror">directory mirrors</a> on answering directory requests. +As opposed to the first graph, all bandwidth histories are included, regardless of whether a relay was listed as running in a consensus. Also, we compute total read directory and total written directory bytes for this fourth graph, not an average of the two.</p> <h3 id="connbidirect" class="hover">Connection usage diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index a78da8a..6c1955a 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -49,7 +49,7 @@ https://metrics.torproject.org/identifier.csv <li><b>September 15, 2018:</b> Removed all pre-aggregated CSV files.</li> <li><b>October 28, 2018:</b> Added and/or removed columns to <a href="#webstats-tb-platform">Tor Browser downloads and updates by platform</a> and <a href="#webstats-tb-locale">Tor Browser downloads and updates by locale</a> graphs.</li> <li><b>December 20, 2018:</b> Removed source parameters and output rows with aggregates over all sources from <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li> -<li><b>December 20, 2018 (scheduled):</b> Remove two graphs <a href="#bandwidth">Total relay bandwidth</a> and <a href="#bwhist-flags">Consumed bandwidth by Exit/Guard flag combination</a>, and update the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flag</a> graph to cover all data previously contained in the first two graphs.</li> +<li><b>December 20, 2018:</b> Removed two graphs Total relay bandwidth and Consumed bandwidth by Exit/Guard flag combination, and updated the data format of the <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph to cover all data previously contained in the first two graphs.</li> </ul> </div> @@ -332,32 +332,7 @@ Servers <a href="#servers" name="servers" class="anchor">#</a></h2> <h2><i class="fa fa-road fa-fw" aria-hidden="true"></i> Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> -<h3>Total relay bandwidth -<a href="/bandwidth.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> -<a href="/bandwidth.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> -<a href="#bandwidth" name="bandwidth" class="anchor">#</a></h3> - -<div class="bs-callout bs-callout-warning"> -<h3>Deprecated</h3> -<p>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph.</p> -</div> - -<h4>Parameters</h4> - -<ul> -<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> -<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> -</ul> - -<h4>Columns</h4> - -<ul> -<li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide.</li> -<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of all relays.</li> -</ul> - -<h3>Advertised and consumed bandwidth by relay flag +<h3>Advertised and consumed bandwidth by relay flags <a href="/bandwidth-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/bandwidth-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> <a href="#bandwidth-flags" name="bandwidth-flags" class="anchor">#</a></h3> @@ -373,14 +348,10 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> <ul> <li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>guard_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Guard"</b> relay flag are capable to provide. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>guard_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Guard"</b> relay flag. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>exit_advbw:</b> Total advertised bandwidth in Gbit/s that relays with the <b>"Exit"</b> relay flag are capable to provide. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>exit_bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with the <b>"Exit"</b> relay flag. <span class="red">This column is going to be removed after December 20, 2018.</span></li> -<li><b>have_guard_flag:</b> Whether relays included in this row had the <code>"Guard"</code> relay flag assigned (<code>"t"</code>) or not (<code>"f"</code>). <span class="blue">This column is going to be added after December 20, 2018.</span></li> -<li><b>have_exit_flag:</b> Whether relays included in this row had the <code>"Exit"</code> relay flag assigned and at the same time the <code>"BadExit"</code> not assigned (<code>"t"</code>) or not (<code>"f"</code>). <span class="blue">This column is going to be added after December 20, 2018.</span></li> -<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide. <span class="blue">This column is going to be added after December 20, 2018.</span></li> -<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic. <span class="blue">This column is going to be added after December 20, 2018.</span></li> +<li><b>have_guard_flag:</b> Whether relays included in this row had the <code>"Guard"</code> relay flag assigned (<code>"t"</code>) or not (<code>"f"</code>).</li> +<li><b>have_exit_flag:</b> Whether relays included in this row had the <code>"Exit"</code> relay flag assigned and at the same time the <code>"BadExit"</code> not assigned (<code>"t"</code>) or not (<code>"f"</code>).</li> +<li><b>advbw:</b> Total advertised bandwidth in Gbit/s that relays are capable to provide.</li> +<li><b>bwhist:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic.</li> </ul> <h3>Advertised bandwidth by IP version @@ -451,33 +422,6 @@ Traffic <a href="#traffic" name="traffic" class="anchor">#</a></h2> <li><b>exits:</b> Advertised bandwidth in Gbit/s of n-th fastest relay with the <b>"Exit"</b> relay flag.</li> </ul> -<h3>Consumed bandwidth by Exit/Guard flag combination -<a href="/bwhist-flags.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> -<a href="/bwhist-flags.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> -<a href="#bwhist-flags" name="bwhist-flags" class="anchor">#</a></h3> - -<div class="bs-callout bs-callout-warning"> -<h3>Deprecated</h3> -<p>This graph will disappear by December 20, 2018, because it won't contain anything new compared to the soon-to-be tweaked <a href="#bandwidth-flags">Advertised and consumed bandwidth by relay flags</a> graph.</p> -</div> - -<h4>Parameters</h4> - -<ul> -<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> -<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> -</ul> - -<h4>Columns</h4> - -<ul> -<li><b>date:</b> UTC date (YYYY-MM-DD) that relays reported bandwidth data for.</li> -<li><b>exit_only:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays without <b>"Guard"</b> and with <b>"Exit"</b> relay flag.</li> -<li><b>guard_and_exit:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with both <b>"Guard"</b> and <b>"Exit"</b> relay flag.</li> -<li><b>guard_only:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with <b>"Guard"</b> and without <b>"Exit"</b> relay flag.</li> -<li><b>middle_only:</b> Total consumed bandwidth in Gbit/s as the average of written and read traffic of relays with neither <b>"Guard"</b> nor <b>"Exit"</b> relay flag.</li> -</ul> - <h3>Bandwidth spent on answering directory requests <a href="/dirbytes.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/dirbytes.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a>

1 0

[metrics-web/release] Document another change to OnionPerf graphs.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 0ca51404b70258e1594df2d88f460fd6a6daf351 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Dec 20 10:33:00 2018 +0100 Document another change to OnionPerf graphs. Still related to #28603. --- src/main/resources/web/jsps/stats.jsp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index 2ae6726..a78da8a 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -610,6 +610,7 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2> <ul> <li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> <li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> +<li><b>server:</b> Either <b>"public"</b> for requests to a server on the public internet, or <b>"onion"</b> for requests to a version 2 onion server.</li> </ul> <h4>Columns</h4>

1 0

[metrics-web/release] Rewrite advbwdist's aggregate.R in Java.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit b605298c66c8c348fe589062dc1ddd3da293c8db Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Dec 10 15:17:46 2018 +0100 Rewrite advbwdist's aggregate.R in Java. This is yet another step torwards making the daily update Java-only. Implements #28801. --- build.xml | 17 -------- src/main/R/advbwdist/aggregate.R | 25 ------------ .../torproject/metrics/stats/advbwdist/Main.java | 46 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 42 deletions(-) diff --git a/build.xml b/build.xml index 93eda7b..42965bf 100644 --- a/build.xml +++ b/build.xml @@ -347,9 +347,6 @@ <target name="advbwdist"> <property name="module.name" value="advbwdist" /> <antcall target="run-java" /> - <antcall target="run-R" > - <param name="module.Rscript" value="aggregate.R" /> - </antcall> </target> <target name="hidserv" > @@ -453,20 +450,6 @@ <echo message="Java module ${module.name} finished. " /> </target> - <target name="run-R" > - <echo message="Running R module ${module.name}, script ${module.Rscript} ... " /> - <property name="Rscript" - value="${Rsources}/${module.name}/${module.Rscript}" /> - <exec executable="R" - dir="${modulebase}/${module.name}" - failonerror="true" > - <arg value="--slave"/> - <arg value="-f"/> - <arg value="${Rscript}" /> - </exec> - <echo message="R module ${module.name}, script ${module.Rscript} finished. " /> - </target> -  diff --git a/src/main/R/advbwdist/aggregate.R b/src/main/R/advbwdist/aggregate.R deleted file mode 100644 index 1c67dff..0000000 --- a/src/main/R/advbwdist/aggregate.R +++ /dev/null @@ -1,25 +0,0 @@ -require(reshape) -t <- read.csv("stats/advbwdist-validafter.csv", - colClasses = c("character", "logical", "integer", "integer", "integer"), - stringsAsFactors = FALSE) - -currSysDate <- paste(Sys.Date() - 1, "23:59:59") -t <- t[t$valid_after < currSysDate, ] -t$date <- as.factor(substr(t$valid_after, 1, 10)) -t$isexit <- !is.na(t$isexit) -t$relay <- ifelse(is.na(t$relay), -1, t$relay) -t$percentile <- ifelse(is.na(t$percentile), -1, t$percentile) - -t <- aggregate(list(advbw = t$advbw), by = list(date = t$date, - isexit = t$isexit, relay = t$relay, percentile = t$percentile), - FUN = median) - -t$isexit <- ifelse(t$isexit, "t", "") -t$relay <- ifelse(t$relay < 0, NA, t$relay) -t$percentile <- ifelse(t$percentile < 0, NA, t$percentile) -t$advbw <- floor(t$advbw) - -t <- t[order(t$date, t$isexit, t$relay, t$percentile), ] - -write.csv(t, "stats/advbwdist.csv", quote = FALSE, row.names = FALSE, na = "") - diff --git a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java index 7216581..6c4f4ac 100644 --- a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java @@ -10,15 +10,19 @@ import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor; +import org.apache.commons.math3.stat.descriptive.rank.Median; import org.apache.commons.math3.stat.descriptive.rank.Percentile; +import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -138,6 +142,39 @@ public class Main { } descriptorReader.saveHistoryFile(historyFile); bw.close(); + + /* Aggregate statistics. */ + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String today = dateFormat.format(new Date()); + SortedMap<String, List<Long>> preAggregatedValues = new TreeMap<>(); + try (BufferedReader br = new BufferedReader(new FileReader(resultsFile))) { + br.readLine(); /* Skip header. */ + String line; + while (null != (line = br.readLine())) { + String[] parts = line.split(","); + String date = parts[0].substring(0, 10); + if (date.compareTo(today) >= 0) { + continue; + } + String isExit = parts[1].equals("TRUE") ? "t" : ""; + String keyWithoutTime = String.format("%s,%s,%s,%s", + date, isExit, parts[2], parts[3]); + long value = Long.parseLong(parts[4]); + preAggregatedValues.putIfAbsent(keyWithoutTime, new ArrayList<>()); + preAggregatedValues.get(keyWithoutTime).add(value); + } + } + File aggregateResultsFile = new File("stats/advbwdist.csv"); + aggregateResultsFile.getParentFile().mkdirs(); + try (BufferedWriter bw2 = new BufferedWriter( + new FileWriter(aggregateResultsFile))) { + bw2.write("date,isexit,relay,percentile,advbw\n"); + for (Map.Entry<String, List<Long>> e : preAggregatedValues.entrySet()) { + bw2.write(String.format("%s,%.0f%n", e.getKey(), + computeMedian(e.getValue()))); + } + } } /** Compute percentiles (between 0 and 100) for the given list of values, and @@ -168,5 +205,14 @@ public class Main { } return computedPercentiles; } + + /** Return the median for the given list of values, or <code>Double.NaN</code> + * if the given list is empty. */ + static double computeMedian(List<Long> valueList) { + Median median = new Median() + .withEstimationType(Percentile.EstimationType.R_7); + median.setData(valueList.stream().mapToDouble(Long::doubleValue).toArray()); + return Math.floor(median.evaluate()); + } }

1 0