commit 7b39042ecfae011d5891af963ae94bd35141fbc0 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu Nov 29 10:09:28 2018 +0100 Break down totalcw numbers by Guard/(Bad)Exit flags. Requires updating the vote table and the totalcw view in the database. Implements #28328. --- src/main/R/rserver/graphs.R | 4 ++- .../torproject/metrics/stats/totalcw/Database.java | 36 ++++++++++------------ .../metrics/stats/totalcw/OutputLine.java | 12 ++++++-- .../torproject/metrics/stats/totalcw/Parser.java | 17 +++++----- .../totalcw/TotalcwRelayNetworkStatusVote.java | 6 ++-- src/main/sql/totalcw/init-totalcw.sql | 16 +++++++--- .../totalcw/TotalcwRelayNetworkStatusVoteTest.java | 18 ++++++----- 7 files changed, 66 insertions(+), 43 deletions(-) diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R index df108e2..e3ac598 100644 --- a/src/main/R/rserver/graphs.R +++ b/src/main/R/rserver/graphs.R @@ -1562,7 +1562,9 @@ prepare_totalcw <- function(start_p, end_p) { filter(if (!is.null(start_p)) valid_after_date >= as.Date(start_p) else TRUE) %>% filter(if (!is.null(end_p)) - valid_after_date <= as.Date(end_p) else TRUE) + valid_after_date <= as.Date(end_p) else TRUE) %>% + group_by(valid_after_date, nickname) %>% + summarize(measured_sum_avg = sum(measured_sum_avg)) } plot_totalcw <- function(start_p, end_p, path_p) { diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java index 66b0366..b6dc87c 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java @@ -66,9 +66,8 @@ class Database implements AutoCloseable { "SELECT EXISTS (SELECT 1 FROM vote " + "WHERE valid_after = ? AND authority_id = ?)"); this.psVoteInsert = this.connection.prepareStatement( - "INSERT INTO vote (valid_after, authority_id, measured_sum) " - + "VALUES (?, ?, ?)", - Statement.RETURN_GENERATED_KEYS); + "INSERT INTO vote (valid_after, authority_id, have_guard_flag, " + + "have_exit_flag, measured_sum) VALUES (?, ?, ?, ?, ?)"); } /** Insert a parsed vote into the vote table. */ @@ -116,22 +115,17 @@ class Database implements AutoCloseable { } } } - int voteId = -1; - this.psVoteInsert.clearParameters(); - this.psVoteInsert.setTimestamp(1, - Timestamp.from(ZonedDateTime.of(vote.validAfter, - ZoneId.of("UTC")).toInstant()), calendar); - this.psVoteInsert.setInt(2, authorityId); - this.psVoteInsert.setLong(3, vote.measuredSum); - this.psVoteInsert.execute(); - try (ResultSet rs = this.psVoteInsert.getGeneratedKeys()) { - if (rs.next()) { - voteId = rs.getInt(1); - } - } - if (voteId < 0) { - throw new SQLException("Could not retrieve auto-generated key for new " - + "vote entry."); + for (int measuredSumsIndex = 0; measuredSumsIndex < 4; + measuredSumsIndex++) { + this.psVoteInsert.clearParameters(); + this.psVoteInsert.setTimestamp(1, + Timestamp.from(ZonedDateTime.of(vote.validAfter, + ZoneId.of("UTC")).toInstant()), calendar); + this.psVoteInsert.setInt(2, authorityId); + this.psVoteInsert.setBoolean(3, 1 == (measuredSumsIndex & 1)); + this.psVoteInsert.setBoolean(4, 2 == (measuredSumsIndex & 2)); + this.psVoteInsert.setLong(5, vote.measuredSums[measuredSumsIndex]); + this.psVoteInsert.execute(); } } @@ -159,6 +153,10 @@ class Database implements AutoCloseable { outputLine.validAfterDate = rs.getDate( OutputLine.Column.VALID_AFTER_DATE.name(), calendar).toLocalDate(); outputLine.nickname = rs.getString(OutputLine.Column.NICKNAME.name()); + outputLine.haveGuardFlag = rs.getBoolean( + OutputLine.Column.HAVE_GUARD_FLAG.name()); + outputLine.haveExitFlag = rs.getBoolean( + OutputLine.Column.HAVE_EXIT_FLAG.name()); outputLine.measuredSumAvg = rs.getLong( OutputLine.Column.MEASURED_SUM_AVG.name()); statistics.add(outputLine); diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java b/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java index 450dbac..5587e5d 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java @@ -13,7 +13,8 @@ class OutputLine { /** Column names used in the database and in the first line of the output * file. */ enum Column { - VALID_AFTER_DATE, NICKNAME, MEASURED_SUM_AVG + VALID_AFTER_DATE, NICKNAME, HAVE_GUARD_FLAG, HAVE_EXIT_FLAG, + MEASURED_SUM_AVG } /** Column headers joined together with the given delimiter. */ @@ -28,6 +29,12 @@ class OutputLine { /** Server type, which can be "relay" or "bridge". */ String nickname; + /** Whether contained relays all have the "Guard" flag. */ + boolean haveGuardFlag; + + /** Whether contained relays all have the "Exit" flag. */ + boolean haveExitFlag; + /** Mean value of total measured bandwidths of all relays over the day. */ Long measuredSumAvg; @@ -35,7 +42,8 @@ class OutputLine { * file. */ @Override public String toString() { - return String.format("%s,%s,%d", validAfterDate, nickname, measuredSumAvg); + return String.format("%s,%s,%s,%s,%d", validAfterDate, nickname, + haveGuardFlag ? "t" : "f", haveExitFlag ? "t" : "f", measuredSumAvg); } } diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java index b6a35b4..6070822 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java @@ -17,18 +17,21 @@ class Parser { * contain any bandwidth measurements. */ TotalcwRelayNetworkStatusVote parseRelayNetworkStatusVote( RelayNetworkStatusVote vote) { - Long measuredSum = null; + boolean containsMeasuredBandwidths = false; + long[] measuredSums = new long[4]; for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { if (null == entry.getFlags() || !entry.getFlags().contains("Running") || entry.getMeasured() < 0L) { continue; } - if (null == measuredSum) { - measuredSum = 0L; - } - measuredSum += entry.getMeasured(); + containsMeasuredBandwidths = true; + /* Encode flags as sum of Guard = 1 and (Exit and !BadExit) = 2. */ + int measuredSumsIndex = (entry.getFlags().contains("Guard") ? 1 : 0) + + (entry.getFlags().contains("Exit") + && !entry.getFlags().contains("BadExit") ? 2 : 0); + measuredSums[measuredSumsIndex] += entry.getMeasured(); } - if (null == measuredSum) { + if (!containsMeasuredBandwidths) { /* Return null, because we wouldn't want to add this vote to the database * anyway. */ return null; @@ -39,7 +42,7 @@ class Parser { .atZone(ZoneId.of("UTC")).toLocalDateTime(); parsedVote.identityHex = vote.getIdentity(); parsedVote.nickname = vote.getNickname(); - parsedVote.measuredSum = measuredSum; + parsedVote.measuredSums = measuredSums; return parsedVote; } } diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java index ff56d91..0c5a095 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java @@ -19,7 +19,9 @@ class TotalcwRelayNetworkStatusVote { * key. */ String identityHex; - /** Sum of bandwidth measurements of all contained status entries. */ - long measuredSum; + /** Sums of bandwidth measurements of all contained status entries with four + * entries: 0 = neither Exit nor Guard, 1 = only Guard, 2 = only Exit, and + * 3 = both Guard and Exit. */ + long[] measuredSums; } diff --git a/src/main/sql/totalcw/init-totalcw.sql b/src/main/sql/totalcw/init-totalcw.sql index d723adb..cdba275 100644 --- a/src/main/sql/totalcw/init-totalcw.sql +++ b/src/main/sql/totalcw/init-totalcw.sql @@ -31,21 +31,27 @@ CREATE TABLE vote ( -- Numeric identifier uniquely identifying the authority generating this vote. authority_id INTEGER REFERENCES authority (authority_id), + -- Whether contained relays had the Guard flag assigned. + have_guard_flag BOOLEAN NOT NULL, + + -- Whether contained relays had the Exit flag assigned. + have_exit_flag BOOLEAN NOT NULL, + -- Sum of bandwidth measurements of all contained status entries. measured_sum BIGINT NOT NULL, - UNIQUE (valid_after, authority_id) + UNIQUE (valid_after, authority_id, have_guard_flag, have_exit_flag) ); -- View on aggregated total consensus weight statistics in a format that is -- compatible for writing to an output CSV file. Votes are only included in the -- output if at least 12 votes are known for a given authority and day. CREATE OR REPLACE VIEW totalcw AS -SELECT DATE(valid_after) AS valid_after_date, nickname, - FLOOR(AVG(measured_sum)) AS measured_sum_avg +SELECT DATE(valid_after) AS valid_after_date, nickname, have_guard_flag, + have_exit_flag, FLOOR(AVG(measured_sum)) AS measured_sum_avg FROM vote NATURAL JOIN authority -GROUP BY DATE(valid_after), nickname +GROUP BY DATE(valid_after), nickname, have_guard_flag, have_exit_flag HAVING COUNT(vote_id) >= 12 AND DATE(valid_after) < (SELECT MAX(DATE(valid_after)) FROM vote) -ORDER BY DATE(valid_after), nickname; +ORDER BY DATE(valid_after), nickname, have_guard_flag, have_exit_flag; diff --git a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java index 7c5ecc7..189b3b7 100644 --- a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java +++ b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java @@ -4,6 +4,7 @@ package org.torproject.metrics.stats.totalcw; import static junit.framework.TestCase.assertEquals; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNull; import org.torproject.descriptor.Descriptor; @@ -35,19 +36,22 @@ public class TotalcwRelayNetworkStatusVoteTest { { "2018-10-15-00-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-" + "55A38ED50848BE1F13C6A35C3CA637B0D962C2EF.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", -1L }, + "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", null }, { "2018-10-15-00-00-00-vote-27102BC123E7AF1D4741AE047E160C91ADC76B21-" + "049AB3179B12DACC391F06A10C2A8904E4339D33.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", 138700L }, + "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", + new long[] { 13700L, 47220L, 17080L, 60700L } }, { "2018-10-15-00-00-00-vote-ED03BB616EB2F60BEC80151114BB25CEF515B226-" + "2669AD153408F88E416CE6206D1A75EC3324A2F4.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", 133370L }, + "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", + new long[] { 18020L, 43200L, 26150L, 46000L } }, { "2018-10-15-00-00-00-vote-EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97-" + "38C6A19F78948B689345EE41D7119D76246C4D3E.part", ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(), - "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", 158395L } + "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", + new long[] { 17365L, 52030L, 35400L, 53600L } } }); } @@ -64,7 +68,7 @@ public class TotalcwRelayNetworkStatusVoteTest { public String expectedIdentityHex; @Parameter(4) - public long expectedMeasuredSum; + public long[] expectedMeasuredSums; @Test public void testParseVote() throws Exception { @@ -82,13 +86,13 @@ public class TotalcwRelayNetworkStatusVoteTest { sb.toString().getBytes(), new File(this.fileName), this.fileName)) { TotalcwRelayNetworkStatusVote parsedVote = new Parser() .parseRelayNetworkStatusVote((RelayNetworkStatusVote) descriptor); - if (this.expectedMeasuredSum < 0L) { + if (null == this.expectedMeasuredSums) { assertNull(parsedVote); } else { assertEquals(this.expectedValidAfter, parsedVote.validAfter); assertEquals(this.expectedNickname, parsedVote.nickname); assertEquals(this.expectedIdentityHex, parsedVote.identityHex); - assertEquals(this.expectedMeasuredSum, parsedVote.measuredSum); + assertArrayEquals(this.expectedMeasuredSums, parsedVote.measuredSums); } } }