[tor-commits] [metrics-web/release] Break down totalcw numbers by Guard/(Bad)Exit flags.
karsten at torproject.org
karsten at torproject.org
Sat Nov 9 21:45:06 UTC 2019
commit 7b39042ecfae011d5891af963ae94bd35141fbc0
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Nov 29 10:09:28 2018 +0100
Break down totalcw numbers by Guard/(Bad)Exit flags.
Requires updating the vote table and the totalcw view in the database.
Implements #28328.
---
src/main/R/rserver/graphs.R | 4 ++-
.../torproject/metrics/stats/totalcw/Database.java | 36 ++++++++++------------
.../metrics/stats/totalcw/OutputLine.java | 12 ++++++--
.../torproject/metrics/stats/totalcw/Parser.java | 17 +++++-----
.../totalcw/TotalcwRelayNetworkStatusVote.java | 6 ++--
src/main/sql/totalcw/init-totalcw.sql | 16 +++++++---
.../totalcw/TotalcwRelayNetworkStatusVoteTest.java | 18 ++++++-----
7 files changed, 66 insertions(+), 43 deletions(-)
diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index df108e2..e3ac598 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -1562,7 +1562,9 @@ prepare_totalcw <- function(start_p, end_p) {
filter(if (!is.null(start_p))
valid_after_date >= as.Date(start_p) else TRUE) %>%
filter(if (!is.null(end_p))
- valid_after_date <= as.Date(end_p) else TRUE)
+ valid_after_date <= as.Date(end_p) else TRUE) %>%
+ group_by(valid_after_date, nickname) %>%
+ summarize(measured_sum_avg = sum(measured_sum_avg))
}
plot_totalcw <- function(start_p, end_p, path_p) {
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
index 66b0366..b6dc87c 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
@@ -66,9 +66,8 @@ class Database implements AutoCloseable {
"SELECT EXISTS (SELECT 1 FROM vote "
+ "WHERE valid_after = ? AND authority_id = ?)");
this.psVoteInsert = this.connection.prepareStatement(
- "INSERT INTO vote (valid_after, authority_id, measured_sum) "
- + "VALUES (?, ?, ?)",
- Statement.RETURN_GENERATED_KEYS);
+ "INSERT INTO vote (valid_after, authority_id, have_guard_flag, "
+ + "have_exit_flag, measured_sum) VALUES (?, ?, ?, ?, ?)");
}
/** Insert a parsed vote into the vote table. */
@@ -116,22 +115,17 @@ class Database implements AutoCloseable {
}
}
}
- int voteId = -1;
- this.psVoteInsert.clearParameters();
- this.psVoteInsert.setTimestamp(1,
- Timestamp.from(ZonedDateTime.of(vote.validAfter,
- ZoneId.of("UTC")).toInstant()), calendar);
- this.psVoteInsert.setInt(2, authorityId);
- this.psVoteInsert.setLong(3, vote.measuredSum);
- this.psVoteInsert.execute();
- try (ResultSet rs = this.psVoteInsert.getGeneratedKeys()) {
- if (rs.next()) {
- voteId = rs.getInt(1);
- }
- }
- if (voteId < 0) {
- throw new SQLException("Could not retrieve auto-generated key for new "
- + "vote entry.");
+ for (int measuredSumsIndex = 0; measuredSumsIndex < 4;
+ measuredSumsIndex++) {
+ this.psVoteInsert.clearParameters();
+ this.psVoteInsert.setTimestamp(1,
+ Timestamp.from(ZonedDateTime.of(vote.validAfter,
+ ZoneId.of("UTC")).toInstant()), calendar);
+ this.psVoteInsert.setInt(2, authorityId);
+ this.psVoteInsert.setBoolean(3, 1 == (measuredSumsIndex & 1));
+ this.psVoteInsert.setBoolean(4, 2 == (measuredSumsIndex & 2));
+ this.psVoteInsert.setLong(5, vote.measuredSums[measuredSumsIndex]);
+ this.psVoteInsert.execute();
}
}
@@ -159,6 +153,10 @@ class Database implements AutoCloseable {
outputLine.validAfterDate = rs.getDate(
OutputLine.Column.VALID_AFTER_DATE.name(), calendar).toLocalDate();
outputLine.nickname = rs.getString(OutputLine.Column.NICKNAME.name());
+ outputLine.haveGuardFlag = rs.getBoolean(
+ OutputLine.Column.HAVE_GUARD_FLAG.name());
+ outputLine.haveExitFlag = rs.getBoolean(
+ OutputLine.Column.HAVE_EXIT_FLAG.name());
outputLine.measuredSumAvg = rs.getLong(
OutputLine.Column.MEASURED_SUM_AVG.name());
statistics.add(outputLine);
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java b/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java
index 450dbac..5587e5d 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java
@@ -13,7 +13,8 @@ class OutputLine {
/** Column names used in the database and in the first line of the output
* file. */
enum Column {
- VALID_AFTER_DATE, NICKNAME, MEASURED_SUM_AVG
+ VALID_AFTER_DATE, NICKNAME, HAVE_GUARD_FLAG, HAVE_EXIT_FLAG,
+ MEASURED_SUM_AVG
}
/** Column headers joined together with the given delimiter. */
@@ -28,6 +29,12 @@ class OutputLine {
/** Server type, which can be "relay" or "bridge". */
String nickname;
+ /** Whether contained relays all have the "Guard" flag. */
+ boolean haveGuardFlag;
+
+ /** Whether contained relays all have the "Exit" flag. */
+ boolean haveExitFlag;
+
/** Mean value of total measured bandwidths of all relays over the day. */
Long measuredSumAvg;
@@ -35,7 +42,8 @@ class OutputLine {
* file. */
@Override
public String toString() {
- return String.format("%s,%s,%d", validAfterDate, nickname, measuredSumAvg);
+ return String.format("%s,%s,%s,%s,%d", validAfterDate, nickname,
+ haveGuardFlag ? "t" : "f", haveExitFlag ? "t" : "f", measuredSumAvg);
}
}
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
index b6a35b4..6070822 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
@@ -17,18 +17,21 @@ class Parser {
* contain any bandwidth measurements. */
TotalcwRelayNetworkStatusVote parseRelayNetworkStatusVote(
RelayNetworkStatusVote vote) {
- Long measuredSum = null;
+ boolean containsMeasuredBandwidths = false;
+ long[] measuredSums = new long[4];
for (NetworkStatusEntry entry : vote.getStatusEntries().values()) {
if (null == entry.getFlags() || !entry.getFlags().contains("Running")
|| entry.getMeasured() < 0L) {
continue;
}
- if (null == measuredSum) {
- measuredSum = 0L;
- }
- measuredSum += entry.getMeasured();
+ containsMeasuredBandwidths = true;
+ /* Encode flags as sum of Guard = 1 and (Exit and !BadExit) = 2. */
+ int measuredSumsIndex = (entry.getFlags().contains("Guard") ? 1 : 0)
+ + (entry.getFlags().contains("Exit")
+ && !entry.getFlags().contains("BadExit") ? 2 : 0);
+ measuredSums[measuredSumsIndex] += entry.getMeasured();
}
- if (null == measuredSum) {
+ if (!containsMeasuredBandwidths) {
/* Return null, because we wouldn't want to add this vote to the database
* anyway. */
return null;
@@ -39,7 +42,7 @@ class Parser {
.atZone(ZoneId.of("UTC")).toLocalDateTime();
parsedVote.identityHex = vote.getIdentity();
parsedVote.nickname = vote.getNickname();
- parsedVote.measuredSum = measuredSum;
+ parsedVote.measuredSums = measuredSums;
return parsedVote;
}
}
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java
index ff56d91..0c5a095 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java
@@ -19,7 +19,9 @@ class TotalcwRelayNetworkStatusVote {
* key. */
String identityHex;
- /** Sum of bandwidth measurements of all contained status entries. */
- long measuredSum;
+ /** Sums of bandwidth measurements of all contained status entries with four
+ * entries: 0 = neither Exit nor Guard, 1 = only Guard, 2 = only Exit, and
+ * 3 = both Guard and Exit. */
+ long[] measuredSums;
}
diff --git a/src/main/sql/totalcw/init-totalcw.sql b/src/main/sql/totalcw/init-totalcw.sql
index d723adb..cdba275 100644
--- a/src/main/sql/totalcw/init-totalcw.sql
+++ b/src/main/sql/totalcw/init-totalcw.sql
@@ -31,21 +31,27 @@ CREATE TABLE vote (
-- Numeric identifier uniquely identifying the authority generating this vote.
authority_id INTEGER REFERENCES authority (authority_id),
+ -- Whether contained relays had the Guard flag assigned.
+ have_guard_flag BOOLEAN NOT NULL,
+
+ -- Whether contained relays had the Exit flag assigned.
+ have_exit_flag BOOLEAN NOT NULL,
+
-- Sum of bandwidth measurements of all contained status entries.
measured_sum BIGINT NOT NULL,
- UNIQUE (valid_after, authority_id)
+ UNIQUE (valid_after, authority_id, have_guard_flag, have_exit_flag)
);
-- View on aggregated total consensus weight statistics in a format that is
-- compatible for writing to an output CSV file. Votes are only included in the
-- output if at least 12 votes are known for a given authority and day.
CREATE OR REPLACE VIEW totalcw AS
-SELECT DATE(valid_after) AS valid_after_date, nickname,
- FLOOR(AVG(measured_sum)) AS measured_sum_avg
+SELECT DATE(valid_after) AS valid_after_date, nickname, have_guard_flag,
+ have_exit_flag, FLOOR(AVG(measured_sum)) AS measured_sum_avg
FROM vote NATURAL JOIN authority
-GROUP BY DATE(valid_after), nickname
+GROUP BY DATE(valid_after), nickname, have_guard_flag, have_exit_flag
HAVING COUNT(vote_id) >= 12
AND DATE(valid_after) < (SELECT MAX(DATE(valid_after)) FROM vote)
-ORDER BY DATE(valid_after), nickname;
+ORDER BY DATE(valid_after), nickname, have_guard_flag, have_exit_flag;
diff --git a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
index 7c5ecc7..189b3b7 100644
--- a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
+++ b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
@@ -4,6 +4,7 @@
package org.torproject.metrics.stats.totalcw;
import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertNull;
import org.torproject.descriptor.Descriptor;
@@ -35,19 +36,22 @@ public class TotalcwRelayNetworkStatusVoteTest {
{ "2018-10-15-00-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-"
+ "55A38ED50848BE1F13C6A35C3CA637B0D962C2EF.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
- "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", -1L },
+ "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", null },
{ "2018-10-15-00-00-00-vote-27102BC123E7AF1D4741AE047E160C91ADC76B21-"
+ "049AB3179B12DACC391F06A10C2A8904E4339D33.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
- "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", 138700L },
+ "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21",
+ new long[] { 13700L, 47220L, 17080L, 60700L } },
{ "2018-10-15-00-00-00-vote-ED03BB616EB2F60BEC80151114BB25CEF515B226-"
+ "2669AD153408F88E416CE6206D1A75EC3324A2F4.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
- "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", 133370L },
+ "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226",
+ new long[] { 18020L, 43200L, 26150L, 46000L } },
{ "2018-10-15-00-00-00-vote-EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97-"
+ "38C6A19F78948B689345EE41D7119D76246C4D3E.part",
ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
- "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", 158395L }
+ "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97",
+ new long[] { 17365L, 52030L, 35400L, 53600L } }
});
}
@@ -64,7 +68,7 @@ public class TotalcwRelayNetworkStatusVoteTest {
public String expectedIdentityHex;
@Parameter(4)
- public long expectedMeasuredSum;
+ public long[] expectedMeasuredSums;
@Test
public void testParseVote() throws Exception {
@@ -82,13 +86,13 @@ public class TotalcwRelayNetworkStatusVoteTest {
sb.toString().getBytes(), new File(this.fileName), this.fileName)) {
TotalcwRelayNetworkStatusVote parsedVote = new Parser()
.parseRelayNetworkStatusVote((RelayNetworkStatusVote) descriptor);
- if (this.expectedMeasuredSum < 0L) {
+ if (null == this.expectedMeasuredSums) {
assertNull(parsedVote);
} else {
assertEquals(this.expectedValidAfter, parsedVote.validAfter);
assertEquals(this.expectedNickname, parsedVote.nickname);
assertEquals(this.expectedIdentityHex, parsedVote.identityHex);
- assertEquals(this.expectedMeasuredSum, parsedVote.measuredSum);
+ assertArrayEquals(this.expectedMeasuredSums, parsedVote.measuredSums);
}
}
}
More information about the tor-commits
mailing list