[tor-commits] [metrics-web/release] Add servers-ipv6 module.
karsten at torproject.org
karsten at torproject.org
Wed May 30 13:45:11 UTC 2018
commit 73205cc3df76d0ca8d2fc62f26e2359093e73c7c
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed Dec 6 11:39:32 2017 +0100
Add servers-ipv6 module.
Implements #24218.
---
modules/ipv6servers/.gitignore | 3 +
modules/ipv6servers/build.xml | 21 ++
.../metrics/stats/ipv6servers/Configuration.java | 18 ++
.../metrics/stats/ipv6servers/Database.java | 218 +++++++++++++++++++++
.../torproject/metrics/stats/ipv6servers/Main.java | 106 ++++++++++
.../metrics/stats/ipv6servers/OutputLine.java | 75 +++++++
.../stats/ipv6servers/ParsedNetworkStatus.java | 47 +++++
.../stats/ipv6servers/ParsedServerDescriptor.java | 26 +++
.../metrics/stats/ipv6servers/Parser.java | 99 ++++++++++
.../metrics/stats/ipv6servers/Writer.java | 37 ++++
.../src/main/resources/init-ipv6servers.sql | 123 ++++++++++++
.../stats/ipv6servers/ParsedNetworkStatusTest.java | 140 +++++++++++++
.../ipv6servers/ParsedServerDescriptorTest.java | 97 +++++++++
.../000a7fe20a17bf5d9839a126b1dff43f998aac6f | 16 ++
.../0018ab4f2f28af683d52f06407edbf7ce1bd3b7d | 51 +++++
.../0041dbf9fe846f9765882f7dc8332f94b709e35a | 19 ++
.../01003df74972ce952ebfa390f468ef63c50efa25 | 189 ++++++++++++++++++
.../018c1229d5f56eebfc1d709d4692673d098800e8 | 54 +++++
.../descriptors/2017-12-04-20-00-00-consensus.part | 149 ++++++++++++++
...7-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part | 12 ++
.../64dd486d89af14027c9a7b4347a94b74dddb5cdb | 18 ++
.../ipv6servers/src/test/sql/test-ipv6servers.sql | 196 ++++++++++++++++++
shared/bin/20-run-ipv6servers-stats.sh | 5 +
src/submods/metrics-lib | 1 -
24 files changed, 1719 insertions(+), 1 deletion(-)
diff --git a/modules/ipv6servers/.gitignore b/modules/ipv6servers/.gitignore
new file mode 100644
index 0000000..c8e90bb
--- /dev/null
+++ b/modules/ipv6servers/.gitignore
@@ -0,0 +1,3 @@
+/stats/
+/status/
+
diff --git a/modules/ipv6servers/build.xml b/modules/ipv6servers/build.xml
new file mode 100644
index 0000000..736b579
--- /dev/null
+++ b/modules/ipv6servers/build.xml
@@ -0,0 +1,21 @@
+<project default="run" name="ipv6servers" basedir=".">
+
+ <property name="mainclass"
+ value="org.torproject.metrics.stats.ipv6servers.Main" />
+
+ <include file="../../shared/build-base.xml" as="basetask"/>
+ <target name="clean" depends="basetask.clean"/>
+ <target name="compile" depends="basetask.compile"/>
+ <target name="test" depends="basetask.test"/>
+ <target name="run" depends="basetask.run"/>
+
+ <path id="classpath">
+ <pathelement path="${classes}"/>
+ <path refid="base.classpath" />
+ <fileset dir="${libs}">
+ <include name="postgresql-jdbc3-9.2.jar"/>
+ </fileset>
+ </path>
+
+</project>
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java
new file mode 100644
index 0000000..dffcdf6
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java
@@ -0,0 +1,18 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+/** Configuration options parsed from Java properties with reasonable hard-coded
+ * defaults. */
+class Configuration {
+ static String descriptors = System.getProperty("descriptors",
+ "../../shared/in/");
+ static String database = System.getProperty("database",
+ "jdbc:postgresql:ipv6servers");
+ static String history = System.getProperty("history",
+ "status/read-descriptors");
+ static String output = System.getProperty("output",
+ "stats/ipv6servers.csv");
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java
new file mode 100644
index 0000000..c334263
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java
@@ -0,0 +1,218 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
+
+/** Database wrapper to connect to the database, insert data, run the stored
+ * procedure for aggregating data, and query aggregated data as output. */
+class Database {
+
+ /** Database connection string. */
+ private String jdbcString;
+
+ /** Connection object for all interactions with the database. */
+ private Connection connection;
+
+ /** Prepared statement for finding out whether a given server descriptor is
+ * already contained in the server_descriptors table. */
+ private PreparedStatement psServerDescriptorsSelect;
+
+ /** Prepared statement for inserting a server descriptor into the
+ * server_descriptors table. */
+ private PreparedStatement psServerDescriptorsInsert;
+
+ /** Prepared statement for checking whether a status has been inserted into
+ * the statuses table before. */
+ private PreparedStatement psStatusesSelect;
+
+ /** Prepared statement for inserting a status (without entries, yet) into
+ * the statuses table. */
+ private PreparedStatement psStatusesInsert;
+
+ /** Prepared statement for inserting a status entry into the status_entries
+ * table. */
+ private PreparedStatement psStatusEntriesInsert;
+
+ /** Create a new Database instance and prepare for inserting or querying
+ * data. */
+ Database(String jdbcString) throws SQLException {
+ this.jdbcString = jdbcString;
+ this.connect();
+ this.prepareStatements();
+ }
+
+ private void connect() throws SQLException {
+ this.connection = DriverManager.getConnection(this.jdbcString);
+ this.connection.setAutoCommit(false);
+ }
+
+ private void prepareStatements() throws SQLException {
+ this.psServerDescriptorsSelect = this.connection.prepareStatement(
+ "SELECT EXISTS (SELECT 1 FROM server_descriptors "
+ + "WHERE descriptor_digest_sha1 = decode(?, 'hex'))");
+ this.psServerDescriptorsInsert = this.connection.prepareStatement(
+ "INSERT INTO server_descriptors (descriptor_digest_sha1, "
+ + "advertised_bandwidth_bytes, announced_ipv6, exiting_ipv6_relay) "
+ + "VALUES (decode(?, 'hex'), ?, ?, ?)");
+ this.psStatusesSelect = this.connection.prepareStatement(
+ "SELECT EXISTS (SELECT 1 FROM statuses "
+ + "WHERE server = CAST(? AS server_enum) AND valid_after = ?)");
+ this.psStatusesInsert = this.connection.prepareStatement(
+ "INSERT INTO statuses (server, valid_after, running_count) "
+ + "VALUES (CAST(? AS server_enum), ?, ?)",
+ Statement.RETURN_GENERATED_KEYS);
+ this.psStatusEntriesInsert = this.connection.prepareStatement(
+ "INSERT INTO status_entries (status_id, descriptor_digest_sha1, "
+ + "guard_relay, exit_relay, reachable_ipv6_relay) "
+ + "VALUES (?, decode(?, 'hex'), ?, ?, ?)");
+ }
+
+ /** Insert a server descriptor into the server_descriptors table. */
+ void insertServerDescriptor(
+ ParsedServerDescriptor parsedServerDescriptor) throws SQLException {
+ this.psServerDescriptorsSelect.clearParameters();
+ this.psServerDescriptorsSelect.setString(1,
+ parsedServerDescriptor.digest);
+ try (ResultSet rs = psServerDescriptorsSelect.executeQuery()) {
+ if (rs.next()) {
+ if (rs.getBoolean(1)) {
+ /* Server descriptor is already contained. */
+ return;
+ }
+ }
+ }
+ this.psServerDescriptorsInsert.clearParameters();
+ this.psServerDescriptorsInsert.setString(1,
+ parsedServerDescriptor.digest);
+ this.psServerDescriptorsInsert.setInt(2,
+ parsedServerDescriptor.advertisedBandwidth);
+ this.psServerDescriptorsInsert.setBoolean(3,
+ parsedServerDescriptor.announced);
+ this.psServerDescriptorsInsert.setBoolean(4,
+ parsedServerDescriptor.exiting);
+ this.psServerDescriptorsInsert.execute();
+ }
+
+ /** Insert a status and all contained entries into the statuses and
+ * status_entries table. */
+ void insertStatus(ParsedNetworkStatus parsedNetworkStatus)
+ throws SQLException {
+ this.psStatusesSelect.clearParameters();
+ this.psStatusesSelect.setString(1,
+ parsedNetworkStatus.isRelay ? "relay" : "bridge");
+ Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"),
+ Locale.US);
+ this.psStatusesSelect.setTimestamp(2,
+ Timestamp.from(ZonedDateTime.of(parsedNetworkStatus.timestamp,
+ ZoneId.of("UTC")).toInstant()), calendar);
+ try (ResultSet rs = this.psStatusesSelect.executeQuery()) {
+ if (rs.next()) {
+ if (rs.getBoolean(1)) {
+ /* Status is already contained. */
+ return;
+ }
+ }
+ }
+ int statusId = -1;
+ this.psStatusesInsert.clearParameters();
+ this.psStatusesInsert.setString(1,
+ parsedNetworkStatus.isRelay ? "relay" : "bridge");
+ this.psStatusesInsert.setTimestamp(2,
+ Timestamp.from(ZonedDateTime.of(parsedNetworkStatus.timestamp,
+ ZoneId.of("UTC")).toInstant()), calendar);
+ this.psStatusesInsert.setInt(3, parsedNetworkStatus.running);
+ this.psStatusesInsert.execute();
+ try (ResultSet rs = this.psStatusesInsert.getGeneratedKeys()) {
+ if (rs.next()) {
+ statusId = rs.getInt(1);
+ }
+ }
+ if (statusId < 0) {
+ throw new SQLException("Could not retrieve auto-generated key for new "
+ + "statuses entry.");
+ }
+ for (ParsedNetworkStatus.Entry entry : parsedNetworkStatus.entries) {
+ this.psStatusEntriesInsert.clearParameters();
+ this.psStatusEntriesInsert.setInt(1, statusId);
+ this.psStatusEntriesInsert.setString(2, entry.digest);
+ this.psStatusEntriesInsert.setBoolean(3, entry.guard);
+ this.psStatusEntriesInsert.setBoolean(4, entry.exit);
+ this.psStatusEntriesInsert.setBoolean(5, entry.reachable);
+ this.psStatusEntriesInsert.addBatch();
+ }
+ this.psStatusEntriesInsert.executeBatch();
+ }
+
+ /** Call the aggregate() function to aggregate rows from the status_entries
+ * and server_descriptors tables into the aggregated table. */
+ void aggregate() throws SQLException {
+ Statement st = this.connection.createStatement();
+ st.executeQuery("SELECT aggregate_ipv6()");
+ }
+
+ /** Roll back any changes made in this execution. */
+ void rollback() throws SQLException {
+ this.connection.rollback();
+ }
+
+ /** Commit all changes made in this execution. */
+ void commit() throws SQLException {
+ this.connection.commit();
+ }
+
+ /** Query the servers_ipv6 view to obtain aggregated statistics. */
+ Iterable<OutputLine> queryServersIpv6() throws SQLException {
+ List<OutputLine> statistics = new ArrayList<>();
+ Statement st = this.connection.createStatement();
+ Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"),
+ Locale.US);
+ String queryString = "SELECT " + OutputLine.getColumnHeaders(", ")
+ + " FROM ipv6servers";
+ try (ResultSet rs = st.executeQuery(queryString)) {
+ while (rs.next()) {
+ OutputLine outputLine = new OutputLine();
+ outputLine.date = rs.getDate(OutputLine.Column.VALID_AFTER_DATE.name(),
+ calendar).toLocalDate();
+ outputLine.server = rs.getString(OutputLine.Column.SERVER.name());
+ outputLine.guard = rs.getString(OutputLine.Column.GUARD_RELAY.name());
+ outputLine.exit = rs.getString(OutputLine.Column.EXIT_RELAY.name());
+ outputLine.announced = rs.getString(
+ OutputLine.Column.ANNOUNCED_IPV6.name());
+ outputLine.exiting = rs.getString(
+ OutputLine.Column.EXITING_IPV6_RELAY.name());
+ outputLine.reachable = rs.getString(
+ OutputLine.Column.REACHABLE_IPV6_RELAY.name());
+ outputLine.count = rs.getLong(
+ OutputLine.Column.SERVER_COUNT_SUM_AVG.name());
+ outputLine.advertisedBandwidth = rs.getLong(
+ OutputLine.Column.ADVERTISED_BANDWIDTH_BYTES_SUM_AVG.name());
+ if (rs.wasNull()) {
+ outputLine.advertisedBandwidth = null;
+ }
+ statistics.add(outputLine);
+ }
+ }
+ return statistics;
+ }
+
+ /** Disconnect from the database. */
+ void disconnect() throws SQLException {
+ this.connection.close();
+ }
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java
new file mode 100644
index 0000000..81433c0
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java
@@ -0,0 +1,106 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.sql.SQLException;
+
+/** Main class of the ipv6servers module that imports relevant parts from server
+ * descriptors and network statuses into a database, and exports aggregate
+ * statistics on IPv6 support to a CSV file. */
+public class Main {
+
+ private static Logger log = LoggerFactory.getLogger(Main.class);
+
+ /** Run the module. */
+ public static void main(String[] args) throws Exception {
+
+ log.info("Starting ipv6servers module.");
+
+ log.info("Reading descriptors and inserting relevant parts into the "
+ + "database.");
+ DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader();
+ File historyFile = new File(Configuration.history);
+ reader.setHistoryFile(historyFile);
+ Parser parser = new Parser();
+ Database database = new Database(Configuration.database);
+ try {
+ for (Descriptor descriptor : reader.readDescriptors(
+ new File(Configuration.descriptors
+ + "recent/relay-descriptors/consensuses"),
+ new File(Configuration.descriptors
+ + "recent/relay-descriptors/server-descriptors"),
+ new File(Configuration.descriptors
+ + "recent/bridge-descriptors/statuses"),
+ new File(Configuration.descriptors
+ + "recent/bridge-descriptors/server-descriptors"),
+ new File(Configuration.descriptors
+ + "archive/relay-descriptors/consensuses"),
+ new File(Configuration.descriptors
+ + "archive/relay-descriptors/server-descriptors"),
+ new File(Configuration.descriptors
+ + "archive/bridge-descriptors/statuses"),
+ new File(Configuration.descriptors
+ + "archive/bridge-descriptors/server-descriptors"))) {
+ if (descriptor instanceof ServerDescriptor) {
+ database.insertServerDescriptor(parser.parseServerDescriptor(
+ (ServerDescriptor) descriptor));
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+ database.insertStatus(parser.parseRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor));
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ database.insertStatus(parser.parseBridgeNetworkStatus(
+ (BridgeNetworkStatus) descriptor));
+ } else {
+ log.debug("Skipping unknown descriptor of type {}.",
+ descriptor.getClass());
+ }
+ }
+
+ log.info("Aggregating database entries.");
+ database.aggregate();
+
+ log.info("Committing all updated parts in the database.");
+ database.commit();
+ } catch (SQLException sqle) {
+ log.error("Cannot recover from SQL exception while inserting or "
+ + "aggregating data. Rolling back and exiting.", sqle);
+ database.rollback();
+ database.disconnect();
+ return;
+ }
+ reader.saveHistoryFile(historyFile);
+
+ log.info("Querying aggregated statistics from the database.");
+ Iterable<OutputLine> output;
+ try {
+ output = database.queryServersIpv6();
+ } catch (SQLException sqle) {
+ log.error("Cannot recover from SQL exception while querying. Not writing "
+ + "output file.", sqle);
+ return;
+ } finally {
+ database.disconnect();
+ }
+
+ log.info("Writing aggregated statistics to {}.", Configuration.output);
+ if (null != output) {
+ new Writer().write(Paths.get(Configuration.output), output);
+ }
+
+ log.info("Terminating ipv6servers module.");
+ }
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java
new file mode 100644
index 0000000..eba5f13
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java
@@ -0,0 +1,75 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Data object holding all parts of an output line. */
+class OutputLine {
+
+ /** Column names used in the database and in the first line of the output
+ * file. */
+ enum Column {
+ VALID_AFTER_DATE, SERVER, GUARD_RELAY, EXIT_RELAY, ANNOUNCED_IPV6,
+ EXITING_IPV6_RELAY, REACHABLE_IPV6_RELAY, SERVER_COUNT_SUM_AVG,
+ ADVERTISED_BANDWIDTH_BYTES_SUM_AVG
+ }
+
+ /** Column headers joined together with the given delimiter. */
+ static String getColumnHeaders(String delimiter) {
+ List<String> columnHeaders = new ArrayList<>();
+ for (Column column : Column.values()) {
+ columnHeaders.add(column.toString());
+ }
+ return String.join(delimiter, columnHeaders).toLowerCase();
+ }
+
+ /** Date. */
+ LocalDate date;
+
+ /** Server type, which can be "relay" or "bridge". */
+ String server;
+
+ /** Whether relays had the Guard flag ("t") or not ("f"). */
+ String guard;
+
+ /** Whether relays had the Exit flag ("t") or not ("f"). */
+ String exit;
+
+ /** Whether relays or bridges have announced an IPv6 address in their server
+ * descriptor ("t") or not ("f"). */
+ String announced;
+
+ /** Whether relays have announced a non-reject-all IPv6 exit policy in their
+ * server descriptor ("t") or not ("f"). */
+ String exiting;
+
+ /** Whether the directory authorities have confirmed IPv6 OR reachability by
+ * including an "a" line for a relay containing an IPv6 address. */
+ String reachable;
+
+ /** Number of relays or bridges matching the previous criteria. */
+ long count;
+
+ /** Total advertised bandwidth of all relays matching the previous
+ * criteria. */
+ Long advertisedBandwidth;
+
+ /** Format all fields in a single output line for inclusion in a CSV
+ * file. */
+ @Override
+ public String toString() {
+ return String.format("%s,%s,%s,%s,%s,%s,%s,%s,%s",
+ date, server, emptyNull(guard), emptyNull(exit), emptyNull(announced),
+ emptyNull(exiting), emptyNull(reachable), emptyNull(count),
+ emptyNull(advertisedBandwidth));
+ }
+
+ private static String emptyNull(Object text) {
+ return null == text ? "" : text.toString();
+ }
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatus.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatus.java
new file mode 100644
index 0000000..f185250
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatus.java
@@ -0,0 +1,47 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Data object holding all relevant parts parsed from a (relay or bridge)
+ * network status. */
+class ParsedNetworkStatus {
+
+ /** Whether this is a relay network status as opposed to a bridge network
+ * status. */
+ boolean isRelay;
+
+ /** Valid-after time in case of relay network status and published time in
+ * case of bridge network status. */
+ LocalDateTime timestamp;
+
+ /** Number of relays or bridges with the Running flag. */
+ int running = 0;
+
+ /** Contained status entries. */
+ List<Entry> entries = new ArrayList<>();
+
+ /** Data object holding all relevant parts from a network status entry. */
+ static class Entry {
+
+ /** Hex-encoded SHA-1 server descriptor digest. */
+ String digest;
+
+ /** Whether this relay has the Guard flag; false for bridges. */
+ boolean guard;
+
+ /** Whether this relay has the Exit flag (and not the BadExit flag at the
+ * same time); false for bridges. */
+ boolean exit;
+
+ /** Whether the directory authorities include an IPv6 address in this
+ * entry's "a" line, confirming the relay's reachability via IPv6; false for
+ * bridges. */
+ boolean reachable;
+ }
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptor.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptor.java
new file mode 100644
index 0000000..c8d0ceb
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptor.java
@@ -0,0 +1,26 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+/** Data object holding all relevant parts parsed from a (relay or bridge)
+ * server descriptor. */
+class ParsedServerDescriptor {
+
+ /** Hex-encoded SHA-1 server descriptor digest. */
+ String digest;
+
+ /** Advertised bandwidth bytes of this relay as the minimum of bandwidth rate,
+ * bandwidth burst, and observed bandwidth (if reported); 0 for bridges. */
+ int advertisedBandwidth;
+
+ /** Whether the relay or bridge announced an IPv6 address in an "or-address"
+ * line. */
+ boolean announced;
+
+ /** Whether the relay allows exiting via IPv6, which is the case if the
+ * server descriptor contains an "ipv6-policy" line that is not
+ * "ipv6-policy reject 1-65535"; false for bridges. */
+ boolean exiting;
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java
new file mode 100644
index 0000000..95b1d5a
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java
@@ -0,0 +1,99 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.RelayServerDescriptor;
+import org.torproject.descriptor.ServerDescriptor;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.time.Instant;
+import java.time.ZoneId;
+
+/** Parser that extracts all relevant parts from (relay and bridge) server
+ * descriptors and (relay and bridge) statuses and creates data objects for
+ * them. */
+class Parser {
+
+ /** Parse a (relay or bridge) server descriptor. */
+ ParsedServerDescriptor parseServerDescriptor(
+ ServerDescriptor serverDescriptor) {
+ ParsedServerDescriptor parsedDescriptor = new ParsedServerDescriptor();
+ parsedDescriptor.digest = serverDescriptor.getDigestSha1Hex();
+ for (String orAddress : serverDescriptor.getOrAddresses()) {
+ if (StringUtils.countMatches(orAddress, ":") >= 2) {
+ parsedDescriptor.announced = true;
+ break;
+ }
+ }
+ if (serverDescriptor instanceof RelayServerDescriptor) {
+ parsedDescriptor.advertisedBandwidth =
+ Math.min(serverDescriptor.getBandwidthRate(),
+ serverDescriptor.getBandwidthBurst());
+ if (serverDescriptor.getBandwidthObserved() >= 0) {
+ parsedDescriptor.advertisedBandwidth =
+ Math.min(parsedDescriptor.advertisedBandwidth,
+ serverDescriptor.getBandwidthObserved());
+ }
+ parsedDescriptor.exiting
+ = null != serverDescriptor.getIpv6DefaultPolicy()
+ && !("reject".equals(serverDescriptor.getIpv6DefaultPolicy())
+ && "1-65535".equals(serverDescriptor.getIpv6PortList()));
+ }
+ return parsedDescriptor;
+ }
+
+ /** Parse a relay network status. */
+ ParsedNetworkStatus parseRelayNetworkStatusConsensus(
+ RelayNetworkStatusConsensus consensus) throws Exception {
+ return this.parseStatus(true, consensus.getValidAfterMillis(),
+ consensus.getStatusEntries().values());
+ }
+
+ /** Parse a bridge network status. */
+ ParsedNetworkStatus parseBridgeNetworkStatus(BridgeNetworkStatus status)
+ throws Exception {
+ return this.parseStatus(false, status.getPublishedMillis(),
+ status.getStatusEntries().values());
+ }
+
+ private ParsedNetworkStatus parseStatus(boolean isRelay, long timestampMillis,
+ Iterable<NetworkStatusEntry> entries) {
+ ParsedNetworkStatus parsedStatus = new ParsedNetworkStatus();
+ parsedStatus.isRelay = isRelay;
+ parsedStatus.timestamp = Instant.ofEpochMilli(timestampMillis)
+ .atZone(ZoneId.of("UTC")).toLocalDateTime();
+ for (NetworkStatusEntry entry : entries) {
+ if (!entry.getFlags().contains("Running")) {
+ continue;
+ }
+ parsedStatus.running++;
+ }
+ for (NetworkStatusEntry entry : entries) {
+ if (!entry.getFlags().contains("Running")) {
+ continue;
+ }
+ ParsedNetworkStatus.Entry parsedEntry = new ParsedNetworkStatus.Entry();
+ parsedEntry.digest = entry.getDescriptor().toLowerCase();
+ if (isRelay) {
+ parsedEntry.guard = entry.getFlags().contains("Guard");
+ parsedEntry.exit = entry.getFlags().contains("Exit")
+ && !entry.getFlags().contains("BadExit");
+ parsedEntry.reachable = false;
+ for (String orAddress : entry.getOrAddresses()) {
+ if (StringUtils.countMatches(orAddress, ":") >= 2) {
+ parsedEntry.reachable = true;
+ break;
+ }
+ }
+ }
+ parsedStatus.entries.add(parsedEntry);
+ }
+ return parsedStatus;
+ }
+}
+
diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java
new file mode 100644
index 0000000..96f8a8d
--- /dev/null
+++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java
@@ -0,0 +1,37 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Writer that takes output line objects and writes them to a file, preceded
+ * by a column header line. */
+class Writer {
+
+ /** Write output lines to the given file. */
+ void write(Path filePath, Iterable<OutputLine> outputLines)
+ throws IOException {
+ File parentFile = filePath.toFile().getParentFile();
+ if (null != parentFile && !parentFile.exists()) {
+ boolean madeDirs = parentFile.mkdirs();
+ if (!madeDirs) {
+ throw new IOException("Unable to create parent directory of output "
+ + "file. Not writing this file.");
+ }
+ }
+ List<String> formattedOutputLines = new ArrayList<>();
+ formattedOutputLines.add(OutputLine.getColumnHeaders(","));
+ for (OutputLine line : outputLines) {
+ formattedOutputLines.add(line.toString());
+ }
+ Files.write(filePath, formattedOutputLines, StandardCharsets.UTF_8);
+ }
+}
+
diff --git a/modules/ipv6servers/src/main/resources/init-ipv6servers.sql b/modules/ipv6servers/src/main/resources/init-ipv6servers.sql
new file mode 100644
index 0000000..6a72167
--- /dev/null
+++ b/modules/ipv6servers/src/main/resources/init-ipv6servers.sql
@@ -0,0 +1,123 @@
+-- Copyright 2017 The Tor Project
+-- See LICENSE for licensing information
+
+-- Table of all relevant parts contained in relay or bridge server descriptors.
+-- We're not deleting from this table, because we can never be sure that we
+-- won't import a previously missing status that we'll want to match against
+-- existing server descriptors.
+CREATE TABLE server_descriptors (
+ descriptor_digest_sha1 BYTEA PRIMARY KEY,
+ advertised_bandwidth_bytes INTEGER NOT NULL,
+ announced_ipv6 BOOLEAN NOT NULL,
+ exiting_ipv6_relay BOOLEAN NOT NULL
+);
+
+-- Enumeration type for servers, which can be either relays or bridges.
+CREATE TYPE server_enum AS ENUM ('relay', 'bridge');
+
+-- Table of all relay or bridge statuses. We're not deleting from this table.
+CREATE TABLE statuses (
+ status_id SERIAL PRIMARY KEY,
+ server server_enum NOT NULL,
+ valid_after TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+ running_count INTEGER NOT NULL,
+ UNIQUE (server, valid_after)
+);
+
+-- Table of relay or bridge status entries. Unlike previous tables, we're
+-- deleting from this table after aggregating rows into the aggregated table.
+-- Otherwise this table would grow too large over time.
+CREATE TABLE status_entries (
+ status_id INTEGER REFERENCES statuses (status_id) NOT NULL,
+ descriptor_digest_sha1 BYTEA NOT NULL,
+ guard_relay BOOLEAN NOT NULL,
+ exit_relay BOOLEAN NOT NULL,
+ reachable_ipv6_relay BOOLEAN NOT NULL,
+ UNIQUE (status_id, descriptor_digest_sha1)
+);
+
+-- Table of joined and aggregated server_descriptors and status_entries rows.
+-- For a given status and combination of flags and IPv6 capabilities, we count
+-- the number of servers and advertised bandwidth bytes.
+CREATE TABLE aggregated_ipv6 (
+ status_id INTEGER REFERENCES statuses (status_id) NOT NULL,
+ guard_relay BOOLEAN NOT NULL,
+ exit_relay BOOLEAN NOT NULL,
+ announced_ipv6 BOOLEAN NOT NULL,
+ exiting_ipv6_relay BOOLEAN NOT NULL,
+ reachable_ipv6_relay BOOLEAN NOT NULL,
+ server_count_sum INTEGER NOT NULL,
+ advertised_bandwidth_bytes_sum BIGINT NOT NULL,
+ CONSTRAINT aggregated_ipv6_unique
+ UNIQUE (status_id, guard_relay, exit_relay, announced_ipv6,
+ exiting_ipv6_relay, reachable_ipv6_relay)
+);
+
+-- Function to aggregate server_descriptors and status_entries rows into the
+-- aggregated table and delete rows from status_entries that are then contained
+-- in the aggregated table. This function is supposed to be called once after
+-- inserting new rows into server_descriptors and/or status_entries. Subsequent
+-- calls won't have any effect.
+CREATE OR REPLACE FUNCTION aggregate_ipv6() RETURNS VOID AS $$
+INSERT INTO aggregated_ipv6
+SELECT status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay,
+ reachable_ipv6_relay, COUNT(*) AS server_count_sum,
+ SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes
+FROM status_entries
+NATURAL JOIN server_descriptors
+NATURAL JOIN statuses
+GROUP BY status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay,
+ reachable_ipv6_relay
+ON CONFLICT ON CONSTRAINT aggregated_ipv6_unique
+DO UPDATE SET server_count_sum = aggregated_ipv6.server_count_sum
+ + EXCLUDED.server_count_sum,
+ advertised_bandwidth_bytes_sum
+ = aggregated_ipv6.advertised_bandwidth_bytes_sum
+ + EXCLUDED.advertised_bandwidth_bytes_sum;
+DELETE FROM status_entries WHERE EXISTS (
+ SELECT 1 FROM server_descriptors
+ WHERE descriptor_digest_sha1 = status_entries.descriptor_digest_sha1);
+$$ LANGUAGE SQL;
+
+-- View on previously aggregated IPv6 server statistics in a format that is
+-- compatible for writing to an output CSV file. Statuses are only included in
+-- the output if they have at least 1 relay or bridge with the Running flag and
+-- if at least 99.9% of referenced server descriptors are present. Dates are
+-- only included in the output if at least 12 statuses are known. The last two
+-- dates are excluded to avoid statistics from flapping if missing descriptors
+-- are provided late.
+CREATE OR REPLACE VIEW ipv6servers AS
+WITH included_statuses AS (
+ SELECT status_id, server, valid_after
+ FROM statuses NATURAL JOIN aggregated_ipv6
+ GROUP BY status_id, server, valid_after
+ HAVING running_count > 0
+ AND 1000 * SUM(server_count_sum) > 999 * running_count
+), included_dates AS (
+ SELECT DATE(valid_after) AS valid_after_date, server
+ FROM included_statuses
+ GROUP BY DATE(valid_after), server
+ HAVING COUNT(status_id) >= 12
+ AND DATE(valid_after)
+ < (SELECT MAX(DATE(valid_after)) FROM included_statuses) - 1
+)
+SELECT DATE(valid_after) AS valid_after_date, server,
+ CASE WHEN server = 'relay' THEN guard_relay ELSE NULL END AS guard_relay,
+ CASE WHEN server = 'relay' THEN exit_relay ELSE NULL END AS exit_relay,
+ announced_ipv6,
+ CASE WHEN server = 'relay' THEN exiting_ipv6_relay ELSE NULL END
+ AS exiting_ipv6_relay,
+ CASE WHEN server = 'relay' THEN reachable_ipv6_relay ELSE NULL END
+ AS reachable_ipv6_relay,
+ FLOOR(AVG(server_count_sum)) AS server_count_sum_avg,
+ CASE WHEN server = 'relay' THEN FLOOR(AVG(advertised_bandwidth_bytes_sum))
+ ELSE NULL END AS advertised_bandwidth_bytes_sum_avg
+FROM statuses NATURAL JOIN aggregated_ipv6
+WHERE status_id IN (SELECT status_id FROM included_statuses)
+AND DATE(valid_after) IN (
+ SELECT valid_after_date FROM included_dates WHERE server = statuses.server)
+GROUP BY DATE(valid_after), server, guard_relay, exit_relay, announced_ipv6,
+ exiting_ipv6_relay, reachable_ipv6_relay
+ORDER BY valid_after_date, server, guard_relay, exit_relay, announced_ipv6,
+ exiting_ipv6_relay, reachable_ipv6_relay;
+
diff --git a/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatusTest.java b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatusTest.java
new file mode 100644
index 0000000..4b07154
--- /dev/null
+++ b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatusTest.java
@@ -0,0 +1,140 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertNotNull;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.util.Arrays;
+import java.util.Collection;
+
+ at RunWith(Parameterized.class)
+public class ParsedNetworkStatusTest {
+
+ /** Provide test data. */
+ @Parameters
+ public static Collection<Object[]> data() {
+ String relayFileName = "descriptors/2017-12-04-20-00-00-consensus.part";
+ String bridgeFileName = "descriptors/"
+ + "20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part";
+ return Arrays.asList(new Object[][] {
+ { "Relay status without Guard or Exit flag and without IPv6 address. ",
+ relayFileName, true,
+ ZonedDateTime.parse("2017-12-04T20:00:00Z").toLocalDateTime(), 3,
+ "19bd830ae419b4c6ea1047370d0a5ac446f1748d", false, false, false },
+ { "Relay status with Guard and Exit flag and without IPv6 address.",
+ relayFileName, true,
+ ZonedDateTime.parse("2017-12-04T20:00:00Z").toLocalDateTime(), 3,
+ "600a614a5ee63f8cb55aa3d4d9e9a8dd8d748d77", true, true, false },
+ { "Relay status with Guard flag only and with IPv6 address.",
+ relayFileName, true,
+ ZonedDateTime.parse("2017-12-04T20:00:00Z").toLocalDateTime(), 3,
+ "d993e03f907f7cb302a877feb7608cbd6c4cfeb0", true, false, true },
+ { "Bridge status with Running flag.",
+ bridgeFileName, false,
+ ZonedDateTime.parse("2017-12-04T19:05:07Z").toLocalDateTime(), 1,
+ "01b2cadfbcc0ebe50f395863665ac376d25f08ed", false, false, false },
+ { "Bridge status without Running flag (skipped!).",
+ bridgeFileName, false,
+ ZonedDateTime.parse("2017-12-04T19:05:07Z").toLocalDateTime(), 1,
+ null, false, false, false }
+ });
+ }
+
+ @Parameter
+ public String description;
+
+ @Parameter(1)
+ public String fileName;
+
+ @Parameter(2)
+ public boolean isRelay;
+
+ @Parameter(3)
+ public LocalDateTime timestamp;
+
+ @Parameter(4)
+ public int running;
+
+ @Parameter(5)
+ public String digest;
+
+ @Parameter(6)
+ public boolean guard;
+
+ @Parameter(7)
+ public boolean exit;
+
+ @Parameter(8)
+ public boolean reachable;
+
+ @Test
+ public void testParseNetworkStatus() throws Exception {
+ InputStream is = getClass().getClassLoader().getResourceAsStream(
+ this.fileName);
+ assertNotNull(this.description, is);
+ StringBuilder sb = new StringBuilder();
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ String line = br.readLine();
+ while (null != line) {
+ sb.append(line).append('\n');
+ line = br.readLine();
+ }
+ for (Descriptor descriptor
+ : DescriptorSourceFactory.createDescriptorParser().parseDescriptors(
+ sb.toString().getBytes(), new File(this.fileName), this.fileName)) {
+ ParsedNetworkStatus parsedNetworkStatus;
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ parsedNetworkStatus = new Parser().parseRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ parsedNetworkStatus = new Parser().parseBridgeNetworkStatus(
+ (BridgeNetworkStatus) descriptor);
+ } else {
+ fail(this.description);
+ return;
+ }
+ assertEquals(this.description, this.isRelay, parsedNetworkStatus.isRelay);
+ assertEquals(this.description, this.timestamp,
+ parsedNetworkStatus.timestamp);
+ assertEquals(this.description, this.running, parsedNetworkStatus.running);
+ if (null != this.digest) {
+ boolean foundEntry = false;
+ for (ParsedNetworkStatus.Entry parsedEntry
+ : parsedNetworkStatus.entries) {
+ if (this.digest.equals(parsedEntry.digest)) {
+ assertEquals(this.description, this.guard, parsedEntry.guard);
+ assertEquals(this.description, this.exit, parsedEntry.exit);
+ assertEquals(this.description, this.reachable,
+ parsedEntry.reachable);
+ foundEntry = true;
+ break;
+ }
+ }
+ if (!foundEntry) {
+ fail(this.description);
+ }
+ }
+ }
+ }
+}
+
diff --git a/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptorTest.java b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptorTest.java
new file mode 100644
index 0000000..5079031
--- /dev/null
+++ b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptorTest.java
@@ -0,0 +1,97 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.stats.ipv6servers;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ServerDescriptor;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Collection;
+
+ at RunWith(Parameterized.class)
+public class ParsedServerDescriptorTest {
+
+ /** Provide test data. */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { "Relay server descriptor without or-address or ipv6-policy line.",
+ "descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d",
+ 819200, false, false },
+ { "Relay server descriptor with or-address and ipv6-policy line.",
+ "descriptors/01003df74972ce952ebfa390f468ef63c50efa25",
+ 6576128, true, true },
+ { "Relay server descriptor with or-address line only.",
+ "descriptors/018c1229d5f56eebfc1d709d4692673d098800e8",
+ 0, true, false },
+ { "Bridge server descriptor without or-address or ipv6-policy line.",
+ "descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f",
+ 0, false, false },
+ { "Bridge server descriptor with or-address line.",
+ "descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a",
+ 0, true, false },
+ { "Bridge server descriptor with (ignored) ipv6-policy accept line.",
+ "descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb",
+ 0, false, false }
+ });
+ }
+
+ @Parameter
+ public String description;
+
+ @Parameter(1)
+ public String fileName;
+
+ @Parameter(2)
+ public int advertisedBandwidth;
+
+ @Parameter(3)
+ public boolean announced;
+
+ @Parameter(4)
+ public boolean exiting;
+
+ @Test
+ public void testParseServerDescriptor() throws Exception {
+ InputStream is = getClass().getClassLoader().getResourceAsStream(
+ this.fileName);
+ assertNotNull(this.description, is);
+ StringBuilder sb = new StringBuilder();
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ String line = br.readLine();
+ while (null != line) {
+ sb.append(line).append('\n');
+ line = br.readLine();
+ }
+ for (Descriptor descriptor
+ : DescriptorSourceFactory.createDescriptorParser().parseDescriptors(
+ sb.toString().getBytes(), new File(this.fileName), this.fileName)) {
+ assertTrue(this.description, descriptor instanceof ServerDescriptor);
+ ParsedServerDescriptor parsedServerDescriptor
+ = new Parser().parseServerDescriptor((ServerDescriptor) descriptor);
+ assertEquals(this.description, this.advertisedBandwidth,
+ parsedServerDescriptor.advertisedBandwidth);
+ assertEquals(this.description, this.announced,
+ parsedServerDescriptor.announced);
+ assertEquals(this.description, this.exiting,
+ parsedServerDescriptor.exiting);
+ }
+ }
+}
+
diff --git a/modules/ipv6servers/src/test/resources/descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f b/modules/ipv6servers/src/test/resources/descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f
new file mode 100644
index 0000000..7911a41
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f
@@ -0,0 +1,16 @@
+ at type bridge-server-descriptor 1.2
+router Unnamed 10.141.52.121 63839 0 0
+master-key-ed25519 OFHu6w2KsTtvbqDlAuA1pHOW4v9EodQI7F39qLLVyho
+platform Tor 0.3.1.8 on Linux
+proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+published 2017-12-01 17:03:25
+fingerprint 47EE 975E 1C8F 63A4 DD3D 8851 4ECC C4F7 65C7 C208
+uptime 1950945
+bandwidth 1073741824 1073741824 428846
+extra-info-digest EEA5CD46F1FA4A12E239CF35217AF17F4AFC7529 c4lqRmT0oC/GZz7Rq9HKZaaTMT9wotA4ZU1AYWjFjXM
+hidden-service-dir
+ntor-onion-key lBsrbMYEECHW0v0o/gTpNbyORbhTSz+oyz/uD/dAIic=
+reject *:*
+tunnelled-dir-server
+router-digest-sha256 PruciDU6rrAgwoedoaGpvDTKWidF0ZNhqdcEX7KjrjI
+router-digest 000A7FE20A17BF5D9839A126B1DFF43F998AAC6F
diff --git a/modules/ipv6servers/src/test/resources/descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d b/modules/ipv6servers/src/test/resources/descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d
new file mode 100644
index 0000000..1a82dc7
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d
@@ -0,0 +1,51 @@
+ at type server-descriptor 1.0
+router NitraniumOne 89.38.149.201 5001 0 5002
+identity-ed25519
+-----BEGIN ED25519 CERT-----
+AQQABmpJAfx0GsMq1jpLcBRJ3jqBqGM/lg6d8jC7Tj4lMoRI8OFvAQAgBAAz5x6X
+UU1Y5htzoGc6PZsbz88xBmlqPLFMO3sOR0ZBzBqGJprWEhMYu+sY5d9y8vV5Lco+
+wxl1RkBc/X4586ab9WnV3e12WxxwFux+Ey/UtC6JFsTRTAZx7W7aKv7HQwQ=
+-----END ED25519 CERT-----
+master-key-ed25519 M+cel1FNWOYbc6BnOj2bG8/PMQZpajyxTDt7DkdGQcw
+platform Tor 0.3.1.8 on Linux
+proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+published 2017-12-01 02:23:58
+fingerprint 6F2B 25E9 2B5E 492A 5754 122A 3486 4BE4 E835 E0C4
+uptime 1159217
+bandwidth 819200 1024000 943771
+extra-info-digest B28408EE76B34E1750B77B8DFA8ACAEBB65C4828 ynrJoGDoIgxAXYlrHQ5V5qaNDbBsm22IxAnuqE4kIgk
+onion-key
+-----BEGIN RSA PUBLIC KEY-----
+MIGJAoGBAMeTc60EcKNyLjUD6KTXRPpOQt89Ivz/NxPgHYW9snYtx5PLXObhgo5x
+ODT2JKS+EPCZbHBnt9x1ZRIKvKIjmJbLIYL9ixk7lBoST9wa+6eVkZOBQFYTS24x
+70D4CDzKDgBg6AatlljCLjcOLcRiqsL39OaITaRiQdDqIofVPHr5AgMBAAE=
+-----END RSA PUBLIC KEY-----
+signing-key
+-----BEGIN RSA PUBLIC KEY-----
+MIGJAoGBAN+gODfQPOFT1c3iAlhqNdevKCObuBpT8mvyl6TfkeRP9Zh7Z/eTGPhH
+P48i1i1dS0vHYhQfIrwvU05DsLRed50W9MUEfSuMxmyE/+Qh8rR3tmVZPlIek9Ww
+wNaW3oBncnz5uodPXM9G370mKPkUYftR1H5yuBJF5HX6heoB+QbhAgMBAAE=
+-----END RSA PUBLIC KEY-----
+onion-key-crosscert
+-----BEGIN CROSSCERT-----
+hhPGeWcfIZggOAm0jwar4edfCIkLA54TbRpyJbl/VHEBTBynSKiAlBcDtFU/UrU+
+MrIh72VdisowF7LgvQUvwBlh4RkIMG/fXigdhI3YkpFqOiM2cD+lljUgIpB6sjXO
+bOHzpTKPiVdPfycu/WtQdags++OVs8yWbROH77q+wzw=
+-----END CROSSCERT-----
+ntor-onion-key-crosscert 0
+-----BEGIN ED25519 CERT-----
+AQoABmtbATPnHpdRTVjmG3OgZzo9mxvPzzEGaWo8sUw7ew5HRkHMAFb1Z9i/lRi0
+WXcoS15f7cBxZOYxwQCZBDJ8jaB0nDBbNsW90CzbwJInmNsXhy61/azD1wNLePra
+f1icWuyXEgc=
+-----END ED25519 CERT-----
+hidden-service-dir
+ntor-onion-key Ok3aAgZ7aN+r8461N3Wuc6Y9l9o3ZwKQwRP6veSybjM=
+reject *:*
+tunnelled-dir-server
+router-sig-ed25519 JFP3nuXCByN4JSJ1MnIPbiOS0Y6Rayi1CBF+bo/+wEXPSNmPmog/XdJD9RtYEHgZJGDejJyZ8nENzLxjlcllBA
+router-signature
+-----BEGIN SIGNATURE-----
+KF/xiANCygTl2dO4+UmVtMvOC5/G/k9XXCxpX9XVaOaqcX2XmLsKdGnj0ihWmgG+
+grQKwzQqQ9QescAI1yjUMLStyy0ta2UTDBEM8eWBylgZKCzrJDNG4zjeL/URJjpd
+YrJeIPl19li82ZY+gMUs6gVFXe/XqkeTxBGqDFysz1Q=
+-----END SIGNATURE-----
diff --git a/modules/ipv6servers/src/test/resources/descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a b/modules/ipv6servers/src/test/resources/descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a
new file mode 100644
index 0000000..0360bf8
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a
@@ -0,0 +1,19 @@
+ at type bridge-server-descriptor 1.2
+router cielarko 10.34.119.160 58783 0 0
+or-address [fd9f:2e19:3bcf::61:e1da]:58783
+master-key-ed25519 YFV9cJzNYy6HPbMjrXW1d1QKdPJyy1CeEtpjj5K7dg8
+platform Tor 0.3.2.5-alpha on Linux
+proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+published 2017-12-01 06:05:21
+fingerprint 9691 9EB7 B966 9F21 855E 85E7 263D 9006 68BA 414A
+uptime 66360
+bandwidth 1073741824 1073741824 272174
+extra-info-digest 31B5A95ECB1EC795C82D6D1BBE51D5B1B0186F22 8e6RyL4f/JFlvOUSCa+a0+SDPIzOEhaDHAL+0Lt1PZQ
+hidden-service-dir
+contact somebody
+bridge-distribution-request any
+ntor-onion-key ntj1wQU/1yrbiUMHDugRqzo+LmOUgGUwOTVoaYF8nWo=
+reject *:*
+tunnelled-dir-server
+router-digest-sha256 PEuRjE6moK4BLQpxJFjNoSyjkqubGWEwg5bXiGhgoMc
+router-digest 0041DBF9FE846F9765882F7DC8332F94B709E35A
diff --git a/modules/ipv6servers/src/test/resources/descriptors/01003df74972ce952ebfa390f468ef63c50efa25 b/modules/ipv6servers/src/test/resources/descriptors/01003df74972ce952ebfa390f468ef63c50efa25
new file mode 100644
index 0000000..ef0f4e3
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/01003df74972ce952ebfa390f468ef63c50efa25
@@ -0,0 +1,189 @@
+ at type server-descriptor 1.0
+router zwiebeltoralf2 5.9.158.75 9001 0 9030
+identity-ed25519
+-----BEGIN ED25519 CERT-----
+AQQABmkLAcEyQpC+Ms4utzY0ooqt7hHwOJdsquonbf5TqLleiAzVAQAgBABhcLLB
+ch5vHWVUQjbwOhxNbdCLSUK2RUYou92uX6wVPSxJy4mkJqbZntgB4+BByB7EvEc/
+ilCcOK9jLRgtGqkvmbtwXw10gOoFZi0iLBX7qZ0jpAgixzJcLlSCo6QcqQs=
+-----END ED25519 CERT-----
+master-key-ed25519 YXCywXIebx1lVEI28DocTW3Qi0lCtkVGKLvdrl+sFT0
+or-address [2a01:4f8:190:514a::2]:9001
+platform Tor 0.3.2.5-alpha on Linux
+proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+published 2017-12-01 00:27:39
+fingerprint D11D 1187 7769 B9E6 1753 7B4B 46BF B92B 443D E33D
+uptime 12050
+bandwidth 1073741824 1073741824 6576128
+extra-info-digest 6EDE204F5C0C1148DA1CEB96A5974A99AC5A6FE5 ulDel5e42xtMGHib91Q1nPeirPhdDTRwEGsA4NBBo1o
+caches-extra-info
+onion-key
+-----BEGIN RSA PUBLIC KEY-----
+MIGJAoGBAL3rowWZqDg5dFnJVSWfneuDtTCU7oNLEpeEN8weL+REPOvMgTApcWmJ
+N84fuTmfN3QDV7gU2PZ3f5Koong5hYo/MEwbiuR3RGgsMxNnu+fYdkaoXf9aSegA
+JyJMrwdIeikVjOspjfGhtmQuvX740XVb7/O98F0eYM1rpBi2EGkhAgMBAAE=
+-----END RSA PUBLIC KEY-----
+signing-key
+-----BEGIN RSA PUBLIC KEY-----
+MIGJAoGBAPEK2TJRvkY6xQspNOvbVAqRA8mS01c72lUillwLx3MDeLn0Grb+N5hZ
+xHp265tChF5nefQUwxTu85SkGa9y1ALyNEm2+fJVMW99c0aiaRfOMrRlOnqwpz4y
+T3OVD7H4AvAUg8hBYqsi5lpJxnXNJNCajou35RyZdA8E+gewuWFbAgMBAAE=
+-----END RSA PUBLIC KEY-----
+onion-key-crosscert
+-----BEGIN CROSSCERT-----
+NkqKSVgAnLC3Xd//qbm3o2e7kG9XrQ1DPAQIF1SORPxL2h5meRrJ5lKrjnQN2nml
+PwLKX18vp4ajpMz/Q9619EMvL9WZ1ENtDbwoqUXFlSGVJ3C369LPvAgrgP0LObdn
+2ovwEvXE1CMpcqfrLjba9LTaeL7DLngwqXTmFexVV3o=
+-----END CROSSCERT-----
+ntor-onion-key-crosscert 1
+-----BEGIN ED25519 CERT-----
+AQoABmtZAWFwssFyHm8dZVRCNvA6HE1t0ItJQrZFRii73a5frBU9AFtd28+1g6/m
+dzQz9tYFAvlyoEoZWlJlUnbliZT6YKLJdExdYy/Nfvzi+14lHUFELb8jEXOk1VsC
+m71SoV2Vxw8=
+-----END ED25519 CERT-----
+family $1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA
+hidden-service-dir
+contact replace k with c : kontakt @ zwiebeltoralf . de
+ntor-onion-key tVAtwlp9HKfF0pm7wYI9TuLc5aDczWAcHg+sVJECnGM=
+reject 0.0.0.0/8:*
+reject 169.254.0.0/16:*
+reject 127.0.0.0/8:*
+reject 192.168.0.0/16:*
+reject 10.0.0.0/8:*
+reject 172.16.0.0/12:*
+reject 5.9.158.75:*
+reject 162.218.232.15/24:*
+reject 162.218.237.3/24:*
+reject 195.78.229.65:*
+reject 195.26.85.200/24:443
+reject 103.37.152.52:*
+reject 104.111.232.180:*
+reject 104.20.44.57:*
+reject 104.244.42.1:*
+reject 104.41.152.17:*
+reject 113.107.57.43:*
+reject 13.107.21.200:*
+reject 149.154.175.50:*
+reject 151.101.112.193:*
+reject 151.101.113.140:*
+reject 152.195.133.74:*
+reject 165.227.251.186:*
+reject 170.81.138.25:*
+reject 172.217.23.130:*
+reject 172.217.23.131:*
+reject 172.217.23.142:*
+reject 174.129.243.60:*
+reject 176.34.155.20:*
+reject 185.36.100.196:*
+reject 195.26.85.200:*
+reject 2.17.7.57:*
+reject 207.182.152.130:*
+reject 209.85.233.95:*
+reject 216.150.210.199:*
+reject 216.150.210.200:*
+reject 216.38.197.179:*
+reject 216.38.197.185:*
+reject 217.182.138.181:*
+reject 217.69.135.132:*
+reject 23.35.100.252:*
+reject 23.35.109.140:*
+reject 31.13.64.17:*
+reject 46.101.175.199:*
+reject 52.222.170.230:*
+reject 52.35.255.219:*
+reject 64.58.116.132:*
+reject 78.140.166.6:*
+reject 91.149.157.121:*
+reject 95.213.11.181:*
+accept *:79
+accept *:110
+accept *:119
+accept *:143
+accept *:194
+accept *:220
+accept *:389
+accept *:443
+accept *:464
+accept *:465
+accept *:531
+accept *:543
+accept *:544
+accept *:554
+accept *:563
+accept *:587
+accept *:636
+accept *:706
+accept *:749
+accept *:853
+accept *:873
+accept *:902
+accept *:903
+accept *:904
+accept *:981
+accept *:989
+accept *:990
+accept *:991
+accept *:992
+accept *:993
+accept *:994
+accept *:995
+accept *:1194
+accept *:1220
+accept *:1293
+accept *:1533
+accept *:1677
+accept *:1723
+accept *:1755
+accept *:1863
+accept *:1883
+accept *:2095
+accept *:2096
+accept *:2102
+accept *:2103
+accept *:2104
+accept *:3128
+accept *:3690
+accept *:4321
+accept *:4643
+accept *:5050
+accept *:5190
+accept *:5222
+accept *:5269
+accept *:5280
+accept *:5900
+accept *:6660
+accept *:6661
+accept *:6662
+accept *:6663
+accept *:6664
+accept *:6665
+accept *:6666
+accept *:6667
+accept *:6668
+accept *:6669
+accept *:6679
+accept *:6697
+accept *:7777
+accept *:8008
+accept *:8074
+accept *:8082
+accept *:8087
+accept *:8232
+accept *:8233
+accept *:8332
+accept *:8333
+accept *:8883
+accept *:9418
+accept *:11371
+accept *:19294
+accept *:50002
+accept *:64738
+reject *:*
+ipv6-policy accept 79,110,119,143,194,220,389,443,464-465,531,543-544,554,563,587,636,706,749,853,873,902-904,981,989-995,1194,1220,1293,1533,1677,1723,1755,1863,1883,2095-2096,2102-2104,3128,3690,4321,4643,5050,5190,5222,5269,5280,5900,6660-6669,6679,6697,7777,8008,8074,8082,8087,8232-8233,8332-8333,8883,9418,11371,19294,50002,64738
+tunnelled-dir-server
+router-sig-ed25519 jg4DdPa8gG5JwnzBtM/YWUALcb/AoGodWkeroChEmC17vdlVFv7UhanWvCFQo4gAFzvB4meRp0F1fcz19Qu3Cw
+router-signature
+-----BEGIN SIGNATURE-----
+sMlEGWDRZJkfB8W+3rS9dUTvXk8UEmlvTT0HIopl/YlvZ/bSEe25VIuMCzODgrrK
+rHBsP6zw2IICVmBnW/TyqOOqwphIPA9RR8gs0BsE1cKh/g6SXsSfpGIR9cGde6q4
++u8IDSEp1uUpDzEgJt/3Enkw3RDYWl5eODeuE9xMKvo=
+-----END SIGNATURE-----
diff --git a/modules/ipv6servers/src/test/resources/descriptors/018c1229d5f56eebfc1d709d4692673d098800e8 b/modules/ipv6servers/src/test/resources/descriptors/018c1229d5f56eebfc1d709d4692673d098800e8
new file mode 100644
index 0000000..0c60c54
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/018c1229d5f56eebfc1d709d4692673d098800e8
@@ -0,0 +1,54 @@
+ at type server-descriptor 1.0
+router HCTOR003 192.30.34.248 9001 0 0
+identity-ed25519
+-----BEGIN ED25519 CERT-----
+AQQABmkCAfbbsyUlm3tvNXIKsdl7FWjCBS27EqU24hlQc4V76pKFAQAgBAC8UP2n
+QP9JHK+trzTtoDAsENdJt5yvp/nIyhQY/TcsY7WQ4w/yJlTJeC5ysSaEmbsgyBiI
+YWEeG3o20mXfIzoX9idsUk1MBDb2+eCpc9JWTkk0zuo80xLyjP7uGLBqKAU=
+-----END ED25519 CERT-----
+master-key-ed25519 vFD9p0D/SRyvra807aAwLBDXSbecr6f5yMoUGP03LGM
+or-address [2604:180::40f1:f45c]:9001
+platform Tor 0.2.9.8 on Linux
+proto Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2
+published 2017-12-01 01:55:47
+fingerprint CF00 0B63 C68D 4FAA 3067 7342 267B D75F 1699 A3C7
+uptime 2264147
+bandwidth 1073741824 1073741824 0
+extra-info-digest A03FF6707C62350ED5AD650010D70DE324FB6A5F xzHCpCEQM4cp9mrDmnkxT7ws1kdv0jZt2HtdyWGRtXw
+onion-key
+-----BEGIN RSA PUBLIC KEY-----
+MIGJAoGBAOiURbkUxwrtqfb+Ykp1rSodLlJHPzGy3+BX5Myve4tqZg2LvynN4o4y
+OPEGNUs3mdHBYZmXpt5ZgGQd3uNb21dvVLX9RrW9uslVmEhJbfNbPxjs1gdhnJya
+64Bcju57PRCr97IG+7mBtyd+6iwmVy3JRKDhXE84raAhAvt4JW2bAgMBAAE=
+-----END RSA PUBLIC KEY-----
+signing-key
+-----BEGIN RSA PUBLIC KEY-----
+MIGJAoGBAPPPj2Rt36C1tiSnZtpKKgxwBxgw9QwwVdtKESDVXXnEwamwQYkSwcPB
+xG0RmXRVoXvKva7+e3N4DkeE+DVrZ4GwkKRKbRhtfmp+TH4+TskI+7WAKuKJdCLP
+fD5P+9RUF7OcShVYzjzuXnha6nNQbHAny8wL2EJxEcc1Yq0H1smfAgMBAAE=
+-----END RSA PUBLIC KEY-----
+onion-key-crosscert
+-----BEGIN CROSSCERT-----
+0UiDaVEDGQx71DnKbFc6HxYsVfitgaPJT+0bjBcqnOh8v+jfTUua4npDpi9RkL0W
+5xy0jN1N8j0JTrXjDDPQTF2akgu+iWCoWom4jzmy3HXj+a4jyPaWKVI4Qhz+znYW
+Ts8joaiNImQ94+59DVvxbT2aqHv6OqH/H0Aa93Q1bhQ=
+-----END CROSSCERT-----
+ntor-onion-key-crosscert 1
+-----BEGIN ED25519 CERT-----
+AQoABmliAbxQ/adA/0kcr62vNO2gMCwQ10m3nK+n+cjKFBj9NyxjAB8b32quZYIR
+D5R7Mw6t4Jvu22eBQtbfXbeUneZ9gPsgnw7EEr6dzwbhMY9t4PRBInsRiU2o4C0c
+E/cOR9EKVgE=
+-----END ED25519 CERT-----
+family $CF000B63C68D4FAA30677342267BD75F1699A3C7 $CF879F5ACD419C915170BB1978CFFA1DF6E7ACFF
+hibernating 1
+hidden-service-dir
+contact PGP 0x9958256C Daniel Hagan <daniel.hagan at hagan-consulting.com>
+ntor-onion-key VKuzSIox0yRwcJWUmHrXIq8H7NiyGIBDmmRZA1v4ghk=
+reject *:*
+router-sig-ed25519 lS8zFzXia00h9YnFucF/zAOI/SsCNQwaKy/psMQheKR1nJ+SLSwHTmBtTDb7xWxyKQuikrML/obZ1CfPIkleAw
+router-signature
+-----BEGIN SIGNATURE-----
+yUcGXmxpsHsiD9O1AHElOaUBpCuhcfSC6GrmYcKhfCDpFyH1vOZMrdUE9doQ91By
+uc3zNwRiaivmlFjOj2jh0PYwO8UIMQUAbQThas1yVZx1Jv/qYc9yTE1W4pPnLdxj
+sHN+fVzYKbGrYo9kUrAuU9GnfLIQv5/9pUd8Nq7+/a4=
+-----END SIGNATURE-----
diff --git a/modules/ipv6servers/src/test/resources/descriptors/2017-12-04-20-00-00-consensus.part b/modules/ipv6servers/src/test/resources/descriptors/2017-12-04-20-00-00-consensus.part
new file mode 100644
index 0000000..73e7a98
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/2017-12-04-20-00-00-consensus.part
@@ -0,0 +1,149 @@
+ at type network-status-consensus-3 1.0
+network-status-version 3
+vote-status consensus
+consensus-method 26
+valid-after 2017-12-04 20:00:00
+fresh-until 2017-12-04 21:00:00
+valid-until 2017-12-04 23:00:00
+voting-delay 300 300
+client-versions 0.2.5.14,0.2.5.15,0.2.5.16,0.2.8.14,0.2.8.15,0.2.8.16,0.2.8.17,0.2.9.11,0.2.9.12,0.2.9.13,0.2.9.14,0.3.0.9,0.3.0.10,0.3.0.11,0.3.0.12,0.3.0.13,0.3.1.5-alpha,0.3.1.6-rc,0.3.1.7,0.3.1.8,0.3.1.9,0.3.2.1-alpha,0.3.2.2-alpha,0.3.2.3-alpha,0.3.2.4-alpha,0.3.2.5-alpha,0.3.2.6-alpha
+server-versions 0.2.5.14,0.2.5.15,0.2.5.16,0.2.8.14,0.2.8.15,0.2.8.16,0.2.8.17,0.2.9.11,0.2.9.12,0.2.9.13,0.2.9.14,0.3.0.9,0.3.0.10,0.3.0.11,0.3.0.12,0.3.0.13,0.3.1.5-alpha,0.3.1.6-rc,0.3.1.7,0.3.1.8,0.3.1.9,0.3.2.1-alpha,0.3.2.2-alpha,0.3.2.3-alpha,0.3.2.4-alpha,0.3.2.5-alpha,0.3.2.6-alpha
+known-flags Authority BadExit Exit Fast Guard HSDir NoEdConsensus Running Stable V2Dir Valid
+recommended-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2
+recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2
+required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2
+required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2
+params CircuitPriorityHalflifeMsec=30000 NumDirectoryGuards=3 NumEntryGuards=1 NumNTorsPerTAP=100 Support022HiddenServices=0 UseNTorHandshake=1 UseOptimisticData=1 bwauthpid=1 cbttestfreq=10 pb_disablepct=0 usecreatefast=0
+shared-rand-previous-value 9 koy+780Z3gcdh2ZavmUHWEwpS4oRouJp+Lr8Kc1HPDY=
+shared-rand-current-value 9 YkqgxViKZUGDyELmvIsVxFfyJSCAwYIznbLRqMwSSI8=
+dir-source dannenberg 0232AF901C31A04EE9848595AF9BB7620D4C5B2E dannenberg.torauth.de 193.23.244.244 80 443
+contact Andreas Lehner
+vote-digest ED565598CA7BD1225DBF9196DDE0C7FED4CD8F17
+dir-source tor26 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 86.59.21.38 86.59.21.38 80 443
+contact Peter Palfrader
+vote-digest 49482F6EB80BF1BD7D95FD2783339FD407754615
+dir-source longclaw 23D15D965BC35114467363C165C4F724B64B4F66 199.58.81.140 199.58.81.140 80 443
+contact Riseup Networks <collective at riseup dot net> - 1nNzekuHGGzBYRzyjfjFEfeisNvxkn4RT
+vote-digest 234A880CAF86B839E92EBABC837D3F1D4F729CA0
+dir-source bastet 27102BC123E7AF1D4741AE047E160C91ADC76B21 204.13.164.118 204.13.164.118 80 443
+contact stefani <nocat at readthefinemanual dot net>
+vote-digest F1C150923BA43C3833241B53657FC5E6D1DEBF44
+dir-source maatuska 49015F787433103580E3B66A1707A00E60F2D15B 171.25.193.9 171.25.193.9 443 80
+contact 4096R/1E8BF34923291265 Linus Nordberg <linus at nordberg.se>
+vote-digest E32521A257B2EED001870E2313322FC5208EB235
+dir-source moria1 D586D18309DED4CD6D57C18FDB97EFA96D330566 128.31.0.34 128.31.0.34 9131 9101
+contact 1024D/28988BF5 arma mit edu
+vote-digest 23A40AB1C8E29ECB10B9CC92B09F7174E4BC9F9E
+dir-source dizum E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212 194.109.206.212 80 443
+contact 1024R/8D56913D Alex de Joode <adejoode at sabotage.org>
+vote-digest 9EF2A2B9D3A62100611C73D146C1CAE90D555B91
+dir-source gabelmoo ED03BB616EB2F60BEC80151114BB25CEF515B226 131.188.40.189 131.188.40.189 80 443
+contact 4096R/261C5FBE77285F88FB0C343266C8C2D7C5AA446D Sebastian Hahn <tor at sebastianhahn.net> - 12NbRAjAG5U3LLWETSF7fSTcdaz32Mu5CN
+vote-digest 543D5B1CC570D7A955471365C12CB937F533708E
+dir-source Faravahar EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 154.35.175.225 154.35.175.225 80 443
+contact 0x0B47D56D Sina Rabbani (inf0) <sina redteam net>
+vote-digest 6A27A6FC019720E95E933F62A3ACAF0D9BB190C2
+r seele AAoQ1DAR6kkoo19hBAX5K0QztNw Gb2DCuQZtMbqEEc3DQpaxEbxdI0 2017-12-04 04:42:01 67.161.31.147 9001 0
+s Running Stable V2Dir Valid
+v Tor 0.3.0.10
+pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+w Bandwidth=28
+p reject 1-65535
+r CalyxInstitute14 ABG9JIWtRdmE7EFZyI/AZuXjMA4 YAphSl7mP4y1WqPU2emo3Y10jXc 2017-12-04 02:14:54 162.247.72.201 443 80
+s Exit Fast Guard HSDir Running Stable V2Dir Valid
+v Tor 0.3.1.5-alpha
+pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+w Bandwidth=6240
+p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544,554,563,636,706,749,873,902-904,981,989-995,1194,1220,1293,1500,1533,1677,1723,1755,1863,2082-2083,2086-2087,2095-2096,2102-2104,3128,3389,3690,4321,4643,5050,5190,5222-5223,5228,5900,6660-6669,6679,6697,8000,8008,8074,8080,8087-8088,8332-8333,8443,8888,9418,9999-10000,11371,12350,19294,19638,23456,33033,64738
+r havingtrouble AEJpGAnlKLJYyf4BKhxGhZFj8UI 2ZPgP5B/fLMCqHf+t2CMvWxM/rA 2017-12-04 04:20:22 159.203.42.254 9001 443
+a [2604:a880:cad:d0::862:4001]:9050
+s Fast Guard HSDir Running Stable V2Dir Valid
+v Tor 0.3.1.8
+pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2
+w Bandwidth=10400
+p reject 1-65535
+directory-footer
+bandwidth-weights Wbd=0 Wbe=0 Wbg=3488 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 Wem=10000 Wgb=10000 Wgd=0 Wgg=6512 Wgm=6512 Wmb=10000 Wmd=0 Wme=0 Wmg=3488 Wmm=10000
+directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E 4F82D69064739702C50C69436997FCB9C29944ED
+-----BEGIN SIGNATURE-----
+SyifLslDf0C7oOzZPTG7UM1P5sxUxVsnKAd3XsRg1UNhHWX+32tZJNlUjtauFlaj
+aQ/AH7cBu49tvomy04MKC+820KQ9wodSFKlb4N6NFwYLgIexy5PNtAtVhAtQszrO
+zym9drmjG39h6rU3SM7GoA7M38K5bv7WauhlV5L3aQClmSOxRY9KxJ+qyid7jfGj
+zeLhxAM1rGrLLoo6nsyifEfD3u1QLNQC1BVLAOZ4wuT2mCSCCSmHN6L//9kkcqh1
+yrPvuJsr/9aP6ibWMav2pTSYXB8Hblps3Q8yBo20O8b8p5cMzDeBJPvu4Vi/ccQ1
+rZVZAM1ercsa1K7c3ajwGQ==
+-----END SIGNATURE-----
+directory-signature 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 6AEC757516B142855BCE29708EBDA965AC3DD06A
+-----BEGIN SIGNATURE-----
+D9hlAHJHDwOs+KesGHmLyKvifVer+LnLjLTNfEGjQ8ox5/w/dXrQ9GWeeA3AndbQ
+DSB2YXrE6TflhIu59RxlPQlR+NAoZ79XdkM6gGzWRBM3lAVcRKZu9kmGVhLqckDj
+OFeMKzrE1PAzDpJWQ7S2gaQHPg2SzCNOln+zHFdFnsShdqo1G7JFJhnew0qexy/g
+WMwYg20ToGRb1kJvwRdQyU+cOgNgUDb83M07W5SpuUZPJdeKPknFR5e4p/YykeOe
+ZrdYFInd+pwBkvTGD7iA/tezEkG6fIp+UFoN+g5R8n4D07k71JRYyyNW+1dLzpGw
+Kdbh62lp/wGN98PotBvG3ycBx31CZTNAQRhPYAMBguWCiLN4mTyzKeF2VwscJUe8
+r3pHkUDaSVWXATndURSmhCdF1z+F8PwSsLXw3B7Hgag+DKnVUOZZFdjnR+p/8RwQ
+/QQdmHHHE1TDao4b1nA6KmSjVjemNDPN0fKtFcDCzjvoosUF2QTLKYh/AncHH8f+
+-----END SIGNATURE-----
+directory-signature 23D15D965BC35114467363C165C4F724B64B4F66 D632ADF38603F34EAFCB461C4DA5F67EF62BE7EC
+-----BEGIN SIGNATURE-----
+nLvLwOuLcSlpkDXyIP22stte72uA/AKnG5wt/1X2Iy6E5kwNCoSCf6Zmr7Xuh6Oq
+4t4KGfkrBW8rrp02ap/nACDB8IP+3xf75AcdrEReVDCXBT7jzUY1KHBPCB3pxaFp
+48jy63r5VgRdqGZXuctZ1DCzF/AfIcWkw5eeXrKtkWH8RlKXgVKm5LOXzvCoscUe
+5eRBzmC1GCPB7pPbQZVWvfvZTB1AulHWcpri8AIBrmalGtX5VTpEfCFXsxnL0Od1
+ibW5JfUXse9Vl5tS/iBnFXfNmNNT46/EcgniB9RkspMmF2fopEaXdutF29uN6ERu
+IBPpR3n9MVTEAnN5uhOFLw==
+-----END SIGNATURE-----
+directory-signature 27102BC123E7AF1D4741AE047E160C91ADC76B21 88E1BDDC8E6363355910D086B37B562F81674DC1
+-----BEGIN SIGNATURE-----
+Op1HXNiCoZaYwexT9Qt2lYa8/DzFZ5twMlku+KpbNJYQnpRsshFr/BUuWkXDGjuX
+ieKs5UkIpJXopyXKR4uZ5htHJAT8ukRjrfNS+43jsbGtoPoD/plyeZsBjyXUk8WY
+CAHTkt4n3t8xnxrTnIWL+Uv0qr39H7WOGXdbkqamEOSUFoGaVp66uNTYGJGssRRj
+G94MdyBGP1hDZTRO3Bf9ozH+Eg+Xl1JAkST2DP4kJXIpCcjRqBeFSMly7SQSkUkv
+oIvLWgfHpnUxqZmE2/T/rP6Sr85CaYfZI73CEQXiSxZbMf6HV0QjlKfrlwWt7KoL
+TPOzFsdHCoGQ/9mzk3272w==
+-----END SIGNATURE-----
+directory-signature 49015F787433103580E3B66A1707A00E60F2D15B C9C96501FD35732C45C424B74CC32739991B1F51
+-----BEGIN SIGNATURE-----
+vFYSlFkWhCcRfYmWAXUZKVTuBWP7vEui1abX6uDeCJticVnfvTdLNHM6qmFW2XcO
+4G6BARj7ukPW/rXjF6+SJ3JWSf0I7u7KLfozo4G1yiEknuIh1egOCwp2SRKgc6ZC
+kfhGM/0N3fXAg7JSmOcM59HMQyk7ky3LbPAOaUDhnSkByo5QDh4b5eSxxAJ87Hif
+4emyDbc49mqTF3TP+sB9Mqtp8zltNPMuhix2CqT4dSTyu/j3YdRVQuKHUu7fWBPn
+NdnLLW0HI/rMjKQ7nCKx6VOEHu9HeYkCxQiMdfiO9LmlXGBeNcNKWt7htS9FAAiW
+1KcGg5KAVeRAIcWCK14xFg==
+-----END SIGNATURE-----
+directory-signature D586D18309DED4CD6D57C18FDB97EFA96D330566 6E44451E3F1CEB435E4D95C1F8B12AA022BB34CF
+-----BEGIN SIGNATURE-----
+W8pGpRMHo03xiNHJ6i6ubF8VvPH39sFlZ7N6DwbZWHCn8zU13tdgpHaIV+REGjN3
+2/cd0s3S7sABhuaMyiGVUVY8jrEM4zm0x8ORfAS+TKDQ6FREJ9aGCP5VRMLKnOU5
+zEC8qHzjAMLlpaHAOlvjv+b2DjYwQn5L9dQZaGBn0sa4CrOn/Bvd/yZE7/XYpUfn
+b8il6DcmpSMrnBvYwQoCSnfVogyP8lh2vJGbS9HnWjta9utyRVPD3c6j50k2k6Kw
+8PvGRYR4GfyMU7TREU8d3YObtZ28cwDfiZBBPk2fZvYeaJIIjvZhuKfQPZI5VUJX
+rzILtPiBqDwogbMOar2HFQ==
+-----END SIGNATURE-----
+directory-signature E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 109A865D7DBE58367C120353CBE9947EE263695A
+-----BEGIN SIGNATURE-----
+gw3VRZAppEw3+USdqj0SHS3FXtZjkWXAv2ym9HwEIEn8gEfTzoFl1TkGTKCkfrt4
+sjJ/x5jOu9+I8feH9hGY9JYcnEFQM9kALjdBUffw4hvAwZ3SfszkwUbwAQZsnQOE
+bFhMu+flpq6XgdqTGD9aalJaW2VrKo+321LHcucyw7Tl/Rf3Q8YGTF7YEIO1vYT7
+/AS1t7n7f58/3Gk5cLhLgnGbK+PaAKq0rdc8RAlCv5Wdhg5pY7W8ipKQWqM9gFEG
+5e0GbVq8UygsljfvH1ABosAY/Y4I5RCH5gLsEzDyM4K4GIkVduUHlywRVGI47ZWC
+thxn3L/3WPGAjaT6RGBcHg==
+-----END SIGNATURE-----
+directory-signature ED03BB616EB2F60BEC80151114BB25CEF515B226 28EB6C6635F1DB90BB60AF8B43F1676E892BB30B
+-----BEGIN SIGNATURE-----
+Mld9hxzoHSIa42Sct3FgcQpFV0wbzE3HkUYEfIsJqA1TwD7PRo9QaCAd85lltnnN
+SNrNd1YqqX15fsFhB16ZeR3D2LH6xOQx2FyyChqXAzVPJXrPlUFR5BgQ2RPjxd1N
+H8nXgsIbBoZLoMLV7MRQAmY19a4H2X2iiUqRoo2gdH6glqhCFKVduCy0/0KZ1/Pu
+AhIVL/P3dVwtccv44/rA4dKKjFi1CaUZhO6bbmxQjhYkuxquz6PkBlLpKUW2cV/K
+e75Qy+dumbnPfVR4N8QdAebXo8g3Mf9MyWJ+lbLdBKFMuUUi9QexJUOVtOjWVzxI
+dOqjEkQbb5QNk4acB1u9sA==
+-----END SIGNATURE-----
+directory-signature EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 E861D5367EE5A469892D3FE6B2A25218FBA133FC
+-----BEGIN SIGNATURE-----
+DoQywyhompXommfwML3lgHbt0r4G4ouVpqbyfmT+HUWnG0fXdGRvlrZfCcocyzNK
+q3c9rvVw/x4FDkRVgc/gU3HYKFRHr2q4CD6lMUwny1WWhibKiKU5HfvfbZ+84gGj
+T3m9WBd1v9TQFkYq0R1dOY+6t18FgFQesLB6YVw9H+yYxbNlUGn+sECmkorzo6XJ
+sv4QxrhEATzMmv1SFA4pfnppq4al2wCW3u1myR1Ufv8wsaXts/z4AECzfOLDP6Z9
+/DMC8akN+XeIlu5ghrM2IyBXXT+yIEM85VYO8Y7Es5O2/Vhv2QbbcnbUYuScV1ap
+soP2uHArMaoA+qk4lVkw0A==
+-----END SIGNATURE-----
diff --git a/modules/ipv6servers/src/test/resources/descriptors/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part b/modules/ipv6servers/src/test/resources/descriptors/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part
new file mode 100644
index 0000000..659548e
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part
@@ -0,0 +1,12 @@
+ at type bridge-network-status 1.2
+published 2017-12-04 19:05:07
+flag-thresholds stable-uptime=1105427 stable-mtbf=4042454 fast-speed=53000 guard-wfu=98.000% guard-tk=691200 guard-bw-inc-exits=512000 guard-bw-exc-exits=512000 enough-mtbf=1 ignoring-advertised-bws=0
+fingerprint 1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1
+r Unnamed ACzV/+Qobny0/Tk7Jodk8cP+ME4 AbLK37zA6+UPOVhjZlrDdtJfCO0 2017-12-04 16:28:59 10.239.66.98 56676 0
+s Fast HSDir Running Stable V2Dir Valid
+w Bandwidth=63
+p reject 1-65535
+r Unnamed ADXqKmHijTlfCArKIkRTlJDnCVA tSDkh5Xc+Mh6tTPCYO6fivU/Apk 2017-12-04 18:01:53 10.237.141.119 51916 0
+s V2Dir Valid
+w Bandwidth=18
+p reject 1-65535
diff --git a/modules/ipv6servers/src/test/resources/descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb b/modules/ipv6servers/src/test/resources/descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb
new file mode 100644
index 0000000..2952ee9
--- /dev/null
+++ b/modules/ipv6servers/src/test/resources/descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb
@@ -0,0 +1,18 @@
+ at type bridge-server-descriptor 1.2
+router EnigmaDMZ 10.111.225.186 52121 0 0
+master-key-ed25519 IpgU7WgO6uxWT8BtEuNhKtH+S+aOOyttZa5kWqVSya8
+platform Tor 0.2.9.12 on Linux
+proto Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2
+published 2017-12-01 11:16:37
+fingerprint 835C E613 1471 0B08 B680 970C AABC 864A 0C00 04B4
+uptime 10918
+bandwidth 10240000 12288000 58783
+extra-info-digest 27FBC6A032A4BF21EF96B8753C67DC8E4B7E456B eshsUBj7pvOpQlYUQgjQt5emrtnqtJUZi8E6JZ4gsvg
+hidden-service-dir
+contact somebody
+ntor-onion-key EOsXa08KZ2cpg8LzsBw531ymC0ixg6pjuwaGLYV6WgI=
+reject *:*
+ipv6-policy accept 1-65535
+tunnelled-dir-server
+router-digest-sha256 /P9VVoa64l5C0G5o0dYbTa975H1Uk9xizCdUgXTVk5A
+router-digest 64DD486D89AF14027C9A7B4347A94B74DDDB5CDB
diff --git a/modules/ipv6servers/src/test/sql/test-ipv6servers.sql b/modules/ipv6servers/src/test/sql/test-ipv6servers.sql
new file mode 100644
index 0000000..7e5ca2e
--- /dev/null
+++ b/modules/ipv6servers/src/test/sql/test-ipv6servers.sql
@@ -0,0 +1,196 @@
+-- Copyright 2017 The Tor Project
+-- See LICENSE for licensing information
+
+-- Hint: You'll need pgTAP in order to run these tests!
+
+CREATE EXTENSION IF NOT EXISTS pgtap;
+
+BEGIN;
+
+SELECT plan(89);
+
+-- Make sure that the server_descriptors table is as expected.
+SELECT has_table('server_descriptors');
+SELECT has_column('server_descriptors', 'descriptor_digest_sha1');
+SELECT col_type_is('server_descriptors', 'descriptor_digest_sha1', 'bytea');
+SELECT col_is_pk('server_descriptors', 'descriptor_digest_sha1');
+SELECT has_column('server_descriptors', 'advertised_bandwidth_bytes');
+SELECT col_type_is('server_descriptors', 'advertised_bandwidth_bytes', 'integer');
+SELECT col_not_null('server_descriptors', 'advertised_bandwidth_bytes');
+SELECT has_column('server_descriptors', 'announced_ipv6');
+SELECT col_type_is('server_descriptors', 'announced_ipv6', 'boolean');
+SELECT col_not_null('server_descriptors', 'announced_ipv6');
+SELECT has_column('server_descriptors', 'exiting_ipv6_relay');
+SELECT col_type_is('server_descriptors', 'exiting_ipv6_relay', 'boolean');
+SELECT col_not_null('server_descriptors', 'exiting_ipv6_relay');
+
+-- Make sure that the server enum is as expected.
+SELECT has_enum('server_enum');
+SELECT enum_has_labels('server_enum', ARRAY['relay', 'bridge']);
+
+-- Make sure that the statuses table is as expected.
+SELECT has_table('statuses');
+SELECT has_column('statuses', 'status_id');
+SELECT col_type_is('statuses', 'status_id', 'integer');
+SELECT col_is_pk('statuses', 'status_id');
+SELECT has_column('statuses', 'server');
+SELECT col_type_is('statuses', 'server', 'server_enum');
+SELECT col_not_null('statuses', 'server');
+SELECT has_column('statuses', 'valid_after');
+SELECT col_type_is('statuses', 'valid_after', 'timestamp without time zone');
+SELECT col_not_null('statuses', 'valid_after');
+SELECT has_column('statuses', 'running_count');
+SELECT col_type_is('statuses', 'running_count', 'integer');
+SELECT col_not_null('statuses', 'running_count');
+SELECT col_is_unique('statuses', ARRAY['server', 'valid_after']);
+
+-- Make sure that the status_entries table is as expected.
+SELECT has_table('status_entries');
+SELECT has_column('status_entries', 'status_id');
+SELECT col_type_is('status_entries', 'status_id', 'integer');
+SELECT fk_ok('status_entries', 'status_id', 'statuses', 'status_id');
+SELECT col_not_null('status_entries', 'status_id');
+SELECT has_column('status_entries', 'descriptor_digest_sha1');
+SELECT col_type_is('status_entries', 'descriptor_digest_sha1', 'bytea');
+SELECT col_not_null('status_entries', 'descriptor_digest_sha1');
+SELECT has_column('status_entries', 'guard_relay');
+SELECT col_type_is('status_entries', 'guard_relay', 'boolean');
+SELECT col_not_null('status_entries', 'guard_relay');
+SELECT has_column('status_entries', 'exit_relay');
+SELECT col_type_is('status_entries', 'exit_relay', 'boolean');
+SELECT col_not_null('status_entries', 'exit_relay');
+SELECT has_column('status_entries', 'reachable_ipv6_relay');
+SELECT col_type_is('status_entries', 'reachable_ipv6_relay', 'boolean');
+SELECT col_not_null('status_entries', 'reachable_ipv6_relay');
+SELECT col_is_unique('status_entries', ARRAY['status_id', 'descriptor_digest_sha1']);
+SELECT hasnt_pk('status_entries');
+
+-- Make sure that the aggregated_ipv6 table is as expected.
+SELECT has_table('aggregated_ipv6');
+SELECT has_column('aggregated_ipv6', 'status_id');
+SELECT col_type_is('aggregated_ipv6', 'status_id', 'integer');
+SELECT fk_ok('aggregated_ipv6', 'status_id', 'statuses', 'status_id');
+SELECT col_not_null('aggregated_ipv6', 'status_id');
+SELECT has_column('aggregated_ipv6', 'guard_relay');
+SELECT col_type_is('aggregated_ipv6', 'guard_relay', 'boolean');
+SELECT col_not_null('aggregated_ipv6', 'guard_relay');
+SELECT has_column('aggregated_ipv6', 'exit_relay');
+SELECT col_type_is('aggregated_ipv6', 'exit_relay', 'boolean');
+SELECT col_not_null('aggregated_ipv6', 'exit_relay');
+SELECT has_column('aggregated_ipv6', 'reachable_ipv6_relay');
+SELECT col_type_is('aggregated_ipv6', 'reachable_ipv6_relay', 'boolean');
+SELECT col_not_null('aggregated_ipv6', 'reachable_ipv6_relay');
+SELECT has_column('aggregated_ipv6', 'announced_ipv6');
+SELECT col_type_is('aggregated_ipv6', 'announced_ipv6', 'boolean');
+SELECT col_not_null('aggregated_ipv6', 'announced_ipv6');
+SELECT has_column('aggregated_ipv6', 'exiting_ipv6_relay');
+SELECT col_type_is('aggregated_ipv6', 'exiting_ipv6_relay', 'boolean');
+SELECT col_not_null('aggregated_ipv6', 'exiting_ipv6_relay');
+SELECT has_column('aggregated_ipv6', 'server_count_sum');
+SELECT col_type_is('aggregated_ipv6', 'server_count_sum', 'integer');
+SELECT col_not_null('aggregated_ipv6', 'server_count_sum');
+SELECT has_column('aggregated_ipv6', 'advertised_bandwidth_bytes_sum');
+SELECT col_type_is('aggregated_ipv6', 'advertised_bandwidth_bytes_sum', 'bigint');
+SELECT col_not_null('aggregated_ipv6', 'advertised_bandwidth_bytes_sum');
+SELECT col_is_unique('aggregated_ipv6',
+ ARRAY['status_id', 'guard_relay', 'exit_relay', 'announced_ipv6',
+ 'exiting_ipv6_relay', 'reachable_ipv6_relay']);
+
+-- Truncate all tables for subsequent tests. This happens inside a transaction,
+-- so we're not actually truncating anything.
+TRUNCATE server_descriptors, statuses, status_entries, aggregated_ipv6;
+
+-- Make sure that the aggregated_ipv6 table is empty.
+SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 0;',
+ 'At the beginning, the aggregated_ipv6 table should be empty.');
+
+-- And make sure that running the aggregate_ipv6() function does not change that.
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 0;',
+ 'Even after aggregating, the aggregated_ipv6 table should be empty.');
+
+-- Insert a server descriptor, then try again.
+INSERT INTO server_descriptors (descriptor_digest_sha1, advertised_bandwidth_bytes, announced_ipv6,
+ exiting_ipv6_relay) VALUES ('\x00', 100, FALSE, TRUE);
+
+-- Try to aggregate, though there's not much to aggregate without corresponding
+-- entry in status_entries.
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 0;',
+ 'At the beginning, the aggregated_ipv6 table should be empty.');
+
+-- Attempt to add an entry to status_entries, but without having inserted an
+-- entry into statuses first.
+SELECT throws_ok('INSERT INTO status_entries (status_id, descriptor_digest_sha1) '
+ || 'VALUES (1, ''\x00'');');
+
+-- Try again in the correct order.
+INSERT INTO statuses (server, valid_after, running_count)
+ VALUES ('relay'::server_enum, '2017-12-04 00:00:00'::TIMESTAMP, 1);
+INSERT INTO status_entries
+ SELECT status_id, '\x00', TRUE, FALSE, FALSE FROM statuses;
+
+-- Now aggregate and see how the status_entries entry gets moved over to the
+-- aggregated_ipv6 table. However, it's just one status, so it doesn't show in the
+-- output view yet.
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM status_entries;', 'SELECT 0;',
+ 'status_entries should not contain aggregated row anymore.');
+SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 1;',
+ 'aggregated_ipv6 table should contain exactly one row now.');
+SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 0;',
+ 'ipv6servers should not contain any results yet.');
+
+-- Try to aggregate once more, but that shouldn't change anything.
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM status_entries;', 'SELECT 0;',
+ 'status_entries should still be empty.');
+SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 1;',
+ 'aggregated_ipv6 table should still contain exactly one row.');
+
+-- Insert statuses for 3 days, of which the last 2 will be cut off in the
+-- output.
+INSERT INTO statuses (server, valid_after, running_count)
+ SELECT 'relay'::server_enum, GENERATE_SERIES('2017-12-04 01:00:00'::TIMESTAMP,
+ '2017-12-06 23:00:00', '1 hour'), 1;
+
+-- Insert the same relay as entries for all statuses except the one that we
+-- added earlier and that is already contained in the aggregated_ipv6 table. (In the
+-- actual import code we'd first check that we already inserted the status and
+-- then not import any entries from it.)
+INSERT INTO status_entries
+ SELECT status_id, '\x00', TRUE, FALSE, FALSE FROM statuses
+ WHERE valid_after > '2017-12-04 00:00:00'::TIMESTAMP;
+
+-- Aggregate, then look at the output.
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM status_entries;', 'SELECT 0;',
+ 'status_entries should not contain anything anymore.');
+SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 72;',
+ 'aggregated_ipv6 table should contain one row per status.');
+SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 1;',
+ 'ipv6servers should now contain a results line.');
+
+-- Insert another status entry for which there is no corresponding server
+-- descriptor to observe how the results line disappears again (because we
+-- require 99.9% of server descriptors to be present). This is just a test case
+-- that would not occur in practice, because we wouLdn't retroactively add new
+-- status entries. It's just server descriptors that we might add later.
+INSERT INTO status_entries
+ SELECT status_id, '\x01', FALSE, FALSE, FALSE FROM statuses;
+UPDATE statuses SET running_count = 2;
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 0;',
+ 'ipv6servers should be empty, because of missing server descriptors.');
+
+-- Okay, okay, provide the missing server descriptor.
+INSERT INTO server_descriptors (descriptor_digest_sha1, advertised_bandwidth_bytes, announced_ipv6,
+ exiting_ipv6_relay) VALUES ('\x01', 100, TRUE, TRUE);
+SELECT aggregate_ipv6();
+SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 2;',
+ 'ipv6servers should be non-empty again.');
+
+SELECT * FROM finish();
+
+ROLLBACK;
+
diff --git a/shared/bin/20-run-ipv6servers-stats.sh b/shared/bin/20-run-ipv6servers-stats.sh
new file mode 100755
index 0000000..5d7bd13
--- /dev/null
+++ b/shared/bin/20-run-ipv6servers-stats.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+cd modules/ipv6servers/
+ant run | grep "\[java\]"
+cd ../../
+
diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib
deleted file mode 160000
index 79a4b98..0000000
--- a/src/submods/metrics-lib
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 79a4b9866f2a342159bd0811d83e9ec62169c6d9
More information about the tor-commits
mailing list