commit 73205cc3df76d0ca8d2fc62f26e2359093e73c7c Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Dec 6 11:39:32 2017 +0100

Add servers-ipv6 module.

Implements #24218. --- modules/ipv6servers/.gitignore | 3 + modules/ipv6servers/build.xml | 21 ++ .../metrics/stats/ipv6servers/Configuration.java | 18 ++ .../metrics/stats/ipv6servers/Database.java | 218 +++++++++++++++++++++ .../torproject/metrics/stats/ipv6servers/Main.java | 106 ++++++++++ .../metrics/stats/ipv6servers/OutputLine.java | 75 +++++++ .../stats/ipv6servers/ParsedNetworkStatus.java | 47 +++++ .../stats/ipv6servers/ParsedServerDescriptor.java | 26 +++ .../metrics/stats/ipv6servers/Parser.java | 99 ++++++++++ .../metrics/stats/ipv6servers/Writer.java | 37 ++++ .../src/main/resources/init-ipv6servers.sql | 123 ++++++++++++ .../stats/ipv6servers/ParsedNetworkStatusTest.java | 140 +++++++++++++ .../ipv6servers/ParsedServerDescriptorTest.java | 97 +++++++++ .../000a7fe20a17bf5d9839a126b1dff43f998aac6f | 16 ++ .../0018ab4f2f28af683d52f06407edbf7ce1bd3b7d | 51 +++++ .../0041dbf9fe846f9765882f7dc8332f94b709e35a | 19 ++ .../01003df74972ce952ebfa390f468ef63c50efa25 | 189 ++++++++++++++++++ .../018c1229d5f56eebfc1d709d4692673d098800e8 | 54 +++++ .../descriptors/2017-12-04-20-00-00-consensus.part | 149 ++++++++++++++ ...7-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part | 12 ++ .../64dd486d89af14027c9a7b4347a94b74dddb5cdb | 18 ++ .../ipv6servers/src/test/sql/test-ipv6servers.sql | 196 ++++++++++++++++++ shared/bin/20-run-ipv6servers-stats.sh | 5 + src/submods/metrics-lib | 1 - 24 files changed, 1719 insertions(+), 1 deletion(-)

diff --git a/modules/ipv6servers/.gitignore b/modules/ipv6servers/.gitignore new file mode 100644 index 0000000..c8e90bb --- /dev/null +++ b/modules/ipv6servers/.gitignore @@ -0,0 +1,3 @@ +/stats/ +/status/ + diff --git a/modules/ipv6servers/build.xml b/modules/ipv6servers/build.xml new file mode 100644 index 0000000..736b579 --- /dev/null +++ b/modules/ipv6servers/build.xml @@ -0,0 +1,21 @@ +<project default="run" name="ipv6servers" basedir="."> + + <property name="mainclass" + value="org.torproject.metrics.stats.ipv6servers.Main" /> + + <include file="../../shared/build-base.xml" as="basetask"/> + <target name="clean" depends="basetask.clean"/> + <target name="compile" depends="basetask.compile"/> + <target name="test" depends="basetask.test"/> + <target name="run" depends="basetask.run"/> + + <path id="classpath"> + <pathelement path="${classes}"/> + <path refid="base.classpath" /> + <fileset dir="${libs}"> + <include name="postgresql-jdbc3-9.2.jar"/> + </fileset> + </path> + +</project> + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java new file mode 100644 index 0000000..dffcdf6 --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Configuration.java @@ -0,0 +1,18 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +/** Configuration options parsed from Java properties with reasonable hard-coded + * defaults. */ +class Configuration { + static String descriptors = System.getProperty("descriptors", + "../../shared/in/"); + static String database = System.getProperty("database", + "jdbc:postgresql:ipv6servers"); + static String history = System.getProperty("history", + "status/read-descriptors"); + static String output = System.getProperty("output", + "stats/ipv6servers.csv"); +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java new file mode 100644 index 0000000..c334263 --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Database.java @@ -0,0 +1,218 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; + +/** Database wrapper to connect to the database, insert data, run the stored + * procedure for aggregating data, and query aggregated data as output. */ +class Database { + + /** Database connection string. */ + private String jdbcString; + + /** Connection object for all interactions with the database. */ + private Connection connection; + + /** Prepared statement for finding out whether a given server descriptor is + * already contained in the server_descriptors table. */ + private PreparedStatement psServerDescriptorsSelect; + + /** Prepared statement for inserting a server descriptor into the + * server_descriptors table. */ + private PreparedStatement psServerDescriptorsInsert; + + /** Prepared statement for checking whether a status has been inserted into + * the statuses table before. */ + private PreparedStatement psStatusesSelect; + + /** Prepared statement for inserting a status (without entries, yet) into + * the statuses table. */ + private PreparedStatement psStatusesInsert; + + /** Prepared statement for inserting a status entry into the status_entries + * table. */ + private PreparedStatement psStatusEntriesInsert; + + /** Create a new Database instance and prepare for inserting or querying + * data. */ + Database(String jdbcString) throws SQLException { + this.jdbcString = jdbcString; + this.connect(); + this.prepareStatements(); + } + + private void connect() throws SQLException { + this.connection = DriverManager.getConnection(this.jdbcString); + this.connection.setAutoCommit(false); + } + + private void prepareStatements() throws SQLException { + this.psServerDescriptorsSelect = this.connection.prepareStatement( + "SELECT EXISTS (SELECT 1 FROM server_descriptors " + + "WHERE descriptor_digest_sha1 = decode(?, 'hex'))"); + this.psServerDescriptorsInsert = this.connection.prepareStatement( + "INSERT INTO server_descriptors (descriptor_digest_sha1, " + + "advertised_bandwidth_bytes, announced_ipv6, exiting_ipv6_relay) " + + "VALUES (decode(?, 'hex'), ?, ?, ?)"); + this.psStatusesSelect = this.connection.prepareStatement( + "SELECT EXISTS (SELECT 1 FROM statuses " + + "WHERE server = CAST(? AS server_enum) AND valid_after = ?)"); + this.psStatusesInsert = this.connection.prepareStatement( + "INSERT INTO statuses (server, valid_after, running_count) " + + "VALUES (CAST(? AS server_enum), ?, ?)", + Statement.RETURN_GENERATED_KEYS); + this.psStatusEntriesInsert = this.connection.prepareStatement( + "INSERT INTO status_entries (status_id, descriptor_digest_sha1, " + + "guard_relay, exit_relay, reachable_ipv6_relay) " + + "VALUES (?, decode(?, 'hex'), ?, ?, ?)"); + } + + /** Insert a server descriptor into the server_descriptors table. */ + void insertServerDescriptor( + ParsedServerDescriptor parsedServerDescriptor) throws SQLException { + this.psServerDescriptorsSelect.clearParameters(); + this.psServerDescriptorsSelect.setString(1, + parsedServerDescriptor.digest); + try (ResultSet rs = psServerDescriptorsSelect.executeQuery()) { + if (rs.next()) { + if (rs.getBoolean(1)) { + /* Server descriptor is already contained. */ + return; + } + } + } + this.psServerDescriptorsInsert.clearParameters(); + this.psServerDescriptorsInsert.setString(1, + parsedServerDescriptor.digest); + this.psServerDescriptorsInsert.setInt(2, + parsedServerDescriptor.advertisedBandwidth); + this.psServerDescriptorsInsert.setBoolean(3, + parsedServerDescriptor.announced); + this.psServerDescriptorsInsert.setBoolean(4, + parsedServerDescriptor.exiting); + this.psServerDescriptorsInsert.execute(); + } + + /** Insert a status and all contained entries into the statuses and + * status_entries table. */ + void insertStatus(ParsedNetworkStatus parsedNetworkStatus) + throws SQLException { + this.psStatusesSelect.clearParameters(); + this.psStatusesSelect.setString(1, + parsedNetworkStatus.isRelay ? "relay" : "bridge"); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + this.psStatusesSelect.setTimestamp(2, + Timestamp.from(ZonedDateTime.of(parsedNetworkStatus.timestamp, + ZoneId.of("UTC")).toInstant()), calendar); + try (ResultSet rs = this.psStatusesSelect.executeQuery()) { + if (rs.next()) { + if (rs.getBoolean(1)) { + /* Status is already contained. */ + return; + } + } + } + int statusId = -1; + this.psStatusesInsert.clearParameters(); + this.psStatusesInsert.setString(1, + parsedNetworkStatus.isRelay ? "relay" : "bridge"); + this.psStatusesInsert.setTimestamp(2, + Timestamp.from(ZonedDateTime.of(parsedNetworkStatus.timestamp, + ZoneId.of("UTC")).toInstant()), calendar); + this.psStatusesInsert.setInt(3, parsedNetworkStatus.running); + this.psStatusesInsert.execute(); + try (ResultSet rs = this.psStatusesInsert.getGeneratedKeys()) { + if (rs.next()) { + statusId = rs.getInt(1); + } + } + if (statusId < 0) { + throw new SQLException("Could not retrieve auto-generated key for new " + + "statuses entry."); + } + for (ParsedNetworkStatus.Entry entry : parsedNetworkStatus.entries) { + this.psStatusEntriesInsert.clearParameters(); + this.psStatusEntriesInsert.setInt(1, statusId); + this.psStatusEntriesInsert.setString(2, entry.digest); + this.psStatusEntriesInsert.setBoolean(3, entry.guard); + this.psStatusEntriesInsert.setBoolean(4, entry.exit); + this.psStatusEntriesInsert.setBoolean(5, entry.reachable); + this.psStatusEntriesInsert.addBatch(); + } + this.psStatusEntriesInsert.executeBatch(); + } + + /** Call the aggregate() function to aggregate rows from the status_entries + * and server_descriptors tables into the aggregated table. */ + void aggregate() throws SQLException { + Statement st = this.connection.createStatement(); + st.executeQuery("SELECT aggregate_ipv6()"); + } + + /** Roll back any changes made in this execution. */ + void rollback() throws SQLException { + this.connection.rollback(); + } + + /** Commit all changes made in this execution. */ + void commit() throws SQLException { + this.connection.commit(); + } + + /** Query the servers_ipv6 view to obtain aggregated statistics. */ + Iterable<OutputLine> queryServersIpv6() throws SQLException { + List<OutputLine> statistics = new ArrayList<>(); + Statement st = this.connection.createStatement(); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), + Locale.US); + String queryString = "SELECT " + OutputLine.getColumnHeaders(", ") + + " FROM ipv6servers"; + try (ResultSet rs = st.executeQuery(queryString)) { + while (rs.next()) { + OutputLine outputLine = new OutputLine(); + outputLine.date = rs.getDate(OutputLine.Column.VALID_AFTER_DATE.name(), + calendar).toLocalDate(); + outputLine.server = rs.getString(OutputLine.Column.SERVER.name()); + outputLine.guard = rs.getString(OutputLine.Column.GUARD_RELAY.name()); + outputLine.exit = rs.getString(OutputLine.Column.EXIT_RELAY.name()); + outputLine.announced = rs.getString( + OutputLine.Column.ANNOUNCED_IPV6.name()); + outputLine.exiting = rs.getString( + OutputLine.Column.EXITING_IPV6_RELAY.name()); + outputLine.reachable = rs.getString( + OutputLine.Column.REACHABLE_IPV6_RELAY.name()); + outputLine.count = rs.getLong( + OutputLine.Column.SERVER_COUNT_SUM_AVG.name()); + outputLine.advertisedBandwidth = rs.getLong( + OutputLine.Column.ADVERTISED_BANDWIDTH_BYTES_SUM_AVG.name()); + if (rs.wasNull()) { + outputLine.advertisedBandwidth = null; + } + statistics.add(outputLine); + } + } + return statistics; + } + + /** Disconnect from the database. */ + void disconnect() throws SQLException { + this.connection.close(); + } +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java new file mode 100644 index 0000000..81433c0 --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Main.java @@ -0,0 +1,106 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Paths; +import java.sql.SQLException; + +/** Main class of the ipv6servers module that imports relevant parts from server + * descriptors and network statuses into a database, and exports aggregate + * statistics on IPv6 support to a CSV file. */ +public class Main { + + private static Logger log = LoggerFactory.getLogger(Main.class); + + /** Run the module. */ + public static void main(String[] args) throws Exception { + + log.info("Starting ipv6servers module."); + + log.info("Reading descriptors and inserting relevant parts into the " + + "database."); + DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); + File historyFile = new File(Configuration.history); + reader.setHistoryFile(historyFile); + Parser parser = new Parser(); + Database database = new Database(Configuration.database); + try { + for (Descriptor descriptor : reader.readDescriptors( + new File(Configuration.descriptors + + "recent/relay-descriptors/consensuses"), + new File(Configuration.descriptors + + "recent/relay-descriptors/server-descriptors"), + new File(Configuration.descriptors + + "recent/bridge-descriptors/statuses"), + new File(Configuration.descriptors + + "recent/bridge-descriptors/server-descriptors"), + new File(Configuration.descriptors + + "archive/relay-descriptors/consensuses"), + new File(Configuration.descriptors + + "archive/relay-descriptors/server-descriptors"), + new File(Configuration.descriptors + + "archive/bridge-descriptors/statuses"), + new File(Configuration.descriptors + + "archive/bridge-descriptors/server-descriptors"))) { + if (descriptor instanceof ServerDescriptor) { + database.insertServerDescriptor(parser.parseServerDescriptor( + (ServerDescriptor) descriptor)); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { + database.insertStatus(parser.parseRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor)); + } else if (descriptor instanceof BridgeNetworkStatus) { + database.insertStatus(parser.parseBridgeNetworkStatus( + (BridgeNetworkStatus) descriptor)); + } else { + log.debug("Skipping unknown descriptor of type {}.", + descriptor.getClass()); + } + } + + log.info("Aggregating database entries."); + database.aggregate(); + + log.info("Committing all updated parts in the database."); + database.commit(); + } catch (SQLException sqle) { + log.error("Cannot recover from SQL exception while inserting or " + + "aggregating data. Rolling back and exiting.", sqle); + database.rollback(); + database.disconnect(); + return; + } + reader.saveHistoryFile(historyFile); + + log.info("Querying aggregated statistics from the database."); + Iterable<OutputLine> output; + try { + output = database.queryServersIpv6(); + } catch (SQLException sqle) { + log.error("Cannot recover from SQL exception while querying. Not writing " + + "output file.", sqle); + return; + } finally { + database.disconnect(); + } + + log.info("Writing aggregated statistics to {}.", Configuration.output); + if (null != output) { + new Writer().write(Paths.get(Configuration.output), output); + } + + log.info("Terminating ipv6servers module."); + } +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java new file mode 100644 index 0000000..eba5f13 --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/OutputLine.java @@ -0,0 +1,75 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; + +/** Data object holding all parts of an output line. */ +class OutputLine { + + /** Column names used in the database and in the first line of the output + * file. */ + enum Column { + VALID_AFTER_DATE, SERVER, GUARD_RELAY, EXIT_RELAY, ANNOUNCED_IPV6, + EXITING_IPV6_RELAY, REACHABLE_IPV6_RELAY, SERVER_COUNT_SUM_AVG, + ADVERTISED_BANDWIDTH_BYTES_SUM_AVG + } + + /** Column headers joined together with the given delimiter. */ + static String getColumnHeaders(String delimiter) { + List<String> columnHeaders = new ArrayList<>(); + for (Column column : Column.values()) { + columnHeaders.add(column.toString()); + } + return String.join(delimiter, columnHeaders).toLowerCase(); + } + + /** Date. */ + LocalDate date; + + /** Server type, which can be "relay" or "bridge". */ + String server; + + /** Whether relays had the Guard flag ("t") or not ("f"). */ + String guard; + + /** Whether relays had the Exit flag ("t") or not ("f"). */ + String exit; + + /** Whether relays or bridges have announced an IPv6 address in their server + * descriptor ("t") or not ("f"). */ + String announced; + + /** Whether relays have announced a non-reject-all IPv6 exit policy in their + * server descriptor ("t") or not ("f"). */ + String exiting; + + /** Whether the directory authorities have confirmed IPv6 OR reachability by + * including an "a" line for a relay containing an IPv6 address. */ + String reachable; + + /** Number of relays or bridges matching the previous criteria. */ + long count; + + /** Total advertised bandwidth of all relays matching the previous + * criteria. */ + Long advertisedBandwidth; + + /** Format all fields in a single output line for inclusion in a CSV + * file. */ + @Override + public String toString() { + return String.format("%s,%s,%s,%s,%s,%s,%s,%s,%s", + date, server, emptyNull(guard), emptyNull(exit), emptyNull(announced), + emptyNull(exiting), emptyNull(reachable), emptyNull(count), + emptyNull(advertisedBandwidth)); + } + + private static String emptyNull(Object text) { + return null == text ? "" : text.toString(); + } +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatus.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatus.java new file mode 100644 index 0000000..f185250 --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatus.java @@ -0,0 +1,47 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +/** Data object holding all relevant parts parsed from a (relay or bridge) + * network status. */ +class ParsedNetworkStatus { + + /** Whether this is a relay network status as opposed to a bridge network + * status. */ + boolean isRelay; + + /** Valid-after time in case of relay network status and published time in + * case of bridge network status. */ + LocalDateTime timestamp; + + /** Number of relays or bridges with the Running flag. */ + int running = 0; + + /** Contained status entries. */ + List<Entry> entries = new ArrayList<>(); + + /** Data object holding all relevant parts from a network status entry. */ + static class Entry { + + /** Hex-encoded SHA-1 server descriptor digest. */ + String digest; + + /** Whether this relay has the Guard flag; false for bridges. */ + boolean guard; + + /** Whether this relay has the Exit flag (and not the BadExit flag at the + * same time); false for bridges. */ + boolean exit; + + /** Whether the directory authorities include an IPv6 address in this + * entry's "a" line, confirming the relay's reachability via IPv6; false for + * bridges. */ + boolean reachable; + } +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptor.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptor.java new file mode 100644 index 0000000..c8d0ceb --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptor.java @@ -0,0 +1,26 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +/** Data object holding all relevant parts parsed from a (relay or bridge) + * server descriptor. */ +class ParsedServerDescriptor { + + /** Hex-encoded SHA-1 server descriptor digest. */ + String digest; + + /** Advertised bandwidth bytes of this relay as the minimum of bandwidth rate, + * bandwidth burst, and observed bandwidth (if reported); 0 for bridges. */ + int advertisedBandwidth; + + /** Whether the relay or bridge announced an IPv6 address in an "or-address" + * line. */ + boolean announced; + + /** Whether the relay allows exiting via IPv6, which is the case if the + * server descriptor contains an "ipv6-policy" line that is not + * "ipv6-policy reject 1-65535"; false for bridges. */ + boolean exiting; +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java new file mode 100644 index 0000000..95b1d5a --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Parser.java @@ -0,0 +1,99 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.RelayServerDescriptor; +import org.torproject.descriptor.ServerDescriptor; + +import org.apache.commons.lang.StringUtils; + +import java.time.Instant; +import java.time.ZoneId; + +/** Parser that extracts all relevant parts from (relay and bridge) server + * descriptors and (relay and bridge) statuses and creates data objects for + * them. */ +class Parser { + + /** Parse a (relay or bridge) server descriptor. */ + ParsedServerDescriptor parseServerDescriptor( + ServerDescriptor serverDescriptor) { + ParsedServerDescriptor parsedDescriptor = new ParsedServerDescriptor(); + parsedDescriptor.digest = serverDescriptor.getDigestSha1Hex(); + for (String orAddress : serverDescriptor.getOrAddresses()) { + if (StringUtils.countMatches(orAddress, ":") >= 2) { + parsedDescriptor.announced = true; + break; + } + } + if (serverDescriptor instanceof RelayServerDescriptor) { + parsedDescriptor.advertisedBandwidth = + Math.min(serverDescriptor.getBandwidthRate(), + serverDescriptor.getBandwidthBurst()); + if (serverDescriptor.getBandwidthObserved() >= 0) { + parsedDescriptor.advertisedBandwidth = + Math.min(parsedDescriptor.advertisedBandwidth, + serverDescriptor.getBandwidthObserved()); + } + parsedDescriptor.exiting + = null != serverDescriptor.getIpv6DefaultPolicy() + && !("reject".equals(serverDescriptor.getIpv6DefaultPolicy()) + && "1-65535".equals(serverDescriptor.getIpv6PortList())); + } + return parsedDescriptor; + } + + /** Parse a relay network status. */ + ParsedNetworkStatus parseRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) throws Exception { + return this.parseStatus(true, consensus.getValidAfterMillis(), + consensus.getStatusEntries().values()); + } + + /** Parse a bridge network status. */ + ParsedNetworkStatus parseBridgeNetworkStatus(BridgeNetworkStatus status) + throws Exception { + return this.parseStatus(false, status.getPublishedMillis(), + status.getStatusEntries().values()); + } + + private ParsedNetworkStatus parseStatus(boolean isRelay, long timestampMillis, + Iterable<NetworkStatusEntry> entries) { + ParsedNetworkStatus parsedStatus = new ParsedNetworkStatus(); + parsedStatus.isRelay = isRelay; + parsedStatus.timestamp = Instant.ofEpochMilli(timestampMillis) + .atZone(ZoneId.of("UTC")).toLocalDateTime(); + for (NetworkStatusEntry entry : entries) { + if (!entry.getFlags().contains("Running")) { + continue; + } + parsedStatus.running++; + } + for (NetworkStatusEntry entry : entries) { + if (!entry.getFlags().contains("Running")) { + continue; + } + ParsedNetworkStatus.Entry parsedEntry = new ParsedNetworkStatus.Entry(); + parsedEntry.digest = entry.getDescriptor().toLowerCase(); + if (isRelay) { + parsedEntry.guard = entry.getFlags().contains("Guard"); + parsedEntry.exit = entry.getFlags().contains("Exit") + && !entry.getFlags().contains("BadExit"); + parsedEntry.reachable = false; + for (String orAddress : entry.getOrAddresses()) { + if (StringUtils.countMatches(orAddress, ":") >= 2) { + parsedEntry.reachable = true; + break; + } + } + } + parsedStatus.entries.add(parsedEntry); + } + return parsedStatus; + } +} + diff --git a/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java new file mode 100644 index 0000000..96f8a8d --- /dev/null +++ b/modules/ipv6servers/src/main/java/org/torproject/metrics/stats/ipv6servers/Writer.java @@ -0,0 +1,37 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** Writer that takes output line objects and writes them to a file, preceded + * by a column header line. */ +class Writer { + + /** Write output lines to the given file. */ + void write(Path filePath, Iterable<OutputLine> outputLines) + throws IOException { + File parentFile = filePath.toFile().getParentFile(); + if (null != parentFile && !parentFile.exists()) { + boolean madeDirs = parentFile.mkdirs(); + if (!madeDirs) { + throw new IOException("Unable to create parent directory of output " + + "file. Not writing this file."); + } + } + List<String> formattedOutputLines = new ArrayList<>(); + formattedOutputLines.add(OutputLine.getColumnHeaders(",")); + for (OutputLine line : outputLines) { + formattedOutputLines.add(line.toString()); + } + Files.write(filePath, formattedOutputLines, StandardCharsets.UTF_8); + } +} + diff --git a/modules/ipv6servers/src/main/resources/init-ipv6servers.sql b/modules/ipv6servers/src/main/resources/init-ipv6servers.sql new file mode 100644 index 0000000..6a72167 --- /dev/null +++ b/modules/ipv6servers/src/main/resources/init-ipv6servers.sql @@ -0,0 +1,123 @@ +-- Copyright 2017 The Tor Project +-- See LICENSE for licensing information + +-- Table of all relevant parts contained in relay or bridge server descriptors. +-- We're not deleting from this table, because we can never be sure that we +-- won't import a previously missing status that we'll want to match against +-- existing server descriptors. +CREATE TABLE server_descriptors ( + descriptor_digest_sha1 BYTEA PRIMARY KEY, + advertised_bandwidth_bytes INTEGER NOT NULL, + announced_ipv6 BOOLEAN NOT NULL, + exiting_ipv6_relay BOOLEAN NOT NULL +); + +-- Enumeration type for servers, which can be either relays or bridges. +CREATE TYPE server_enum AS ENUM ('relay', 'bridge'); + +-- Table of all relay or bridge statuses. We're not deleting from this table. +CREATE TABLE statuses ( + status_id SERIAL PRIMARY KEY, + server server_enum NOT NULL, + valid_after TIMESTAMP WITHOUT TIME ZONE NOT NULL, + running_count INTEGER NOT NULL, + UNIQUE (server, valid_after) +); + +-- Table of relay or bridge status entries. Unlike previous tables, we're +-- deleting from this table after aggregating rows into the aggregated table. +-- Otherwise this table would grow too large over time. +CREATE TABLE status_entries ( + status_id INTEGER REFERENCES statuses (status_id) NOT NULL, + descriptor_digest_sha1 BYTEA NOT NULL, + guard_relay BOOLEAN NOT NULL, + exit_relay BOOLEAN NOT NULL, + reachable_ipv6_relay BOOLEAN NOT NULL, + UNIQUE (status_id, descriptor_digest_sha1) +); + +-- Table of joined and aggregated server_descriptors and status_entries rows. +-- For a given status and combination of flags and IPv6 capabilities, we count +-- the number of servers and advertised bandwidth bytes. +CREATE TABLE aggregated_ipv6 ( + status_id INTEGER REFERENCES statuses (status_id) NOT NULL, + guard_relay BOOLEAN NOT NULL, + exit_relay BOOLEAN NOT NULL, + announced_ipv6 BOOLEAN NOT NULL, + exiting_ipv6_relay BOOLEAN NOT NULL, + reachable_ipv6_relay BOOLEAN NOT NULL, + server_count_sum INTEGER NOT NULL, + advertised_bandwidth_bytes_sum BIGINT NOT NULL, + CONSTRAINT aggregated_ipv6_unique + UNIQUE (status_id, guard_relay, exit_relay, announced_ipv6, + exiting_ipv6_relay, reachable_ipv6_relay) +); + +-- Function to aggregate server_descriptors and status_entries rows into the +-- aggregated table and delete rows from status_entries that are then contained +-- in the aggregated table. This function is supposed to be called once after +-- inserting new rows into server_descriptors and/or status_entries. Subsequent +-- calls won't have any effect. +CREATE OR REPLACE FUNCTION aggregate_ipv6() RETURNS VOID AS $$ +INSERT INTO aggregated_ipv6 +SELECT status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, + reachable_ipv6_relay, COUNT(*) AS server_count_sum, + SUM(advertised_bandwidth_bytes) AS advertised_bandwidth_bytes +FROM status_entries +NATURAL JOIN server_descriptors +NATURAL JOIN statuses +GROUP BY status_id, guard_relay, exit_relay, announced_ipv6, exiting_ipv6_relay, + reachable_ipv6_relay +ON CONFLICT ON CONSTRAINT aggregated_ipv6_unique +DO UPDATE SET server_count_sum = aggregated_ipv6.server_count_sum + + EXCLUDED.server_count_sum, + advertised_bandwidth_bytes_sum + = aggregated_ipv6.advertised_bandwidth_bytes_sum + + EXCLUDED.advertised_bandwidth_bytes_sum; +DELETE FROM status_entries WHERE EXISTS ( + SELECT 1 FROM server_descriptors + WHERE descriptor_digest_sha1 = status_entries.descriptor_digest_sha1); +$$ LANGUAGE SQL; + +-- View on previously aggregated IPv6 server statistics in a format that is +-- compatible for writing to an output CSV file. Statuses are only included in +-- the output if they have at least 1 relay or bridge with the Running flag and +-- if at least 99.9% of referenced server descriptors are present. Dates are +-- only included in the output if at least 12 statuses are known. The last two +-- dates are excluded to avoid statistics from flapping if missing descriptors +-- are provided late. +CREATE OR REPLACE VIEW ipv6servers AS +WITH included_statuses AS ( + SELECT status_id, server, valid_after + FROM statuses NATURAL JOIN aggregated_ipv6 + GROUP BY status_id, server, valid_after + HAVING running_count > 0 + AND 1000 * SUM(server_count_sum) > 999 * running_count +), included_dates AS ( + SELECT DATE(valid_after) AS valid_after_date, server + FROM included_statuses + GROUP BY DATE(valid_after), server + HAVING COUNT(status_id) >= 12 + AND DATE(valid_after) + < (SELECT MAX(DATE(valid_after)) FROM included_statuses) - 1 +) +SELECT DATE(valid_after) AS valid_after_date, server, + CASE WHEN server = 'relay' THEN guard_relay ELSE NULL END AS guard_relay, + CASE WHEN server = 'relay' THEN exit_relay ELSE NULL END AS exit_relay, + announced_ipv6, + CASE WHEN server = 'relay' THEN exiting_ipv6_relay ELSE NULL END + AS exiting_ipv6_relay, + CASE WHEN server = 'relay' THEN reachable_ipv6_relay ELSE NULL END + AS reachable_ipv6_relay, + FLOOR(AVG(server_count_sum)) AS server_count_sum_avg, + CASE WHEN server = 'relay' THEN FLOOR(AVG(advertised_bandwidth_bytes_sum)) + ELSE NULL END AS advertised_bandwidth_bytes_sum_avg +FROM statuses NATURAL JOIN aggregated_ipv6 +WHERE status_id IN (SELECT status_id FROM included_statuses) +AND DATE(valid_after) IN ( + SELECT valid_after_date FROM included_dates WHERE server = statuses.server) +GROUP BY DATE(valid_after), server, guard_relay, exit_relay, announced_ipv6, + exiting_ipv6_relay, reachable_ipv6_relay +ORDER BY valid_after_date, server, guard_relay, exit_relay, announced_ipv6, + exiting_ipv6_relay, reachable_ipv6_relay; + diff --git a/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatusTest.java b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatusTest.java new file mode 100644 index 0000000..4b07154 --- /dev/null +++ b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedNetworkStatusTest.java @@ -0,0 +1,140 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.fail; +import static org.junit.Assert.assertNotNull; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.RelayNetworkStatusConsensus; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +public class ParsedNetworkStatusTest { + + /** Provide test data. */ + @Parameters + public static Collection<Object[]> data() { + String relayFileName = "descriptors/2017-12-04-20-00-00-consensus.part"; + String bridgeFileName = "descriptors/" + + "20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part"; + return Arrays.asList(new Object[][] { + { "Relay status without Guard or Exit flag and without IPv6 address. ", + relayFileName, true, + ZonedDateTime.parse("2017-12-04T20:00:00Z").toLocalDateTime(), 3, + "19bd830ae419b4c6ea1047370d0a5ac446f1748d", false, false, false }, + { "Relay status with Guard and Exit flag and without IPv6 address.", + relayFileName, true, + ZonedDateTime.parse("2017-12-04T20:00:00Z").toLocalDateTime(), 3, + "600a614a5ee63f8cb55aa3d4d9e9a8dd8d748d77", true, true, false }, + { "Relay status with Guard flag only and with IPv6 address.", + relayFileName, true, + ZonedDateTime.parse("2017-12-04T20:00:00Z").toLocalDateTime(), 3, + "d993e03f907f7cb302a877feb7608cbd6c4cfeb0", true, false, true }, + { "Bridge status with Running flag.", + bridgeFileName, false, + ZonedDateTime.parse("2017-12-04T19:05:07Z").toLocalDateTime(), 1, + "01b2cadfbcc0ebe50f395863665ac376d25f08ed", false, false, false }, + { "Bridge status without Running flag (skipped!).", + bridgeFileName, false, + ZonedDateTime.parse("2017-12-04T19:05:07Z").toLocalDateTime(), 1, + null, false, false, false } + }); + } + + @Parameter + public String description; + + @Parameter(1) + public String fileName; + + @Parameter(2) + public boolean isRelay; + + @Parameter(3) + public LocalDateTime timestamp; + + @Parameter(4) + public int running; + + @Parameter(5) + public String digest; + + @Parameter(6) + public boolean guard; + + @Parameter(7) + public boolean exit; + + @Parameter(8) + public boolean reachable; + + @Test + public void testParseNetworkStatus() throws Exception { + InputStream is = getClass().getClassLoader().getResourceAsStream( + this.fileName); + assertNotNull(this.description, is); + StringBuilder sb = new StringBuilder(); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line = br.readLine(); + while (null != line) { + sb.append(line).append('\n'); + line = br.readLine(); + } + for (Descriptor descriptor + : DescriptorSourceFactory.createDescriptorParser().parseDescriptors( + sb.toString().getBytes(), new File(this.fileName), this.fileName)) { + ParsedNetworkStatus parsedNetworkStatus; + if (descriptor instanceof RelayNetworkStatusConsensus) { + parsedNetworkStatus = new Parser().parseRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + parsedNetworkStatus = new Parser().parseBridgeNetworkStatus( + (BridgeNetworkStatus) descriptor); + } else { + fail(this.description); + return; + } + assertEquals(this.description, this.isRelay, parsedNetworkStatus.isRelay); + assertEquals(this.description, this.timestamp, + parsedNetworkStatus.timestamp); + assertEquals(this.description, this.running, parsedNetworkStatus.running); + if (null != this.digest) { + boolean foundEntry = false; + for (ParsedNetworkStatus.Entry parsedEntry + : parsedNetworkStatus.entries) { + if (this.digest.equals(parsedEntry.digest)) { + assertEquals(this.description, this.guard, parsedEntry.guard); + assertEquals(this.description, this.exit, parsedEntry.exit); + assertEquals(this.description, this.reachable, + parsedEntry.reachable); + foundEntry = true; + break; + } + } + if (!foundEntry) { + fail(this.description); + } + } + } + } +} + diff --git a/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptorTest.java b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptorTest.java new file mode 100644 index 0000000..5079031 --- /dev/null +++ b/modules/ipv6servers/src/test/java/org/torproject/metrics/stats/ipv6servers/ParsedServerDescriptorTest.java @@ -0,0 +1,97 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.stats.ipv6servers; + +import static junit.framework.TestCase.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.ServerDescriptor; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +public class ParsedServerDescriptorTest { + + /** Provide test data. */ + @Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + { "Relay server descriptor without or-address or ipv6-policy line.", + "descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d", + 819200, false, false }, + { "Relay server descriptor with or-address and ipv6-policy line.", + "descriptors/01003df74972ce952ebfa390f468ef63c50efa25", + 6576128, true, true }, + { "Relay server descriptor with or-address line only.", + "descriptors/018c1229d5f56eebfc1d709d4692673d098800e8", + 0, true, false }, + { "Bridge server descriptor without or-address or ipv6-policy line.", + "descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f", + 0, false, false }, + { "Bridge server descriptor with or-address line.", + "descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a", + 0, true, false }, + { "Bridge server descriptor with (ignored) ipv6-policy accept line.", + "descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb", + 0, false, false } + }); + } + + @Parameter + public String description; + + @Parameter(1) + public String fileName; + + @Parameter(2) + public int advertisedBandwidth; + + @Parameter(3) + public boolean announced; + + @Parameter(4) + public boolean exiting; + + @Test + public void testParseServerDescriptor() throws Exception { + InputStream is = getClass().getClassLoader().getResourceAsStream( + this.fileName); + assertNotNull(this.description, is); + StringBuilder sb = new StringBuilder(); + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line = br.readLine(); + while (null != line) { + sb.append(line).append('\n'); + line = br.readLine(); + } + for (Descriptor descriptor + : DescriptorSourceFactory.createDescriptorParser().parseDescriptors( + sb.toString().getBytes(), new File(this.fileName), this.fileName)) { + assertTrue(this.description, descriptor instanceof ServerDescriptor); + ParsedServerDescriptor parsedServerDescriptor + = new Parser().parseServerDescriptor((ServerDescriptor) descriptor); + assertEquals(this.description, this.advertisedBandwidth, + parsedServerDescriptor.advertisedBandwidth); + assertEquals(this.description, this.announced, + parsedServerDescriptor.announced); + assertEquals(this.description, this.exiting, + parsedServerDescriptor.exiting); + } + } +} + diff --git a/modules/ipv6servers/src/test/resources/descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f b/modules/ipv6servers/src/test/resources/descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f new file mode 100644 index 0000000..7911a41 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/000a7fe20a17bf5d9839a126b1dff43f998aac6f @@ -0,0 +1,16 @@ +@type bridge-server-descriptor 1.2 +router Unnamed 10.141.52.121 63839 0 0 +master-key-ed25519 OFHu6w2KsTtvbqDlAuA1pHOW4v9EodQI7F39qLLVyho +platform Tor 0.3.1.8 on Linux +proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +published 2017-12-01 17:03:25 +fingerprint 47EE 975E 1C8F 63A4 DD3D 8851 4ECC C4F7 65C7 C208 +uptime 1950945 +bandwidth 1073741824 1073741824 428846 +extra-info-digest EEA5CD46F1FA4A12E239CF35217AF17F4AFC7529 c4lqRmT0oC/GZz7Rq9HKZaaTMT9wotA4ZU1AYWjFjXM +hidden-service-dir +ntor-onion-key lBsrbMYEECHW0v0o/gTpNbyORbhTSz+oyz/uD/dAIic= +reject *:* +tunnelled-dir-server +router-digest-sha256 PruciDU6rrAgwoedoaGpvDTKWidF0ZNhqdcEX7KjrjI +router-digest 000A7FE20A17BF5D9839A126B1DFF43F998AAC6F diff --git a/modules/ipv6servers/src/test/resources/descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d b/modules/ipv6servers/src/test/resources/descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d new file mode 100644 index 0000000..1a82dc7 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/0018ab4f2f28af683d52f06407edbf7ce1bd3b7d @@ -0,0 +1,51 @@ +@type server-descriptor 1.0 +router NitraniumOne 89.38.149.201 5001 0 5002 +identity-ed25519 +-----BEGIN ED25519 CERT----- +AQQABmpJAfx0GsMq1jpLcBRJ3jqBqGM/lg6d8jC7Tj4lMoRI8OFvAQAgBAAz5x6X +UU1Y5htzoGc6PZsbz88xBmlqPLFMO3sOR0ZBzBqGJprWEhMYu+sY5d9y8vV5Lco+ +wxl1RkBc/X4586ab9WnV3e12WxxwFux+Ey/UtC6JFsTRTAZx7W7aKv7HQwQ= +-----END ED25519 CERT----- +master-key-ed25519 M+cel1FNWOYbc6BnOj2bG8/PMQZpajyxTDt7DkdGQcw +platform Tor 0.3.1.8 on Linux +proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +published 2017-12-01 02:23:58 +fingerprint 6F2B 25E9 2B5E 492A 5754 122A 3486 4BE4 E835 E0C4 +uptime 1159217 +bandwidth 819200 1024000 943771 +extra-info-digest B28408EE76B34E1750B77B8DFA8ACAEBB65C4828 ynrJoGDoIgxAXYlrHQ5V5qaNDbBsm22IxAnuqE4kIgk +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAMeTc60EcKNyLjUD6KTXRPpOQt89Ivz/NxPgHYW9snYtx5PLXObhgo5x +ODT2JKS+EPCZbHBnt9x1ZRIKvKIjmJbLIYL9ixk7lBoST9wa+6eVkZOBQFYTS24x +70D4CDzKDgBg6AatlljCLjcOLcRiqsL39OaITaRiQdDqIofVPHr5AgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAN+gODfQPOFT1c3iAlhqNdevKCObuBpT8mvyl6TfkeRP9Zh7Z/eTGPhH +P48i1i1dS0vHYhQfIrwvU05DsLRed50W9MUEfSuMxmyE/+Qh8rR3tmVZPlIek9Ww +wNaW3oBncnz5uodPXM9G370mKPkUYftR1H5yuBJF5HX6heoB+QbhAgMBAAE= +-----END RSA PUBLIC KEY----- +onion-key-crosscert +-----BEGIN CROSSCERT----- +hhPGeWcfIZggOAm0jwar4edfCIkLA54TbRpyJbl/VHEBTBynSKiAlBcDtFU/UrU+ +MrIh72VdisowF7LgvQUvwBlh4RkIMG/fXigdhI3YkpFqOiM2cD+lljUgIpB6sjXO +bOHzpTKPiVdPfycu/WtQdags++OVs8yWbROH77q+wzw= +-----END CROSSCERT----- +ntor-onion-key-crosscert 0 +-----BEGIN ED25519 CERT----- +AQoABmtbATPnHpdRTVjmG3OgZzo9mxvPzzEGaWo8sUw7ew5HRkHMAFb1Z9i/lRi0 +WXcoS15f7cBxZOYxwQCZBDJ8jaB0nDBbNsW90CzbwJInmNsXhy61/azD1wNLePra +f1icWuyXEgc= +-----END ED25519 CERT----- +hidden-service-dir +ntor-onion-key Ok3aAgZ7aN+r8461N3Wuc6Y9l9o3ZwKQwRP6veSybjM= +reject *:* +tunnelled-dir-server +router-sig-ed25519 JFP3nuXCByN4JSJ1MnIPbiOS0Y6Rayi1CBF+bo/+wEXPSNmPmog/XdJD9RtYEHgZJGDejJyZ8nENzLxjlcllBA +router-signature +-----BEGIN SIGNATURE----- +KF/xiANCygTl2dO4+UmVtMvOC5/G/k9XXCxpX9XVaOaqcX2XmLsKdGnj0ihWmgG+ +grQKwzQqQ9QescAI1yjUMLStyy0ta2UTDBEM8eWBylgZKCzrJDNG4zjeL/URJjpd +YrJeIPl19li82ZY+gMUs6gVFXe/XqkeTxBGqDFysz1Q= +-----END SIGNATURE----- diff --git a/modules/ipv6servers/src/test/resources/descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a b/modules/ipv6servers/src/test/resources/descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a new file mode 100644 index 0000000..0360bf8 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/0041dbf9fe846f9765882f7dc8332f94b709e35a @@ -0,0 +1,19 @@ +@type bridge-server-descriptor 1.2 +router cielarko 10.34.119.160 58783 0 0 +or-address [fd9f:2e19:3bcf::61:e1da]:58783 +master-key-ed25519 YFV9cJzNYy6HPbMjrXW1d1QKdPJyy1CeEtpjj5K7dg8 +platform Tor 0.3.2.5-alpha on Linux +proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +published 2017-12-01 06:05:21 +fingerprint 9691 9EB7 B966 9F21 855E 85E7 263D 9006 68BA 414A +uptime 66360 +bandwidth 1073741824 1073741824 272174 +extra-info-digest 31B5A95ECB1EC795C82D6D1BBE51D5B1B0186F22 8e6RyL4f/JFlvOUSCa+a0+SDPIzOEhaDHAL+0Lt1PZQ +hidden-service-dir +contact somebody +bridge-distribution-request any +ntor-onion-key ntj1wQU/1yrbiUMHDugRqzo+LmOUgGUwOTVoaYF8nWo= +reject *:* +tunnelled-dir-server +router-digest-sha256 PEuRjE6moK4BLQpxJFjNoSyjkqubGWEwg5bXiGhgoMc +router-digest 0041DBF9FE846F9765882F7DC8332F94B709E35A diff --git a/modules/ipv6servers/src/test/resources/descriptors/01003df74972ce952ebfa390f468ef63c50efa25 b/modules/ipv6servers/src/test/resources/descriptors/01003df74972ce952ebfa390f468ef63c50efa25 new file mode 100644 index 0000000..ef0f4e3 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/01003df74972ce952ebfa390f468ef63c50efa25 @@ -0,0 +1,189 @@ +@type server-descriptor 1.0 +router zwiebeltoralf2 5.9.158.75 9001 0 9030 +identity-ed25519 +-----BEGIN ED25519 CERT----- +AQQABmkLAcEyQpC+Ms4utzY0ooqt7hHwOJdsquonbf5TqLleiAzVAQAgBABhcLLB +ch5vHWVUQjbwOhxNbdCLSUK2RUYou92uX6wVPSxJy4mkJqbZntgB4+BByB7EvEc/ +ilCcOK9jLRgtGqkvmbtwXw10gOoFZi0iLBX7qZ0jpAgixzJcLlSCo6QcqQs= +-----END ED25519 CERT----- +master-key-ed25519 YXCywXIebx1lVEI28DocTW3Qi0lCtkVGKLvdrl+sFT0 +or-address [2a01:4f8:190:514a::2]:9001 +platform Tor 0.3.2.5-alpha on Linux +proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +published 2017-12-01 00:27:39 +fingerprint D11D 1187 7769 B9E6 1753 7B4B 46BF B92B 443D E33D +uptime 12050 +bandwidth 1073741824 1073741824 6576128 +extra-info-digest 6EDE204F5C0C1148DA1CEB96A5974A99AC5A6FE5 ulDel5e42xtMGHib91Q1nPeirPhdDTRwEGsA4NBBo1o +caches-extra-info +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAL3rowWZqDg5dFnJVSWfneuDtTCU7oNLEpeEN8weL+REPOvMgTApcWmJ +N84fuTmfN3QDV7gU2PZ3f5Koong5hYo/MEwbiuR3RGgsMxNnu+fYdkaoXf9aSegA +JyJMrwdIeikVjOspjfGhtmQuvX740XVb7/O98F0eYM1rpBi2EGkhAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAPEK2TJRvkY6xQspNOvbVAqRA8mS01c72lUillwLx3MDeLn0Grb+N5hZ +xHp265tChF5nefQUwxTu85SkGa9y1ALyNEm2+fJVMW99c0aiaRfOMrRlOnqwpz4y +T3OVD7H4AvAUg8hBYqsi5lpJxnXNJNCajou35RyZdA8E+gewuWFbAgMBAAE= +-----END RSA PUBLIC KEY----- +onion-key-crosscert +-----BEGIN CROSSCERT----- +NkqKSVgAnLC3Xd//qbm3o2e7kG9XrQ1DPAQIF1SORPxL2h5meRrJ5lKrjnQN2nml +PwLKX18vp4ajpMz/Q9619EMvL9WZ1ENtDbwoqUXFlSGVJ3C369LPvAgrgP0LObdn +2ovwEvXE1CMpcqfrLjba9LTaeL7DLngwqXTmFexVV3o= +-----END CROSSCERT----- +ntor-onion-key-crosscert 1 +-----BEGIN ED25519 CERT----- +AQoABmtZAWFwssFyHm8dZVRCNvA6HE1t0ItJQrZFRii73a5frBU9AFtd28+1g6/m +dzQz9tYFAvlyoEoZWlJlUnbliZT6YKLJdExdYy/Nfvzi+14lHUFELb8jEXOk1VsC +m71SoV2Vxw8= +-----END ED25519 CERT----- +family $1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA +hidden-service-dir +contact replace k with c : kontakt @ zwiebeltoralf . de +ntor-onion-key tVAtwlp9HKfF0pm7wYI9TuLc5aDczWAcHg+sVJECnGM= +reject 0.0.0.0/8:* +reject 169.254.0.0/16:* +reject 127.0.0.0/8:* +reject 192.168.0.0/16:* +reject 10.0.0.0/8:* +reject 172.16.0.0/12:* +reject 5.9.158.75:* +reject 162.218.232.15/24:* +reject 162.218.237.3/24:* +reject 195.78.229.65:* +reject 195.26.85.200/24:443 +reject 103.37.152.52:* +reject 104.111.232.180:* +reject 104.20.44.57:* +reject 104.244.42.1:* +reject 104.41.152.17:* +reject 113.107.57.43:* +reject 13.107.21.200:* +reject 149.154.175.50:* +reject 151.101.112.193:* +reject 151.101.113.140:* +reject 152.195.133.74:* +reject 165.227.251.186:* +reject 170.81.138.25:* +reject 172.217.23.130:* +reject 172.217.23.131:* +reject 172.217.23.142:* +reject 174.129.243.60:* +reject 176.34.155.20:* +reject 185.36.100.196:* +reject 195.26.85.200:* +reject 2.17.7.57:* +reject 207.182.152.130:* +reject 209.85.233.95:* +reject 216.150.210.199:* +reject 216.150.210.200:* +reject 216.38.197.179:* +reject 216.38.197.185:* +reject 217.182.138.181:* +reject 217.69.135.132:* +reject 23.35.100.252:* +reject 23.35.109.140:* +reject 31.13.64.17:* +reject 46.101.175.199:* +reject 52.222.170.230:* +reject 52.35.255.219:* +reject 64.58.116.132:* +reject 78.140.166.6:* +reject 91.149.157.121:* +reject 95.213.11.181:* +accept *:79 +accept *:110 +accept *:119 +accept *:143 +accept *:194 +accept *:220 +accept *:389 +accept *:443 +accept *:464 +accept *:465 +accept *:531 +accept *:543 +accept *:544 +accept *:554 +accept *:563 +accept *:587 +accept *:636 +accept *:706 +accept *:749 +accept *:853 +accept *:873 +accept *:902 +accept *:903 +accept *:904 +accept *:981 +accept *:989 +accept *:990 +accept *:991 +accept *:992 +accept *:993 +accept *:994 +accept *:995 +accept *:1194 +accept *:1220 +accept *:1293 +accept *:1533 +accept *:1677 +accept *:1723 +accept *:1755 +accept *:1863 +accept *:1883 +accept *:2095 +accept *:2096 +accept *:2102 +accept *:2103 +accept *:2104 +accept *:3128 +accept *:3690 +accept *:4321 +accept *:4643 +accept *:5050 +accept *:5190 +accept *:5222 +accept *:5269 +accept *:5280 +accept *:5900 +accept *:6660 +accept *:6661 +accept *:6662 +accept *:6663 +accept *:6664 +accept *:6665 +accept *:6666 +accept *:6667 +accept *:6668 +accept *:6669 +accept *:6679 +accept *:6697 +accept *:7777 +accept *:8008 +accept *:8074 +accept *:8082 +accept *:8087 +accept *:8232 +accept *:8233 +accept *:8332 +accept *:8333 +accept *:8883 +accept *:9418 +accept *:11371 +accept *:19294 +accept *:50002 +accept *:64738 +reject *:* +ipv6-policy accept 79,110,119,143,194,220,389,443,464-465,531,543-544,554,563,587,636,706,749,853,873,902-904,981,989-995,1194,1220,1293,1533,1677,1723,1755,1863,1883,2095-2096,2102-2104,3128,3690,4321,4643,5050,5190,5222,5269,5280,5900,6660-6669,6679,6697,7777,8008,8074,8082,8087,8232-8233,8332-8333,8883,9418,11371,19294,50002,64738 +tunnelled-dir-server +router-sig-ed25519 jg4DdPa8gG5JwnzBtM/YWUALcb/AoGodWkeroChEmC17vdlVFv7UhanWvCFQo4gAFzvB4meRp0F1fcz19Qu3Cw +router-signature +-----BEGIN SIGNATURE----- +sMlEGWDRZJkfB8W+3rS9dUTvXk8UEmlvTT0HIopl/YlvZ/bSEe25VIuMCzODgrrK +rHBsP6zw2IICVmBnW/TyqOOqwphIPA9RR8gs0BsE1cKh/g6SXsSfpGIR9cGde6q4 ++u8IDSEp1uUpDzEgJt/3Enkw3RDYWl5eODeuE9xMKvo= +-----END SIGNATURE----- diff --git a/modules/ipv6servers/src/test/resources/descriptors/018c1229d5f56eebfc1d709d4692673d098800e8 b/modules/ipv6servers/src/test/resources/descriptors/018c1229d5f56eebfc1d709d4692673d098800e8 new file mode 100644 index 0000000..0c60c54 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/018c1229d5f56eebfc1d709d4692673d098800e8 @@ -0,0 +1,54 @@ +@type server-descriptor 1.0 +router HCTOR003 192.30.34.248 9001 0 0 +identity-ed25519 +-----BEGIN ED25519 CERT----- +AQQABmkCAfbbsyUlm3tvNXIKsdl7FWjCBS27EqU24hlQc4V76pKFAQAgBAC8UP2n +QP9JHK+trzTtoDAsENdJt5yvp/nIyhQY/TcsY7WQ4w/yJlTJeC5ysSaEmbsgyBiI +YWEeG3o20mXfIzoX9idsUk1MBDb2+eCpc9JWTkk0zuo80xLyjP7uGLBqKAU= +-----END ED25519 CERT----- +master-key-ed25519 vFD9p0D/SRyvra807aAwLBDXSbecr6f5yMoUGP03LGM +or-address [2604:180::40f1:f45c]:9001 +platform Tor 0.2.9.8 on Linux +proto Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2 +published 2017-12-01 01:55:47 +fingerprint CF00 0B63 C68D 4FAA 3067 7342 267B D75F 1699 A3C7 +uptime 2264147 +bandwidth 1073741824 1073741824 0 +extra-info-digest A03FF6707C62350ED5AD650010D70DE324FB6A5F xzHCpCEQM4cp9mrDmnkxT7ws1kdv0jZt2HtdyWGRtXw +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAOiURbkUxwrtqfb+Ykp1rSodLlJHPzGy3+BX5Myve4tqZg2LvynN4o4y +OPEGNUs3mdHBYZmXpt5ZgGQd3uNb21dvVLX9RrW9uslVmEhJbfNbPxjs1gdhnJya +64Bcju57PRCr97IG+7mBtyd+6iwmVy3JRKDhXE84raAhAvt4JW2bAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAPPPj2Rt36C1tiSnZtpKKgxwBxgw9QwwVdtKESDVXXnEwamwQYkSwcPB +xG0RmXRVoXvKva7+e3N4DkeE+DVrZ4GwkKRKbRhtfmp+TH4+TskI+7WAKuKJdCLP +fD5P+9RUF7OcShVYzjzuXnha6nNQbHAny8wL2EJxEcc1Yq0H1smfAgMBAAE= +-----END RSA PUBLIC KEY----- +onion-key-crosscert +-----BEGIN CROSSCERT----- +0UiDaVEDGQx71DnKbFc6HxYsVfitgaPJT+0bjBcqnOh8v+jfTUua4npDpi9RkL0W +5xy0jN1N8j0JTrXjDDPQTF2akgu+iWCoWom4jzmy3HXj+a4jyPaWKVI4Qhz+znYW +Ts8joaiNImQ94+59DVvxbT2aqHv6OqH/H0Aa93Q1bhQ= +-----END CROSSCERT----- +ntor-onion-key-crosscert 1 +-----BEGIN ED25519 CERT----- +AQoABmliAbxQ/adA/0kcr62vNO2gMCwQ10m3nK+n+cjKFBj9NyxjAB8b32quZYIR +D5R7Mw6t4Jvu22eBQtbfXbeUneZ9gPsgnw7EEr6dzwbhMY9t4PRBInsRiU2o4C0c +E/cOR9EKVgE= +-----END ED25519 CERT----- +family $CF000B63C68D4FAA30677342267BD75F1699A3C7 $CF879F5ACD419C915170BB1978CFFA1DF6E7ACFF +hibernating 1 +hidden-service-dir +contact PGP 0x9958256C Daniel Hagan daniel.hagan@hagan-consulting.com +ntor-onion-key VKuzSIox0yRwcJWUmHrXIq8H7NiyGIBDmmRZA1v4ghk= +reject *:* +router-sig-ed25519 lS8zFzXia00h9YnFucF/zAOI/SsCNQwaKy/psMQheKR1nJ+SLSwHTmBtTDb7xWxyKQuikrML/obZ1CfPIkleAw +router-signature +-----BEGIN SIGNATURE----- +yUcGXmxpsHsiD9O1AHElOaUBpCuhcfSC6GrmYcKhfCDpFyH1vOZMrdUE9doQ91By +uc3zNwRiaivmlFjOj2jh0PYwO8UIMQUAbQThas1yVZx1Jv/qYc9yTE1W4pPnLdxj +sHN+fVzYKbGrYo9kUrAuU9GnfLIQv5/9pUd8Nq7+/a4= +-----END SIGNATURE----- diff --git a/modules/ipv6servers/src/test/resources/descriptors/2017-12-04-20-00-00-consensus.part b/modules/ipv6servers/src/test/resources/descriptors/2017-12-04-20-00-00-consensus.part new file mode 100644 index 0000000..73e7a98 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/2017-12-04-20-00-00-consensus.part @@ -0,0 +1,149 @@ +@type network-status-consensus-3 1.0 +network-status-version 3 +vote-status consensus +consensus-method 26 +valid-after 2017-12-04 20:00:00 +fresh-until 2017-12-04 21:00:00 +valid-until 2017-12-04 23:00:00 +voting-delay 300 300 +client-versions 0.2.5.14,0.2.5.15,0.2.5.16,0.2.8.14,0.2.8.15,0.2.8.16,0.2.8.17,0.2.9.11,0.2.9.12,0.2.9.13,0.2.9.14,0.3.0.9,0.3.0.10,0.3.0.11,0.3.0.12,0.3.0.13,0.3.1.5-alpha,0.3.1.6-rc,0.3.1.7,0.3.1.8,0.3.1.9,0.3.2.1-alpha,0.3.2.2-alpha,0.3.2.3-alpha,0.3.2.4-alpha,0.3.2.5-alpha,0.3.2.6-alpha +server-versions 0.2.5.14,0.2.5.15,0.2.5.16,0.2.8.14,0.2.8.15,0.2.8.16,0.2.8.17,0.2.9.11,0.2.9.12,0.2.9.13,0.2.9.14,0.3.0.9,0.3.0.10,0.3.0.11,0.3.0.12,0.3.0.13,0.3.1.5-alpha,0.3.1.6-rc,0.3.1.7,0.3.1.8,0.3.1.9,0.3.2.1-alpha,0.3.2.2-alpha,0.3.2.3-alpha,0.3.2.4-alpha,0.3.2.5-alpha,0.3.2.6-alpha +known-flags Authority BadExit Exit Fast Guard HSDir NoEdConsensus Running Stable V2Dir Valid +recommended-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 +recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 +required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 LinkAuth=1 Microdesc=1-2 Relay=2 +required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2 +params CircuitPriorityHalflifeMsec=30000 NumDirectoryGuards=3 NumEntryGuards=1 NumNTorsPerTAP=100 Support022HiddenServices=0 UseNTorHandshake=1 UseOptimisticData=1 bwauthpid=1 cbttestfreq=10 pb_disablepct=0 usecreatefast=0 +shared-rand-previous-value 9 koy+780Z3gcdh2ZavmUHWEwpS4oRouJp+Lr8Kc1HPDY= +shared-rand-current-value 9 YkqgxViKZUGDyELmvIsVxFfyJSCAwYIznbLRqMwSSI8= +dir-source dannenberg 0232AF901C31A04EE9848595AF9BB7620D4C5B2E dannenberg.torauth.de 193.23.244.244 80 443 +contact Andreas Lehner +vote-digest ED565598CA7BD1225DBF9196DDE0C7FED4CD8F17 +dir-source tor26 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 86.59.21.38 86.59.21.38 80 443 +contact Peter Palfrader +vote-digest 49482F6EB80BF1BD7D95FD2783339FD407754615 +dir-source longclaw 23D15D965BC35114467363C165C4F724B64B4F66 199.58.81.140 199.58.81.140 80 443 +contact Riseup Networks <collective at riseup dot net> - 1nNzekuHGGzBYRzyjfjFEfeisNvxkn4RT +vote-digest 234A880CAF86B839E92EBABC837D3F1D4F729CA0 +dir-source bastet 27102BC123E7AF1D4741AE047E160C91ADC76B21 204.13.164.118 204.13.164.118 80 443 +contact stefani <nocat at readthefinemanual dot net> +vote-digest F1C150923BA43C3833241B53657FC5E6D1DEBF44 +dir-source maatuska 49015F787433103580E3B66A1707A00E60F2D15B 171.25.193.9 171.25.193.9 443 80 +contact 4096R/1E8BF34923291265 Linus Nordberg linus@nordberg.se +vote-digest E32521A257B2EED001870E2313322FC5208EB235 +dir-source moria1 D586D18309DED4CD6D57C18FDB97EFA96D330566 128.31.0.34 128.31.0.34 9131 9101 +contact 1024D/28988BF5 arma mit edu +vote-digest 23A40AB1C8E29ECB10B9CC92B09F7174E4BC9F9E +dir-source dizum E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212 194.109.206.212 80 443 +contact 1024R/8D56913D Alex de Joode adejoode@sabotage.org +vote-digest 9EF2A2B9D3A62100611C73D146C1CAE90D555B91 +dir-source gabelmoo ED03BB616EB2F60BEC80151114BB25CEF515B226 131.188.40.189 131.188.40.189 80 443 +contact 4096R/261C5FBE77285F88FB0C343266C8C2D7C5AA446D Sebastian Hahn tor@sebastianhahn.net - 12NbRAjAG5U3LLWETSF7fSTcdaz32Mu5CN +vote-digest 543D5B1CC570D7A955471365C12CB937F533708E +dir-source Faravahar EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 154.35.175.225 154.35.175.225 80 443 +contact 0x0B47D56D Sina Rabbani (inf0) <sina redteam net> +vote-digest 6A27A6FC019720E95E933F62A3ACAF0D9BB190C2 +r seele AAoQ1DAR6kkoo19hBAX5K0QztNw Gb2DCuQZtMbqEEc3DQpaxEbxdI0 2017-12-04 04:42:01 67.161.31.147 9001 0 +s Running Stable V2Dir Valid +v Tor 0.3.0.10 +pr Cons=1-2 Desc=1-2 DirCache=1 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +w Bandwidth=28 +p reject 1-65535 +r CalyxInstitute14 ABG9JIWtRdmE7EFZyI/AZuXjMA4 YAphSl7mP4y1WqPU2emo3Y10jXc 2017-12-04 02:14:54 162.247.72.201 443 80 +s Exit Fast Guard HSDir Running Stable V2Dir Valid +v Tor 0.3.1.5-alpha +pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +w Bandwidth=6240 +p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544,554,563,636,706,749,873,902-904,981,989-995,1194,1220,1293,1500,1533,1677,1723,1755,1863,2082-2083,2086-2087,2095-2096,2102-2104,3128,3389,3690,4321,4643,5050,5190,5222-5223,5228,5900,6660-6669,6679,6697,8000,8008,8074,8080,8087-8088,8332-8333,8443,8888,9418,9999-10000,11371,12350,19294,19638,23456,33033,64738 +r havingtrouble AEJpGAnlKLJYyf4BKhxGhZFj8UI 2ZPgP5B/fLMCqHf+t2CMvWxM/rA 2017-12-04 04:20:22 159.203.42.254 9001 443 +a [2604:a880:cad:d0::862:4001]:9050 +s Fast Guard HSDir Running Stable V2Dir Valid +v Tor 0.3.1.8 +pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-4 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +w Bandwidth=10400 +p reject 1-65535 +directory-footer +bandwidth-weights Wbd=0 Wbe=0 Wbg=3488 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 Wem=10000 Wgb=10000 Wgd=0 Wgg=6512 Wgm=6512 Wmb=10000 Wmd=0 Wme=0 Wmg=3488 Wmm=10000 +directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E 4F82D69064739702C50C69436997FCB9C29944ED +-----BEGIN SIGNATURE----- +SyifLslDf0C7oOzZPTG7UM1P5sxUxVsnKAd3XsRg1UNhHWX+32tZJNlUjtauFlaj +aQ/AH7cBu49tvomy04MKC+820KQ9wodSFKlb4N6NFwYLgIexy5PNtAtVhAtQszrO +zym9drmjG39h6rU3SM7GoA7M38K5bv7WauhlV5L3aQClmSOxRY9KxJ+qyid7jfGj +zeLhxAM1rGrLLoo6nsyifEfD3u1QLNQC1BVLAOZ4wuT2mCSCCSmHN6L//9kkcqh1 +yrPvuJsr/9aP6ibWMav2pTSYXB8Hblps3Q8yBo20O8b8p5cMzDeBJPvu4Vi/ccQ1 +rZVZAM1ercsa1K7c3ajwGQ== +-----END SIGNATURE----- +directory-signature 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 6AEC757516B142855BCE29708EBDA965AC3DD06A +-----BEGIN SIGNATURE----- +D9hlAHJHDwOs+KesGHmLyKvifVer+LnLjLTNfEGjQ8ox5/w/dXrQ9GWeeA3AndbQ +DSB2YXrE6TflhIu59RxlPQlR+NAoZ79XdkM6gGzWRBM3lAVcRKZu9kmGVhLqckDj +OFeMKzrE1PAzDpJWQ7S2gaQHPg2SzCNOln+zHFdFnsShdqo1G7JFJhnew0qexy/g +WMwYg20ToGRb1kJvwRdQyU+cOgNgUDb83M07W5SpuUZPJdeKPknFR5e4p/YykeOe +ZrdYFInd+pwBkvTGD7iA/tezEkG6fIp+UFoN+g5R8n4D07k71JRYyyNW+1dLzpGw +Kdbh62lp/wGN98PotBvG3ycBx31CZTNAQRhPYAMBguWCiLN4mTyzKeF2VwscJUe8 +r3pHkUDaSVWXATndURSmhCdF1z+F8PwSsLXw3B7Hgag+DKnVUOZZFdjnR+p/8RwQ +/QQdmHHHE1TDao4b1nA6KmSjVjemNDPN0fKtFcDCzjvoosUF2QTLKYh/AncHH8f+ +-----END SIGNATURE----- +directory-signature 23D15D965BC35114467363C165C4F724B64B4F66 D632ADF38603F34EAFCB461C4DA5F67EF62BE7EC +-----BEGIN SIGNATURE----- +nLvLwOuLcSlpkDXyIP22stte72uA/AKnG5wt/1X2Iy6E5kwNCoSCf6Zmr7Xuh6Oq +4t4KGfkrBW8rrp02ap/nACDB8IP+3xf75AcdrEReVDCXBT7jzUY1KHBPCB3pxaFp +48jy63r5VgRdqGZXuctZ1DCzF/AfIcWkw5eeXrKtkWH8RlKXgVKm5LOXzvCoscUe +5eRBzmC1GCPB7pPbQZVWvfvZTB1AulHWcpri8AIBrmalGtX5VTpEfCFXsxnL0Od1 +ibW5JfUXse9Vl5tS/iBnFXfNmNNT46/EcgniB9RkspMmF2fopEaXdutF29uN6ERu +IBPpR3n9MVTEAnN5uhOFLw== +-----END SIGNATURE----- +directory-signature 27102BC123E7AF1D4741AE047E160C91ADC76B21 88E1BDDC8E6363355910D086B37B562F81674DC1 +-----BEGIN SIGNATURE----- +Op1HXNiCoZaYwexT9Qt2lYa8/DzFZ5twMlku+KpbNJYQnpRsshFr/BUuWkXDGjuX +ieKs5UkIpJXopyXKR4uZ5htHJAT8ukRjrfNS+43jsbGtoPoD/plyeZsBjyXUk8WY +CAHTkt4n3t8xnxrTnIWL+Uv0qr39H7WOGXdbkqamEOSUFoGaVp66uNTYGJGssRRj +G94MdyBGP1hDZTRO3Bf9ozH+Eg+Xl1JAkST2DP4kJXIpCcjRqBeFSMly7SQSkUkv +oIvLWgfHpnUxqZmE2/T/rP6Sr85CaYfZI73CEQXiSxZbMf6HV0QjlKfrlwWt7KoL +TPOzFsdHCoGQ/9mzk3272w== +-----END SIGNATURE----- +directory-signature 49015F787433103580E3B66A1707A00E60F2D15B C9C96501FD35732C45C424B74CC32739991B1F51 +-----BEGIN SIGNATURE----- +vFYSlFkWhCcRfYmWAXUZKVTuBWP7vEui1abX6uDeCJticVnfvTdLNHM6qmFW2XcO +4G6BARj7ukPW/rXjF6+SJ3JWSf0I7u7KLfozo4G1yiEknuIh1egOCwp2SRKgc6ZC +kfhGM/0N3fXAg7JSmOcM59HMQyk7ky3LbPAOaUDhnSkByo5QDh4b5eSxxAJ87Hif +4emyDbc49mqTF3TP+sB9Mqtp8zltNPMuhix2CqT4dSTyu/j3YdRVQuKHUu7fWBPn +NdnLLW0HI/rMjKQ7nCKx6VOEHu9HeYkCxQiMdfiO9LmlXGBeNcNKWt7htS9FAAiW +1KcGg5KAVeRAIcWCK14xFg== +-----END SIGNATURE----- +directory-signature D586D18309DED4CD6D57C18FDB97EFA96D330566 6E44451E3F1CEB435E4D95C1F8B12AA022BB34CF +-----BEGIN SIGNATURE----- +W8pGpRMHo03xiNHJ6i6ubF8VvPH39sFlZ7N6DwbZWHCn8zU13tdgpHaIV+REGjN3 +2/cd0s3S7sABhuaMyiGVUVY8jrEM4zm0x8ORfAS+TKDQ6FREJ9aGCP5VRMLKnOU5 +zEC8qHzjAMLlpaHAOlvjv+b2DjYwQn5L9dQZaGBn0sa4CrOn/Bvd/yZE7/XYpUfn +b8il6DcmpSMrnBvYwQoCSnfVogyP8lh2vJGbS9HnWjta9utyRVPD3c6j50k2k6Kw +8PvGRYR4GfyMU7TREU8d3YObtZ28cwDfiZBBPk2fZvYeaJIIjvZhuKfQPZI5VUJX +rzILtPiBqDwogbMOar2HFQ== +-----END SIGNATURE----- +directory-signature E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 109A865D7DBE58367C120353CBE9947EE263695A +-----BEGIN SIGNATURE----- +gw3VRZAppEw3+USdqj0SHS3FXtZjkWXAv2ym9HwEIEn8gEfTzoFl1TkGTKCkfrt4 +sjJ/x5jOu9+I8feH9hGY9JYcnEFQM9kALjdBUffw4hvAwZ3SfszkwUbwAQZsnQOE +bFhMu+flpq6XgdqTGD9aalJaW2VrKo+321LHcucyw7Tl/Rf3Q8YGTF7YEIO1vYT7 +/AS1t7n7f58/3Gk5cLhLgnGbK+PaAKq0rdc8RAlCv5Wdhg5pY7W8ipKQWqM9gFEG +5e0GbVq8UygsljfvH1ABosAY/Y4I5RCH5gLsEzDyM4K4GIkVduUHlywRVGI47ZWC +thxn3L/3WPGAjaT6RGBcHg== +-----END SIGNATURE----- +directory-signature ED03BB616EB2F60BEC80151114BB25CEF515B226 28EB6C6635F1DB90BB60AF8B43F1676E892BB30B +-----BEGIN SIGNATURE----- +Mld9hxzoHSIa42Sct3FgcQpFV0wbzE3HkUYEfIsJqA1TwD7PRo9QaCAd85lltnnN +SNrNd1YqqX15fsFhB16ZeR3D2LH6xOQx2FyyChqXAzVPJXrPlUFR5BgQ2RPjxd1N +H8nXgsIbBoZLoMLV7MRQAmY19a4H2X2iiUqRoo2gdH6glqhCFKVduCy0/0KZ1/Pu +AhIVL/P3dVwtccv44/rA4dKKjFi1CaUZhO6bbmxQjhYkuxquz6PkBlLpKUW2cV/K +e75Qy+dumbnPfVR4N8QdAebXo8g3Mf9MyWJ+lbLdBKFMuUUi9QexJUOVtOjWVzxI +dOqjEkQbb5QNk4acB1u9sA== +-----END SIGNATURE----- +directory-signature EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 E861D5367EE5A469892D3FE6B2A25218FBA133FC +-----BEGIN SIGNATURE----- +DoQywyhompXommfwML3lgHbt0r4G4ouVpqbyfmT+HUWnG0fXdGRvlrZfCcocyzNK +q3c9rvVw/x4FDkRVgc/gU3HYKFRHr2q4CD6lMUwny1WWhibKiKU5HfvfbZ+84gGj +T3m9WBd1v9TQFkYq0R1dOY+6t18FgFQesLB6YVw9H+yYxbNlUGn+sECmkorzo6XJ +sv4QxrhEATzMmv1SFA4pfnppq4al2wCW3u1myR1Ufv8wsaXts/z4AECzfOLDP6Z9 +/DMC8akN+XeIlu5ghrM2IyBXXT+yIEM85VYO8Y7Es5O2/Vhv2QbbcnbUYuScV1ap +soP2uHArMaoA+qk4lVkw0A== +-----END SIGNATURE----- diff --git a/modules/ipv6servers/src/test/resources/descriptors/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part b/modules/ipv6servers/src/test/resources/descriptors/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part new file mode 100644 index 0000000..659548e --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/20171204-190507-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1.part @@ -0,0 +1,12 @@ +@type bridge-network-status 1.2 +published 2017-12-04 19:05:07 +flag-thresholds stable-uptime=1105427 stable-mtbf=4042454 fast-speed=53000 guard-wfu=98.000% guard-tk=691200 guard-bw-inc-exits=512000 guard-bw-exc-exits=512000 enough-mtbf=1 ignoring-advertised-bws=0 +fingerprint 1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1 +r Unnamed ACzV/+Qobny0/Tk7Jodk8cP+ME4 AbLK37zA6+UPOVhjZlrDdtJfCO0 2017-12-04 16:28:59 10.239.66.98 56676 0 +s Fast HSDir Running Stable V2Dir Valid +w Bandwidth=63 +p reject 1-65535 +r Unnamed ADXqKmHijTlfCArKIkRTlJDnCVA tSDkh5Xc+Mh6tTPCYO6fivU/Apk 2017-12-04 18:01:53 10.237.141.119 51916 0 +s V2Dir Valid +w Bandwidth=18 +p reject 1-65535 diff --git a/modules/ipv6servers/src/test/resources/descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb b/modules/ipv6servers/src/test/resources/descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb new file mode 100644 index 0000000..2952ee9 --- /dev/null +++ b/modules/ipv6servers/src/test/resources/descriptors/64dd486d89af14027c9a7b4347a94b74dddb5cdb @@ -0,0 +1,18 @@ +@type bridge-server-descriptor 1.2 +router EnigmaDMZ 10.111.225.186 52121 0 0 +master-key-ed25519 IpgU7WgO6uxWT8BtEuNhKtH+S+aOOyttZa5kWqVSya8 +platform Tor 0.2.9.12 on Linux +proto Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2 +published 2017-12-01 11:16:37 +fingerprint 835C E613 1471 0B08 B680 970C AABC 864A 0C00 04B4 +uptime 10918 +bandwidth 10240000 12288000 58783 +extra-info-digest 27FBC6A032A4BF21EF96B8753C67DC8E4B7E456B eshsUBj7pvOpQlYUQgjQt5emrtnqtJUZi8E6JZ4gsvg +hidden-service-dir +contact somebody +ntor-onion-key EOsXa08KZ2cpg8LzsBw531ymC0ixg6pjuwaGLYV6WgI= +reject *:* +ipv6-policy accept 1-65535 +tunnelled-dir-server +router-digest-sha256 /P9VVoa64l5C0G5o0dYbTa975H1Uk9xizCdUgXTVk5A +router-digest 64DD486D89AF14027C9A7B4347A94B74DDDB5CDB diff --git a/modules/ipv6servers/src/test/sql/test-ipv6servers.sql b/modules/ipv6servers/src/test/sql/test-ipv6servers.sql new file mode 100644 index 0000000..7e5ca2e --- /dev/null +++ b/modules/ipv6servers/src/test/sql/test-ipv6servers.sql @@ -0,0 +1,196 @@ +-- Copyright 2017 The Tor Project +-- See LICENSE for licensing information + +-- Hint: You'll need pgTAP in order to run these tests! + +CREATE EXTENSION IF NOT EXISTS pgtap; + +BEGIN; + +SELECT plan(89); + +-- Make sure that the server_descriptors table is as expected. +SELECT has_table('server_descriptors'); +SELECT has_column('server_descriptors', 'descriptor_digest_sha1'); +SELECT col_type_is('server_descriptors', 'descriptor_digest_sha1', 'bytea'); +SELECT col_is_pk('server_descriptors', 'descriptor_digest_sha1'); +SELECT has_column('server_descriptors', 'advertised_bandwidth_bytes'); +SELECT col_type_is('server_descriptors', 'advertised_bandwidth_bytes', 'integer'); +SELECT col_not_null('server_descriptors', 'advertised_bandwidth_bytes'); +SELECT has_column('server_descriptors', 'announced_ipv6'); +SELECT col_type_is('server_descriptors', 'announced_ipv6', 'boolean'); +SELECT col_not_null('server_descriptors', 'announced_ipv6'); +SELECT has_column('server_descriptors', 'exiting_ipv6_relay'); +SELECT col_type_is('server_descriptors', 'exiting_ipv6_relay', 'boolean'); +SELECT col_not_null('server_descriptors', 'exiting_ipv6_relay'); + +-- Make sure that the server enum is as expected. +SELECT has_enum('server_enum'); +SELECT enum_has_labels('server_enum', ARRAY['relay', 'bridge']); + +-- Make sure that the statuses table is as expected. +SELECT has_table('statuses'); +SELECT has_column('statuses', 'status_id'); +SELECT col_type_is('statuses', 'status_id', 'integer'); +SELECT col_is_pk('statuses', 'status_id'); +SELECT has_column('statuses', 'server'); +SELECT col_type_is('statuses', 'server', 'server_enum'); +SELECT col_not_null('statuses', 'server'); +SELECT has_column('statuses', 'valid_after'); +SELECT col_type_is('statuses', 'valid_after', 'timestamp without time zone'); +SELECT col_not_null('statuses', 'valid_after'); +SELECT has_column('statuses', 'running_count'); +SELECT col_type_is('statuses', 'running_count', 'integer'); +SELECT col_not_null('statuses', 'running_count'); +SELECT col_is_unique('statuses', ARRAY['server', 'valid_after']); + +-- Make sure that the status_entries table is as expected. +SELECT has_table('status_entries'); +SELECT has_column('status_entries', 'status_id'); +SELECT col_type_is('status_entries', 'status_id', 'integer'); +SELECT fk_ok('status_entries', 'status_id', 'statuses', 'status_id'); +SELECT col_not_null('status_entries', 'status_id'); +SELECT has_column('status_entries', 'descriptor_digest_sha1'); +SELECT col_type_is('status_entries', 'descriptor_digest_sha1', 'bytea'); +SELECT col_not_null('status_entries', 'descriptor_digest_sha1'); +SELECT has_column('status_entries', 'guard_relay'); +SELECT col_type_is('status_entries', 'guard_relay', 'boolean'); +SELECT col_not_null('status_entries', 'guard_relay'); +SELECT has_column('status_entries', 'exit_relay'); +SELECT col_type_is('status_entries', 'exit_relay', 'boolean'); +SELECT col_not_null('status_entries', 'exit_relay'); +SELECT has_column('status_entries', 'reachable_ipv6_relay'); +SELECT col_type_is('status_entries', 'reachable_ipv6_relay', 'boolean'); +SELECT col_not_null('status_entries', 'reachable_ipv6_relay'); +SELECT col_is_unique('status_entries', ARRAY['status_id', 'descriptor_digest_sha1']); +SELECT hasnt_pk('status_entries'); + +-- Make sure that the aggregated_ipv6 table is as expected. +SELECT has_table('aggregated_ipv6'); +SELECT has_column('aggregated_ipv6', 'status_id'); +SELECT col_type_is('aggregated_ipv6', 'status_id', 'integer'); +SELECT fk_ok('aggregated_ipv6', 'status_id', 'statuses', 'status_id'); +SELECT col_not_null('aggregated_ipv6', 'status_id'); +SELECT has_column('aggregated_ipv6', 'guard_relay'); +SELECT col_type_is('aggregated_ipv6', 'guard_relay', 'boolean'); +SELECT col_not_null('aggregated_ipv6', 'guard_relay'); +SELECT has_column('aggregated_ipv6', 'exit_relay'); +SELECT col_type_is('aggregated_ipv6', 'exit_relay', 'boolean'); +SELECT col_not_null('aggregated_ipv6', 'exit_relay'); +SELECT has_column('aggregated_ipv6', 'reachable_ipv6_relay'); +SELECT col_type_is('aggregated_ipv6', 'reachable_ipv6_relay', 'boolean'); +SELECT col_not_null('aggregated_ipv6', 'reachable_ipv6_relay'); +SELECT has_column('aggregated_ipv6', 'announced_ipv6'); +SELECT col_type_is('aggregated_ipv6', 'announced_ipv6', 'boolean'); +SELECT col_not_null('aggregated_ipv6', 'announced_ipv6'); +SELECT has_column('aggregated_ipv6', 'exiting_ipv6_relay'); +SELECT col_type_is('aggregated_ipv6', 'exiting_ipv6_relay', 'boolean'); +SELECT col_not_null('aggregated_ipv6', 'exiting_ipv6_relay'); +SELECT has_column('aggregated_ipv6', 'server_count_sum'); +SELECT col_type_is('aggregated_ipv6', 'server_count_sum', 'integer'); +SELECT col_not_null('aggregated_ipv6', 'server_count_sum'); +SELECT has_column('aggregated_ipv6', 'advertised_bandwidth_bytes_sum'); +SELECT col_type_is('aggregated_ipv6', 'advertised_bandwidth_bytes_sum', 'bigint'); +SELECT col_not_null('aggregated_ipv6', 'advertised_bandwidth_bytes_sum'); +SELECT col_is_unique('aggregated_ipv6', + ARRAY['status_id', 'guard_relay', 'exit_relay', 'announced_ipv6', + 'exiting_ipv6_relay', 'reachable_ipv6_relay']); + +-- Truncate all tables for subsequent tests. This happens inside a transaction, +-- so we're not actually truncating anything. +TRUNCATE server_descriptors, statuses, status_entries, aggregated_ipv6; + +-- Make sure that the aggregated_ipv6 table is empty. +SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 0;', + 'At the beginning, the aggregated_ipv6 table should be empty.'); + +-- And make sure that running the aggregate_ipv6() function does not change that. +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 0;', + 'Even after aggregating, the aggregated_ipv6 table should be empty.'); + +-- Insert a server descriptor, then try again. +INSERT INTO server_descriptors (descriptor_digest_sha1, advertised_bandwidth_bytes, announced_ipv6, + exiting_ipv6_relay) VALUES ('\x00', 100, FALSE, TRUE); + +-- Try to aggregate, though there's not much to aggregate without corresponding +-- entry in status_entries. +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 0;', + 'At the beginning, the aggregated_ipv6 table should be empty.'); + +-- Attempt to add an entry to status_entries, but without having inserted an +-- entry into statuses first. +SELECT throws_ok('INSERT INTO status_entries (status_id, descriptor_digest_sha1) ' + || 'VALUES (1, ''\x00'');'); + +-- Try again in the correct order. +INSERT INTO statuses (server, valid_after, running_count) + VALUES ('relay'::server_enum, '2017-12-04 00:00:00'::TIMESTAMP, 1); +INSERT INTO status_entries + SELECT status_id, '\x00', TRUE, FALSE, FALSE FROM statuses; + +-- Now aggregate and see how the status_entries entry gets moved over to the +-- aggregated_ipv6 table. However, it's just one status, so it doesn't show in the +-- output view yet. +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM status_entries;', 'SELECT 0;', + 'status_entries should not contain aggregated row anymore.'); +SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 1;', + 'aggregated_ipv6 table should contain exactly one row now.'); +SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 0;', + 'ipv6servers should not contain any results yet.'); + +-- Try to aggregate once more, but that shouldn't change anything. +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM status_entries;', 'SELECT 0;', + 'status_entries should still be empty.'); +SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 1;', + 'aggregated_ipv6 table should still contain exactly one row.'); + +-- Insert statuses for 3 days, of which the last 2 will be cut off in the +-- output. +INSERT INTO statuses (server, valid_after, running_count) + SELECT 'relay'::server_enum, GENERATE_SERIES('2017-12-04 01:00:00'::TIMESTAMP, + '2017-12-06 23:00:00', '1 hour'), 1; + +-- Insert the same relay as entries for all statuses except the one that we +-- added earlier and that is already contained in the aggregated_ipv6 table. (In the +-- actual import code we'd first check that we already inserted the status and +-- then not import any entries from it.) +INSERT INTO status_entries + SELECT status_id, '\x00', TRUE, FALSE, FALSE FROM statuses + WHERE valid_after > '2017-12-04 00:00:00'::TIMESTAMP; + +-- Aggregate, then look at the output. +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM status_entries;', 'SELECT 0;', + 'status_entries should not contain anything anymore.'); +SELECT set_eq('SELECT COUNT(*) FROM aggregated_ipv6;', 'SELECT 72;', + 'aggregated_ipv6 table should contain one row per status.'); +SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 1;', + 'ipv6servers should now contain a results line.'); + +-- Insert another status entry for which there is no corresponding server +-- descriptor to observe how the results line disappears again (because we +-- require 99.9% of server descriptors to be present). This is just a test case +-- that would not occur in practice, because we wouLdn't retroactively add new +-- status entries. It's just server descriptors that we might add later. +INSERT INTO status_entries + SELECT status_id, '\x01', FALSE, FALSE, FALSE FROM statuses; +UPDATE statuses SET running_count = 2; +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 0;', + 'ipv6servers should be empty, because of missing server descriptors.'); + +-- Okay, okay, provide the missing server descriptor. +INSERT INTO server_descriptors (descriptor_digest_sha1, advertised_bandwidth_bytes, announced_ipv6, + exiting_ipv6_relay) VALUES ('\x01', 100, TRUE, TRUE); +SELECT aggregate_ipv6(); +SELECT set_eq('SELECT COUNT(*) FROM ipv6servers;', 'SELECT 2;', + 'ipv6servers should be non-empty again.'); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/shared/bin/20-run-ipv6servers-stats.sh b/shared/bin/20-run-ipv6servers-stats.sh new file mode 100755 index 0000000..5d7bd13 --- /dev/null +++ b/shared/bin/20-run-ipv6servers-stats.sh @@ -0,0 +1,5 @@ +#!/bin/sh +cd modules/ipv6servers/ +ant run | grep "[java]" +cd ../../ + diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib deleted file mode 160000 index 79a4b98..0000000 --- a/src/submods/metrics-lib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 79a4b9866f2a342159bd0811d83e9ec62169c6d9