commit 25072720b90f5f725c50ee7b645efc4777d68da6 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Apr 20 11:04:14 2019 +0200

Add BandwidthFile for parsed bandwidth files.

Implements #30216. --- CHANGELOG.md | 6 + .../org/torproject/descriptor/BandwidthFile.java | 249 +++++++++ .../descriptor/impl/BandwidthFileImpl.java | 431 +++++++++++++++ .../descriptor/impl/DescriptorParserImpl.java | 9 + .../descriptor/impl/BandwidthFileImplTest.java | 596 +++++++++++++++++++++ .../descriptor/impl/TestDescriptorBuilder.java | 120 +++++ 6 files changed, 1411 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 3353a4c..d9cb62a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 2.6.0 - 2019-04-?? + + * Medium changes + - Add new BandwidthFile descriptor for parsed bandwidth files. + + # Changes in version 2.5.0 - 2018-09-25

* Medium changes diff --git a/src/main/java/org/torproject/descriptor/BandwidthFile.java b/src/main/java/org/torproject/descriptor/BandwidthFile.java new file mode 100644 index 0000000..34b9414 --- /dev/null +++ b/src/main/java/org/torproject/descriptor/BandwidthFile.java @@ -0,0 +1,249 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A bandwidth file contains information on relays' bandwidth capacities and is + * produced by bandwidth generators, previously known as bandwidth scanners. + * + * @since 2.6.0 + */ +public interface BandwidthFile extends Descriptor { + + /** + * Time of the most recent generator bandwidth result. + * + * @since 2.6.0 + */ + LocalDateTime timestamp(); + + /** + * Document format version. + * + * @since 2.6.0 + */ + String version(); + + /** + * Name of the software that created the document. + * + * @since 2.6.0 + */ + String software(); + + /** + * Version of the software that created the document. + * + * @since 2.6.0 + */ + Optional<String> softwareVersion(); + + /** + * Timestamp in UTC time zone when the file was created. + * + * @since 2.6.0 + */ + Optional<LocalDateTime> fileCreated(); + + /** + * Timestamp in UTC time zone when the generator was started. + * + * @since 2.6.0 + */ + Optional<LocalDateTime> generatorStarted(); + + /** + * Timestamp in UTC time zone when the first relay bandwidth was obtained. + * + * @since 2.6.0 + */ + Optional<LocalDateTime> earliestBandwidth(); + + /** + * Timestamp in UTC time zone of the most recent generator bandwidth result. + * + * @since 2.6.0 + */ + Optional<LocalDateTime> latestBandwidth(); + + /** + * Number of relays that have enough measurements to be included in the + * bandwidth file. + * + * @since 2.6.0 + */ + Optional<Integer> numberEligibleRelays(); + + /** + * Percentage of relays in the consensus that should be included in every + * generated bandwidth file. + * + * @since 2.6.0 + */ + Optional<Integer> minimumPercentEligibleRelays(); + + /** + * Number of relays in the consensus. + * + * @since 2.6.0 + */ + Optional<Integer> numberConsensusRelays(); + + /** + * The number of eligible relays, as a percentage of the number of relays in + * the consensus. + * + * @since 2.6.0 + */ + Optional<Integer> percentEligibleRelays(); + + /** + * Minimum number of relays that should be included in the bandwidth file. + * + * @since 2.6.0 + */ + Optional<Integer> minimumNumberEligibleRelays(); + + /** + * Country, as in political geolocation, where the generator is run. + * + * @since 2.6.0 + */ + Optional<String> scannerCountry(); + + /** + * Country, as in political geolocation, or countries where the destination + * web server(s) are located. + * + * @since 2.6.0 + */ + Optional<String[]> destinationsCountries(); + + /** + * Number of the different consensuses seen in the last data period. + * + * @since 2.6.0 + */ + Optional<Integer> recentConsensusCount(); + + /** + * Number of times that a list with a subset of relays prioritized to be + * measured has been created in the last data period. + * + * @since 2.6.0 + */ + Optional<Integer> recentPriorityListCount(); + + /** + * Number of relays that has been in in the list of relays prioritized to be + * measured in the last data period. + * + * @since 2.6.0 + */ + Optional<Integer> recentPriorityRelayCount(); + + /** + * Number of times that any relay has been queued to be measured in the last + * data period. + * + * @since 2.6.0 + */ + Optional<Integer> recentMeasurementAttemptCount(); + + /** + * Number of times that the scanner attempted to measure a relay in the last + * data period, but the relay has not been measured because of system, network + * or implementation issues. + * + * @since 2.6.0 + */ + Optional<Integer> recentMeasurementFailureCount(); + + /** + * Number of relays that have no successful measurements in the last data + * period. + * + * @since 2.6.0 + */ + Optional<Integer> recentMeasurementsExcludedErrorCount(); + + /** + * Number of relays that have some successful measurements in the last data + * period, but all those measurements were performed in a period of time that + * was too short. + * + * @since 2.6.0 + */ + Optional<Integer> recentMeasurementsExcludedNearCount(); + + /** + * Number of relays that have some successful measurements, but all those + * measurements are too old. + * + * @since 2.6.0 + */ + Optional<Integer> recentMeasurementsExcludedOldCount(); + + /** + * Number of relays that don't have enough recent successful measurements. + * + * @since 2.6.0 + */ + Optional<Integer> recentMeasurementsExcludedFewCount(); + + /** + * Time that it would take to report measurements about half of the network, + * given the number of eligible relays and the time it took in the last days. + * + * @since 2.6.0 + */ + Optional<Duration> timeToReportHalfNetwork(); + + /** + * List of zero or more {@link RelayLine}s containing relay identities and + * bandwidths in the order as they are contained in the bandwidth file. + * + * @since 2.6.0 + */ + List<RelayLine> relayLines(); + + interface RelayLine { + + /** + * Fingerprint for the relay's RSA identity key. + * + * @since 2.6.0 + */ + Optional<String> nodeId(); + + /** + * Relays's master Ed25519 key, base64 encoded, without trailing "="s. + * + * @since 2.6.0 + */ + Optional<String> masterKeyEd25519(); + + /** + * Bandwidth of this relay in kilobytes per second. + * + * @since 2.6.0 + */ + int bw(); + + /** + * Additional relay key-value pairs, excluding the key value pairs already + * parsed for relay identities and bandwidths. + * + * @since 2.6.0 + */ + Map<String, String> additionalKeyValues(); + } +} + diff --git a/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java b/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java new file mode 100644 index 0000000..5d661e4 --- /dev/null +++ b/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java @@ -0,0 +1,431 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.impl; + +import org.torproject.descriptor.BandwidthFile; +import org.torproject.descriptor.DescriptorParseException; + +import java.io.File; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Scanner; + +public class BandwidthFileImpl extends DescriptorImpl implements BandwidthFile { + + private enum KeyWithStringValue { + version, software, software_version + } + + private enum KeyWithLocalDateTimeValue { + file_created, generator_started, earliest_bandwidth, latest_bandwidth + } + + private enum KeyWithIntValue { + number_eligible_relays, minimum_percent_eligible_relays, + number_consensus_relays, percent_eligible_relays, + minimum_number_eligible_relays, recent_consensus_count, + recent_priority_list_count, recent_priority_relay_count, + recent_measurement_attempt_count, recent_measurement_failure_count, + recent_measurements_excluded_error_count, + recent_measurements_excluded_near_count, + recent_measurements_excluded_old_count, + recent_measurements_excluded_few_count + } + + BandwidthFileImpl(byte[] rawDescriptorBytes, File descriptorfile) + throws DescriptorParseException { + super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length }, + descriptorfile, false); + Scanner scanner = this.newScanner().useDelimiter("\n"); + this.parseTimestampLine(scanner.nextLine()); + boolean haveFinishedParsingHeader = false; + while (scanner.hasNext()) { + String line = scanner.nextLine(); + if (!haveFinishedParsingHeader) { + if (line.startsWith("bw=") || line.contains(" bw=")) { + haveFinishedParsingHeader = true; + } else if ("====".equals(line) || "=====".equals(line)) { + haveFinishedParsingHeader = true; + continue; + } + } + if (!haveFinishedParsingHeader) { + this.parseHeaderLine(line); + } else { + this.parseRelayLine(line); + } + } + } + + private void parseTimestampLine(String line) throws DescriptorParseException { + try { + this.timestamp = LocalDateTime.ofInstant(Instant.ofEpochSecond( + Long.parseLong(line)), ZoneOffset.UTC); + } catch (NumberFormatException | DateTimeParseException e) { + throw new DescriptorParseException(String.format( + "Unable to parse timestamp in first line: '%s'.", line), e); + } + } + + private void parseHeaderLine(String line) throws DescriptorParseException { + String[] keyValueParts = line.split("=", 2); + if (keyValueParts.length != 2) { + throw new DescriptorParseException(String.format( + "Unrecognized line '%s' without '=' character.", line)); + } + String key = keyValueParts[0]; + if (key.length() < 1) { + throw new DescriptorParseException(String.format( + "Unrecognized line '%s' starting with '=' character.", line)); + } + String value = keyValueParts[1]; + switch (key) { + case "version": + case "software": + case "software_version": + this.parsedStrings.put(KeyWithStringValue.valueOf(key), value); + break; + case "file_created": + case "generator_started": + case "earliest_bandwidth": + case "latest_bandwidth": + try { + this.parsedLocalDateTimes.put(KeyWithLocalDateTimeValue.valueOf(key), + LocalDateTime.parse(value)); + } catch (DateTimeParseException e) { + throw new DescriptorParseException(String.format( + "Unable to parse date-time string: '%s'.", value), e); + } + break; + case "number_eligible_relays": + case "minimum_percent_eligible_relays": + case "number_consensus_relays": + case "percent_eligible_relays": + case "minimum_number_eligible_relays": + case "recent_consensus_count": + case "recent_priority_list_count": + case "recent_priority_relay_count": + case "recent_measurement_attempt_count": + case "recent_measurement_failure_count": + case "recent_measurements_excluded_error_count": + case "recent_measurements_excluded_near_count": + case "recent_measurements_excluded_old_count": + case "recent_measurements_excluded_few_count": + try { + this.parsedInts.put(KeyWithIntValue.valueOf(key), + Integer.parseInt(value)); + } catch (NumberFormatException e) { + throw new DescriptorParseException(String.format( + "Unable to parse int: '%s'.", value), e); + } + break; + case "scanner_country": + if (!value.matches("[A-Z]{2}")) { + throw new DescriptorParseException(String.format( + "Invalid country code '%s'.", value)); + } + this.scannerCountry = value; + break; + case "destinations_countries": + if (!value.matches("[A-Z]{2}(,[A-Z]{2})*")) { + throw new DescriptorParseException(String.format( + "Invalid country code list '%s'.", value)); + } + this.destinationsCountries = value.split(","); + break; + case "time_to_report_half_network": + try { + this.timeToReportHalfNetwork + = Duration.ofSeconds(Long.parseLong(value)); + } catch (NumberFormatException | DateTimeParseException e) { + throw new DescriptorParseException(String.format( + "Unable to parse duration: '%s'.", value), e); + } + break; + case "node_id": + case "master_key_ed25519": + case "bw": + throw new DescriptorParseException(String.format( + "Either additional header line must not use keywords specified in " + + "relay lines, or relay line is missing required keys: '%s'.", + line)); + default: + /* Ignore additional header lines. */ + } + } + + private class RelayLineImpl implements RelayLine { + + private String nodeId; + + @Override + public Optional<String> nodeId() { + return Optional.ofNullable(this.nodeId); + } + + private String masterKeyEd25519; + + @Override + public Optional<String> masterKeyEd25519() { + return Optional.ofNullable(this.masterKeyEd25519); + } + + private int bw; + + @Override + public int bw() { + return this.bw; + } + + private Map<String, String> additionalKeyValues; + + @Override + public Map<String, String> additionalKeyValues() { + return null == this.additionalKeyValues ? Collections.emptyMap() + : Collections.unmodifiableMap(this.additionalKeyValues); + } + + private RelayLineImpl(String nodeId, String masterKeyEd25519, int bw, + Map<String, String> additionalKeyValues) { + this.nodeId = nodeId; + this.masterKeyEd25519 = masterKeyEd25519; + this.bw = bw; + this.additionalKeyValues = additionalKeyValues; + } + } + + private void parseRelayLine(String line) throws DescriptorParseException { + String[] spaceSeparatedLineParts = line.split(" "); + String nodeId = null; + String masterKeyEd25519 = null; + Integer bw = null; + Map<String, String> additionalKeyValues = new LinkedHashMap<>(); + for (String spaceSeparatedLinePart : spaceSeparatedLineParts) { + String[] keyValueParts = spaceSeparatedLinePart.split("=", 2); + if (keyValueParts.length != 2) { + throw new DescriptorParseException(String.format( + "Unrecognized space-separated line part '%s' without '=' " + + "character in line '%s'.", spaceSeparatedLinePart, line)); + } + String key = keyValueParts[0]; + if (key.length() < 1) { + throw new DescriptorParseException(String.format( + "Unrecognized space-separated line part '%s' starting with '=' " + + "character in line '%s'.", spaceSeparatedLinePart, line)); + } + String value = keyValueParts[1]; + switch (key) { + case "node_id": + nodeId = value; + break; + case "master_key_ed25519": + masterKeyEd25519 = value; + break; + case "bw": + try { + bw = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new DescriptorParseException(String.format( + "Unable to parse bw '%s' in line '%s'.", value, line), e); + } + break; + default: + additionalKeyValues.put(key, value); + } + } + if (null == nodeId && null == masterKeyEd25519) { + throw new DescriptorParseException(String.format( + "Expected relay line, but line contains neither node_id nor " + + "master_key_ed25519: '%s'.", line)); + } + if (null == bw) { + throw new DescriptorParseException(String.format( + "Expected relay line, but line does not contain bw: '%s'.", line)); + } + this.relayLines.add(new RelayLineImpl(nodeId, masterKeyEd25519, bw, + additionalKeyValues.isEmpty() ? null : additionalKeyValues)); + } + + private LocalDateTime timestamp; + + @Override + public LocalDateTime timestamp() { + return this.timestamp; + } + + private EnumMap<KeyWithStringValue, String> parsedStrings + = new EnumMap<>(KeyWithStringValue.class); + + @Override + public String version() { + return this.parsedStrings.getOrDefault(KeyWithStringValue.version, + "1.0.0"); + } + + @Override + public String software() { + return this.parsedStrings.getOrDefault(KeyWithStringValue.software, + "torflow"); + } + + @Override + public Optional<String> softwareVersion() { + return Optional.ofNullable( + this.parsedStrings.get(KeyWithStringValue.software_version)); + } + + private EnumMap<KeyWithLocalDateTimeValue, LocalDateTime> parsedLocalDateTimes + = new EnumMap<>(KeyWithLocalDateTimeValue.class); + + @Override + public Optional<LocalDateTime> fileCreated() { + return Optional.ofNullable(this.parsedLocalDateTimes.get( + KeyWithLocalDateTimeValue.file_created)); + } + + @Override + public Optional<LocalDateTime> generatorStarted() { + return Optional.ofNullable(this.parsedLocalDateTimes.get( + KeyWithLocalDateTimeValue.generator_started)); + } + + @Override + public Optional<LocalDateTime> earliestBandwidth() { + return Optional.ofNullable(this.parsedLocalDateTimes.get( + KeyWithLocalDateTimeValue.earliest_bandwidth)); + } + + @Override + public Optional<LocalDateTime> latestBandwidth() { + return Optional.ofNullable(this.parsedLocalDateTimes.get( + KeyWithLocalDateTimeValue.latest_bandwidth)); + } + + private EnumMap<KeyWithIntValue, Integer> parsedInts + = new EnumMap<>(KeyWithIntValue.class); + + @Override + public Optional<Integer> numberEligibleRelays() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.number_eligible_relays)); + } + + @Override + public Optional<Integer> minimumPercentEligibleRelays() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.minimum_percent_eligible_relays)); + } + + @Override + public Optional<Integer> numberConsensusRelays() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.number_consensus_relays)); + } + + @Override + public Optional<Integer> percentEligibleRelays() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.percent_eligible_relays)); + } + + @Override + public Optional<Integer> minimumNumberEligibleRelays() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.minimum_number_eligible_relays)); + } + + private String scannerCountry; + + @Override + public Optional<String> scannerCountry() { + return Optional.ofNullable(this.scannerCountry); + } + + private String[] destinationsCountries; + + @Override + public Optional<String[]> destinationsCountries() { + return Optional.ofNullable(this.destinationsCountries); + } + + @Override + public Optional<Integer> recentConsensusCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_consensus_count)); + } + + @Override + public Optional<Integer> recentPriorityListCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_priority_list_count)); + } + + @Override + public Optional<Integer> recentPriorityRelayCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_priority_relay_count)); + } + + @Override + public Optional<Integer> recentMeasurementAttemptCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_measurement_attempt_count)); + } + + @Override + public Optional<Integer> recentMeasurementFailureCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_measurement_failure_count)); + } + + @Override + public Optional<Integer> recentMeasurementsExcludedErrorCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_measurements_excluded_error_count)); + } + + @Override + public Optional<Integer> recentMeasurementsExcludedNearCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_measurements_excluded_near_count)); + } + + @Override + public Optional<Integer> recentMeasurementsExcludedOldCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_measurements_excluded_old_count)); + } + + @Override + public Optional<Integer> recentMeasurementsExcludedFewCount() { + return Optional.ofNullable(this.parsedInts.get( + KeyWithIntValue.recent_measurements_excluded_few_count)); + } + + private Duration timeToReportHalfNetwork; + + @Override + public Optional<Duration> timeToReportHalfNetwork() { + return Optional.ofNullable(this.timeToReportHalfNetwork); + } + + private List<RelayLine> relayLines = new ArrayList<>(); + + @Override + public List<RelayLine> relayLines() { + return this.relayLines.isEmpty() ? Collections.emptyList() + : Collections.unmodifiableList(this.relayLines); + } +} + diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java index e8b8b08..119fe09 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -132,6 +132,15 @@ public class DescriptorParserImpl implements DescriptorParser { sourceFile); } else if (fileName.contains(LogDescriptorImpl.MARKER)) { return LogDescriptorImpl.parse(rawDescriptorBytes, sourceFile, fileName); + } else if (firstLines.matches("^[0-9]{10}\n")) { + /* Identifying bandwidth files by a 10-digit timestamp in the first line + * breaks with files generated before 2002 or after 2286 and when the next + * descriptor identifier starts with just a timestamp in the first line + * rather than a document type identifier. */ + List<Descriptor> parsedDescriptors = new ArrayList<>(); + parsedDescriptors.add(new BandwidthFileImpl(rawDescriptorBytes, + sourceFile)); + return parsedDescriptors; } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); diff --git a/src/test/java/org/torproject/descriptor/impl/BandwidthFileImplTest.java b/src/test/java/org/torproject/descriptor/impl/BandwidthFileImplTest.java new file mode 100644 index 0000000..d19b7e7 --- /dev/null +++ b/src/test/java/org/torproject/descriptor/impl/BandwidthFileImplTest.java @@ -0,0 +1,596 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.impl; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import org.torproject.descriptor.BandwidthFile; +import org.torproject.descriptor.DescriptorParseException; + +import org.hamcrest.Matchers; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +public class BandwidthFileImplTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + /** + * Example from bandwidth-file-spec.txt: Version 1.0.0, generated by Torflow. + */ + private static final String[] specExample100 = new String[] { + "1523911758", + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=760 nick=Test " + + "measured_at=1523911725 updated_at=1523911725 " + + "pid_error=4.11374090719 pid_error_sum=4.11374090719 " + + "pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 " + + "scanner=/filepath", + "node_id=$96C15995F30895689291F455587BD94CA427B6FC bw=189 nick=Test2 " + + "measured_at=1523911623 updated_at=1523911623 " + + "pid_error=3.96703337994 pid_error_sum=3.96703337994 " + + "pid_bw=47422125 pid_delta=2.65469736988 circ_fail=0.0 " + + "scanner=/filepath" }; + + @Test + public void testSpecExample100() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample100).build(), null); + assertEquals(LocalDateTime.of(2018, 4, 16, 20, 49, 18), + bandwidthFile.timestamp()); + assertEquals("1.0.0", bandwidthFile.version()); + assertEquals("torflow", bandwidthFile.software()); + assertFalse(bandwidthFile.softwareVersion().isPresent()); + assertFalse(bandwidthFile.fileCreated().isPresent()); + assertFalse(bandwidthFile.generatorStarted().isPresent()); + assertFalse(bandwidthFile.earliestBandwidth().isPresent()); + assertFalse(bandwidthFile.latestBandwidth().isPresent()); + assertFalse(bandwidthFile.numberEligibleRelays().isPresent()); + assertFalse(bandwidthFile.minimumPercentEligibleRelays().isPresent()); + assertFalse(bandwidthFile.numberConsensusRelays().isPresent()); + assertFalse(bandwidthFile.percentEligibleRelays().isPresent()); + assertFalse(bandwidthFile.minimumNumberEligibleRelays().isPresent()); + assertFalse(bandwidthFile.scannerCountry().isPresent()); + assertFalse(bandwidthFile.destinationsCountries().isPresent()); + assertFalse(bandwidthFile.recentConsensusCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityListCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityRelayCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementAttemptCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementFailureCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedErrorCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedNearCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedOldCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedFewCount().isPresent()); + assertFalse(bandwidthFile.timeToReportHalfNetwork().isPresent()); + assertEquals(2, bandwidthFile.relayLines().size()); + BandwidthFile.RelayLine firstRelayLine = bandwidthFile.relayLines().get(0); + assertEquals(Optional.of("$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80"), + firstRelayLine.nodeId()); + assertFalse(firstRelayLine.masterKeyEd25519().isPresent()); + assertEquals(760, firstRelayLine.bw()); + Map<String, String> expectedFirstAdditionalKeyValues + = new LinkedHashMap<>(); + expectedFirstAdditionalKeyValues.put("nick", "Test"); + expectedFirstAdditionalKeyValues.put("measured_at", "1523911725"); + expectedFirstAdditionalKeyValues.put("updated_at", "1523911725"); + expectedFirstAdditionalKeyValues.put("pid_error", "4.11374090719"); + expectedFirstAdditionalKeyValues.put("pid_error_sum", "4.11374090719"); + expectedFirstAdditionalKeyValues.put("pid_bw", "57136645"); + expectedFirstAdditionalKeyValues.put("pid_delta", "2.12168374577"); + expectedFirstAdditionalKeyValues.put("circ_fail", "0.2"); + expectedFirstAdditionalKeyValues.put("scanner", "/filepath"); + assertEquals(expectedFirstAdditionalKeyValues, + firstRelayLine.additionalKeyValues()); + } + + @Test + public void testTimestampAsKeyValue() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Unable to parse timestamp in first line")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample100) + .replaceLineStartingWith("1523911758", "timestamp=1523911758") + .build(), null); + } + + @Test + public void testEmptyLine() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Blank lines are not allowed.")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample100) + .appendLines("") + .build(), null); + } + + @Test + public void testHeaderLineAtEnd() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Expected relay line, but line contains neither node_id nor " + + "master_key_ed25519")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample100) + .appendLines("version=1.0.0") + .build(), null); + } + + @Test + public void testRelayLineWithoutRelayId() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Either additional header line must not use keywords specified in " + + "relay lines, or relay line is missing required keys")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample100) + .replaceLineStartingWith( + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80", + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80") + .build(), null); + } + + @Test + public void testRelayLineWithoutBw() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Expected relay line, but line contains neither node_id nor " + + "master_key_ed25519")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample100) + .replaceLineStartingWith( + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80", "bw=760") + .build(), null); + } + + @Test + public void testBwNotANumber() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Unable to parse bw 'slow' in line")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample100) + .replaceLineStartingWith( + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80", + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=slow") + .build(), null); + } + + @Test + public void testRelayLineTrailingSpace() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample100) + .replaceLineStartingWith( + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80", + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=760 ") + .build(), null); + /* It's okay that this line ends with a space, we're parsing it anyway. */ + assertEquals(2, bandwidthFile.relayLines().size()); + BandwidthFile.RelayLine firstRelayLine = bandwidthFile.relayLines().get(0); + assertEquals(Optional.of("$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80"), + firstRelayLine.nodeId()); + assertEquals(760, firstRelayLine.bw()); + } + + /** + * Example from bandwidth-file-spec.txt: Version 1.1.0, generated by sbws + * version 0.1.0. + */ + private static final String[] specExample110 = new String[] { + "1523911758", + "version=1.1.0", + "software=sbws", + "software_version=0.1.0", + "latest_bandwidth=2018-04-16T20:49:18", + "file_created=2018-04-16T21:49:18", + "generator_started=2018-04-16T15:13:25", + "earliest_bandwidth=2018-04-16T15:13:26", + "====", + "bw=380 error_circ=0 error_misc=0 error_stream=1 " + + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + + "nick=Test node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + + "rtt=380 success=1 time=2018-05-08T16:13:26", + "bw=189 error_circ=0 error_misc=0 error_stream=0 " + + "master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I " + + "nick=Test2 node_id=$96C15995F30895689291F455587BD94CA427B6FC " + + "rtt=378 success=1 time=2018-05-08T16:13:36" }; + + @Test + public void testSpecExample110() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample110).build(), null); + assertEquals(LocalDateTime.of(2018, 4, 16, 20, 49, 18), + bandwidthFile.timestamp()); + assertEquals("1.1.0", bandwidthFile.version()); + assertEquals("sbws", bandwidthFile.software()); + assertEquals(Optional.of("0.1.0"), bandwidthFile.softwareVersion()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 21, 49, 18)), + bandwidthFile.fileCreated()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 25)), + bandwidthFile.generatorStarted()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 26)), + bandwidthFile.earliestBandwidth()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 20, 49, 18)), + bandwidthFile.latestBandwidth()); + assertFalse(bandwidthFile.numberEligibleRelays().isPresent()); + assertFalse(bandwidthFile.minimumPercentEligibleRelays().isPresent()); + assertFalse(bandwidthFile.numberConsensusRelays().isPresent()); + assertFalse(bandwidthFile.percentEligibleRelays().isPresent()); + assertFalse(bandwidthFile.minimumNumberEligibleRelays().isPresent()); + assertFalse(bandwidthFile.scannerCountry().isPresent()); + assertFalse(bandwidthFile.destinationsCountries().isPresent()); + assertFalse(bandwidthFile.recentConsensusCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityListCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityRelayCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementAttemptCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementFailureCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedErrorCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedNearCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedOldCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedFewCount().isPresent()); + assertFalse(bandwidthFile.timeToReportHalfNetwork().isPresent()); + } + + @Test + public void testTerminatorLineTooShort() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Unrecognized line '===' starting with '=' character")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample110) + .replaceLineStartingWith("====", "===").build(), null); + } + + @Test + public void testDateTimeContainingSpace() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Unable to parse date-time string")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample110) + .replaceLineStartingWith("earliest_bandwidth", + "earliest_bandwidth=2018-04-16 15:13:26") + .build(), null); + } + + /** + * Example from bandwidth-file-spec.txt: Version 1.2.0, generated by sbws + * version 1.0.3. + */ + private static final String[] specExample120 = new String[] { + "1523911758", + "version=1.2.0", + "latest_bandwidth=2018-04-16T20:49:18", + "file_created=2018-04-16T21:49:18", + "generator_started=2018-04-16T15:13:25", + "earliest_bandwidth=2018-04-16T15:13:26", + "minimum_number_eligible_relays=3862", + "minimum_percent_eligible_relays=60", + "number_consensus_relays=6436", + "number_eligible_relays=6000", + "percent_eligible_relays=93", + "software=sbws", + "software_version=1.0.3", + "=====", + "bw=38000 bw_mean=1127824 bw_median=1180062 desc_avg_bw=1073741824 " + + "desc_obs_bw_last=17230879 desc_obs_bw_mean=14732306 error_circ=0 " + + "error_misc=0 error_stream=1 " + + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + + "nick=Test node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + + "rtt=380 success=1 time=2018-05-08T16:13:26", + "bw=1 bw_mean=199162 bw_median=185675 desc_avg_bw=409600 " + + "desc_obs_bw_last=836165 desc_obs_bw_mean=858030 error_circ=0 " + + "error_misc=0 error_stream=0 " + + "master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I " + + "nick=Test2 node_id=$96C15995F30895689291F455587BD94CA427B6FC " + + "rtt=378 success=1 time=2018-05-08T16:13:36" }; + + @Test + public void testSpecExample120() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample120).build(), null); + assertEquals(LocalDateTime.of(2018, 4, 16, 20, 49, 18), + bandwidthFile.timestamp()); + assertEquals("1.2.0", bandwidthFile.version()); + assertEquals("sbws", bandwidthFile.software()); + assertEquals(Optional.of("1.0.3"), bandwidthFile.softwareVersion()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 21, 49, 18)), + bandwidthFile.fileCreated()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 25)), + bandwidthFile.generatorStarted()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 26)), + bandwidthFile.earliestBandwidth()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 20, 49, 18)), + bandwidthFile.latestBandwidth()); + assertEquals(Optional.of(6000), bandwidthFile.numberEligibleRelays()); + assertEquals(Optional.of(60), bandwidthFile.minimumPercentEligibleRelays()); + assertEquals(Optional.of(6436), bandwidthFile.numberConsensusRelays()); + assertEquals(Optional.of(93), bandwidthFile.percentEligibleRelays()); + assertEquals(Optional.of(3862), + bandwidthFile.minimumNumberEligibleRelays()); + assertFalse(bandwidthFile.scannerCountry().isPresent()); + assertFalse(bandwidthFile.destinationsCountries().isPresent()); + assertFalse(bandwidthFile.recentConsensusCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityListCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityRelayCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementAttemptCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementFailureCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedErrorCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedNearCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedOldCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedFewCount().isPresent()); + assertFalse(bandwidthFile.timeToReportHalfNetwork().isPresent()); + } + + @Test + public void testNumberEligibleRelaysNotAnInt() + throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Unable to parse int")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample120) + .replaceLineStartingWith("number_eligible_relays=6000", + "number_eligible_relays=sixthousand").build(), null); + } + + /** + * Example from bandwidth-file-spec.txt: Version 1.2.0, generated by sbws + * version 1.0.3 when there are not enough eligible measured relays. + */ + private static final String[] specExample120NotEnough = new String[] { + "1540496079", + "version=1.2.0", + "earliest_bandwidth=2018-10-20T19:35:52", + "file_created=2018-10-25T19:35:03", + "generator_started=2018-10-25T11:42:56", + "latest_bandwidth=2018-10-25T19:34:39", + "minimum_number_eligible_relays=3862", + "minimum_percent_eligible_relays=60", + "number_consensus_relays=6436", + "number_eligible_relays=2960", + "percent_eligible_relays=46", + "software=sbws", + "software_version=1.0.3", + "=====" }; + + @Test + public void testSpecExample120NotEnough() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample120NotEnough).build(), null); + assertEquals(LocalDateTime.of(2018, 10, 25, 19, 34, 39), + bandwidthFile.timestamp()); + assertEquals("1.2.0", bandwidthFile.version()); + assertEquals("sbws", bandwidthFile.software()); + assertEquals(Optional.of("1.0.3"), bandwidthFile.softwareVersion()); + assertEquals(Optional.of(LocalDateTime.of(2018, 10, 25, 19, 35, 3)), + bandwidthFile.fileCreated()); + assertEquals(Optional.of(LocalDateTime.of(2018, 10, 25, 11, 42, 56)), + bandwidthFile.generatorStarted()); + assertEquals(Optional.of(LocalDateTime.of(2018, 10, 20, 19, 35, 52)), + bandwidthFile.earliestBandwidth()); + assertEquals(Optional.of(LocalDateTime.of(2018, 10, 25, 19, 34, 39)), + bandwidthFile.latestBandwidth()); + assertEquals(Optional.of(2960), bandwidthFile.numberEligibleRelays()); + assertEquals(Optional.of(60), bandwidthFile.minimumPercentEligibleRelays()); + assertEquals(Optional.of(6436), bandwidthFile.numberConsensusRelays()); + assertEquals(Optional.of(46), bandwidthFile.percentEligibleRelays()); + assertEquals(Optional.of(3862), + bandwidthFile.minimumNumberEligibleRelays()); + assertFalse(bandwidthFile.scannerCountry().isPresent()); + assertFalse(bandwidthFile.destinationsCountries().isPresent()); + assertFalse(bandwidthFile.recentConsensusCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityListCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityRelayCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementAttemptCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementFailureCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedErrorCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedNearCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedOldCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedFewCount().isPresent()); + assertFalse(bandwidthFile.timeToReportHalfNetwork().isPresent()); + } + + /** + * Example from bandwidth-file-spec.txt: Version 1.3.0 headers generated by + * sbws version 1.0.4. + */ + private static final String[] specExample130Headers = new String[] { + "1523911758", + "version=1.3.0", + "latest_bandwidth=2018-04-16T20:49:18", + "destinations_countries=TH,ZZ", + "file_created=2018-04-16T21:49:18", + "generator_started=2018-04-16T15:13:25", + "earliest_bandwidth=2018-04-16T15:13:26", + "minimum_number_eligible_relays=3862", + "minimum_percent_eligible_relays=60", + "number_consensus_relays=6436", + "number_eligible_relays=6000", + "percent_eligible_relays=93", + "scanner_country=SN", + "software=sbws", + "software_version=1.0.4", + "=====" }; + + @Test + public void testSpecExample130Headers() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample130Headers).build(), null); + assertEquals(LocalDateTime.of(2018, 4, 16, 20, 49, 18), + bandwidthFile.timestamp()); + assertEquals("1.3.0", bandwidthFile.version()); + assertEquals("sbws", bandwidthFile.software()); + assertEquals(Optional.of("1.0.4"), bandwidthFile.softwareVersion()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 21, 49, 18)), + bandwidthFile.fileCreated()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 25)), + bandwidthFile.generatorStarted()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 26)), + bandwidthFile.earliestBandwidth()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 20, 49, 18)), + bandwidthFile.latestBandwidth()); + assertEquals(Optional.of(6000), bandwidthFile.numberEligibleRelays()); + assertEquals(Optional.of(60), bandwidthFile.minimumPercentEligibleRelays()); + assertEquals(Optional.of(6436), bandwidthFile.numberConsensusRelays()); + assertEquals(Optional.of(93), bandwidthFile.percentEligibleRelays()); + assertEquals(Optional.of(3862), + bandwidthFile.minimumNumberEligibleRelays()); + assertEquals(Optional.of("SN"), bandwidthFile.scannerCountry()); + assertArrayEquals(new String[] { "TH", "ZZ" }, + bandwidthFile.destinationsCountries().orElse(null)); + assertFalse(bandwidthFile.recentConsensusCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityListCount().isPresent()); + assertFalse(bandwidthFile.recentPriorityRelayCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementAttemptCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementFailureCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedErrorCount().isPresent()); + assertFalse( + bandwidthFile.recentMeasurementsExcludedNearCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedOldCount().isPresent()); + assertFalse(bandwidthFile.recentMeasurementsExcludedFewCount().isPresent()); + assertFalse(bandwidthFile.timeToReportHalfNetwork().isPresent()); + } + + @Test + public void testScannerCountryLowerCase() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Invalid country code 'sn'.")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample130Headers) + .replaceLineStartingWith("scanner_country", "scanner_country=sn") + .build(), null); + } + + @Test + public void testDestinationsCountriesLowerCase() + throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Invalid country code list 'th,zz'.")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample130Headers) + .replaceLineStartingWith("destinations_countries", + "destinations_countries=th,zz") + .build(), null); + } + + @Test + public void testDestinationsCountriesEndingWithComma() + throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Invalid country code list 'TH,'.")); + new BandwidthFileImpl(new TestDescriptorBuilder(specExample130Headers) + .replaceLineStartingWith("destinations_countries", + "destinations_countries=TH,") + .build(), null); + } + + /** + * Example from bandwidth-file-spec.txt: Version 1.4.0 generated by sbws + * version 1.1.0. + */ + private static final String[] specExample140 = new String[] { + "1523911758", + "version=1.4.0", + "latest_bandwidth=2018-04-16T20:49:18", + "destinations_countries=TH,ZZ", + "file_created=2018-04-16T21:49:18", + "generator_started=2018-04-16T15:13:25", + "earliest_bandwidth=2018-04-16T15:13:26", + "minimum_number_eligible_relays=3862", + "minimum_percent_eligible_relays=60", + "number_consensus_relays=6436", + "number_eligible_relays=6000", + "percent_eligible_relays=93", + "recent_measurement_attempt_count=6243", + "recent_measurement_failure_count=732", + "recent_measurements_excluded_error_count=969", + "recent_measurements_excluded_few_count=3946", + "recent_measurements_excluded_near_count=90", + "recent_measurements_excluded_old_count=0", + "recent_priority_list_count=20", + "recent_priority_relay_count=6243", + "scanner_country=SN", + "software=sbws", + "software_version=1.1.0", + "time_to_report_half_network=57273", + "=====", + "bw=1 error_circ=1 error_destination=0 error_misc=0 error_second_relay=0 " + + "error_stream=0 " + + "master_key_ed25519=J3HQ24kOQWac3L1xlFLp7gY91qkb5NuKxjj1BhDi+m8 " + + "nick=snap269 node_id=$DC4D609F95A52614D1E69C752168AF1FCAE0B05F " + + "relay_recent_measurement_attempt_count=3 " + + "relay_recent_measurements_excluded_error_count=1 " + + "relay_recent_measurements_excluded_near_count=3 " + + "relay_recent_consensus_count=3 relay_recent_priority_list_count=3 " + + "success=3 time=2019-03-16T18:20:57 unmeasured=1 vote=0", + "bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 " + + "error_stream=2 " + + "master_key_ed25519=h6ZB1E1yBFWIMloUm9IWwjgaPXEpL5cUbuoQDgdSDKg " + + "nick=relay node_id=$C4544F9E209A9A9B99591D548B3E2822236C0503 " + + "relay_recent_measurement_attempt_count=3 " + + "relay_recent_measurements_excluded_error_count=2 " + + "relay_recent_measurements_excluded_few_count=1 " + + "relay_recent_consensus_count=3 relay_recent_priority_list_count=3 " + + "success=1 time=2019-03-17T06:50:58 unmeasured=1 vote=0" }; + + @Test + public void testSpecExample140() throws DescriptorParseException { + BandwidthFile bandwidthFile = new BandwidthFileImpl( + new TestDescriptorBuilder(specExample140).build(), null); + assertEquals(LocalDateTime.of(2018, 4, 16, 20, 49, 18), + bandwidthFile.timestamp()); + assertEquals("1.4.0", bandwidthFile.version()); + assertEquals("sbws", bandwidthFile.software()); + assertEquals(Optional.of("1.1.0"), bandwidthFile.softwareVersion()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 21, 49, 18)), + bandwidthFile.fileCreated()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 25)), + bandwidthFile.generatorStarted()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 15, 13, 26)), + bandwidthFile.earliestBandwidth()); + assertEquals(Optional.of(LocalDateTime.of(2018, 4, 16, 20, 49, 18)), + bandwidthFile.latestBandwidth()); + assertEquals(Optional.of(6000), bandwidthFile.numberEligibleRelays()); + assertEquals(Optional.of(60), bandwidthFile.minimumPercentEligibleRelays()); + assertEquals(Optional.of(6436), bandwidthFile.numberConsensusRelays()); + assertEquals(Optional.of(93), bandwidthFile.percentEligibleRelays()); + assertEquals(Optional.of(3862), + bandwidthFile.minimumNumberEligibleRelays()); + assertEquals(Optional.of("SN"), bandwidthFile.scannerCountry()); + assertArrayEquals(new String[] { "TH", "ZZ" }, + bandwidthFile.destinationsCountries().orElse(null)); + assertFalse(bandwidthFile.recentConsensusCount().isPresent()); + assertEquals(Optional.of(20), + bandwidthFile.recentPriorityListCount()); + assertEquals(Optional.of(6243), + bandwidthFile.recentPriorityRelayCount()); + assertEquals(Optional.of(6243), + bandwidthFile.recentMeasurementAttemptCount()); + assertEquals(Optional.of(732), + bandwidthFile.recentMeasurementFailureCount()); + assertEquals(Optional.of(969), + bandwidthFile.recentMeasurementsExcludedErrorCount()); + assertEquals(Optional.of(90), + bandwidthFile.recentMeasurementsExcludedNearCount()); + assertEquals(Optional.of(0), + bandwidthFile.recentMeasurementsExcludedOldCount()); + assertEquals(Optional.of(3946), + bandwidthFile.recentMeasurementsExcludedFewCount()); + assertEquals(Optional.of(Duration.ofSeconds(57273L)), + bandwidthFile.timeToReportHalfNetwork()); + } +} + diff --git a/src/test/java/org/torproject/descriptor/impl/TestDescriptorBuilder.java b/src/test/java/org/torproject/descriptor/impl/TestDescriptorBuilder.java new file mode 100644 index 0000000..a596c9d --- /dev/null +++ b/src/test/java/org/torproject/descriptor/impl/TestDescriptorBuilder.java @@ -0,0 +1,120 @@ +/* Copyright 2016--2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.impl; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Builds a test descriptor by concatenating the given lines with newlines and + * writing the output to a byte array. + */ +class TestDescriptorBuilder extends ArrayList<String> { + + /** + * Initializes a new test descriptor builder with the given lines. + */ + TestDescriptorBuilder(String ... lines) { + this.addAll(Arrays.asList(lines)); + } + + /** + * Appends the given line or lines. + */ + TestDescriptorBuilder appendLines(String ... lines) { + this.addAll(Arrays.asList(lines)); + return this; + } + + /** + * Removes the given line, or fails if that line cannot be found. + */ + TestDescriptorBuilder removeLine(String line) { + if (!this.remove(line)) { + fail("Line not contained: " + line); + } + return this; + } + + /** + * Removes all but the given line, or fails if that line cannot be found. + */ + TestDescriptorBuilder removeAllExcept(String line) { + assertTrue("Line not contained: " + line, this.contains(line)); + this.retainAll(Arrays.asList(line)); + return this; + } + + /** + * Finds the first line that starts with the given line start and inserts the + * given lines before it, or fails if no line can be found with that line + * start. + */ + TestDescriptorBuilder insertBeforeLineStartingWith(String lineStart, + String ... linesToInsert) { + for (int i = 0; i < this.size(); i++) { + if (this.get(i).startsWith(lineStart)) { + this.addAll(i, Arrays.asList(linesToInsert)); + return this; + } + } + fail("Line start not found: " + lineStart); + return this; + } + + /** + * Finds the first line that starts with the given line start and replaces + * that line and possibly subsequent lines, or fails if no line can be found + * with that line start or there are not enough lines left to replace. + */ + TestDescriptorBuilder replaceLineStartingWith(String lineStart, + String ... linesToReplace) { + for (int i = 0; i < this.size(); i++) { + if (this.get(i).startsWith(lineStart)) { + for (int j = 0; j < linesToReplace.length; j++) { + assertTrue("Not enough lines left to replace.", + this.size() > i + j); + this.set(i + j, linesToReplace[j]); + } + return this; + } + } + fail("Line start not found: " + lineStart); + return this; + } + + /** + * Finds the first line that starts with the given line start and truncates + * that line and possibly subsequent lines, or fails if no line can be found + * with that line start. + */ + TestDescriptorBuilder truncateAtLineStartingWith(String lineStart) { + for (int i = 0; i < this.size(); i++) { + if (this.get(i).startsWith(lineStart)) { + while (this.size() > i) { + this.remove(i); + } + return this; + } + } + fail("Line start not found: " + lineStart); + return this; + } + + /** + * Concatenates all descriptor lines with newlines and returns the raw + * descriptor bytes as byte array. + */ + byte[] build() { + StringBuilder sb = new StringBuilder(); + for (String line : this) { + sb.append(line).append('\n'); + } + return sb.toString().getBytes(); + } +} +