commit 81570c4dbc097089f367c104c7ef5a77bee29763 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Feb 26 21:31:08 2020 +0100

Parse recently added lines.

- Compute bandwidth file digests. - Parse bandwidth file header and bandwidth file digest in votes. - Parse bridge distribution requests in bridge server descriptors. - Parse authority fingerprint in bridge network statuses.

Implements #33206. --- CHANGELOG.md | 8 ++++- .../org/torproject/descriptor/BandwidthFile.java | 9 +++++ .../torproject/descriptor/BridgeNetworkStatus.java | 8 +++++ .../descriptor/RelayNetworkStatusVote.java | 20 +++++++++++ .../torproject/descriptor/ServerDescriptor.java | 8 +++++ .../descriptor/impl/BandwidthFileImpl.java | 6 ++++ .../descriptor/impl/BridgeNetworkStatusImpl.java | 26 ++++++++++++++ .../torproject/descriptor/impl/DescriptorImpl.java | 9 ++++- .../java/org/torproject/descriptor/impl/Key.java | 3 ++ .../impl/RelayNetworkStatusVoteImpl.java | 41 +++++++++++++++++++++- .../descriptor/impl/ServerDescriptorImpl.java | 20 ++++++++++- .../descriptor/impl/BridgeNetworkStatusTest.java | 31 ++++++++++++++++ .../descriptor/impl/ServerDescriptorImplTest.java | 30 ++++++++++++++++ 13 files changed, 215 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index d58e92b..068c0b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ -# Changes in version 2.1?.? - 2020-0?-?? +# Changes in version 2.11.0 - 2020-0?-?? + + * Medium changes + - Compute bandwidth file digests. + - Parse bandwidth file header and bandwidth file digest in votes. + - Parse bridge distribution requests in bridge server descriptors. + - Parse authority fingerprint in bridge network statuses.

* Minor changes - Avoid invoking overridable methods from constructors. diff --git a/src/main/java/org/torproject/descriptor/BandwidthFile.java b/src/main/java/org/torproject/descriptor/BandwidthFile.java index bbc4056..25d738e 100644 --- a/src/main/java/org/torproject/descriptor/BandwidthFile.java +++ b/src/main/java/org/torproject/descriptor/BandwidthFile.java @@ -18,6 +18,15 @@ import java.util.Optional; public interface BandwidthFile extends Descriptor {

/** + * Return the SHA-256 bandwidth file digest, encoded as 43 base64 characters + * without padding characters, that is used to reference this bandwidth file + * from a vote. + * + * @since 2.11.0 + */ + String digestSha256Base64(); + + /** * Time of the most recent generator bandwidth result. * * @since 2.6.0 diff --git a/src/main/java/org/torproject/descriptor/BridgeNetworkStatus.java b/src/main/java/org/torproject/descriptor/BridgeNetworkStatus.java index 9c78b77..6ae4751 100644 --- a/src/main/java/org/torproject/descriptor/BridgeNetworkStatus.java +++ b/src/main/java/org/torproject/descriptor/BridgeNetworkStatus.java @@ -117,6 +117,14 @@ public interface BridgeNetworkStatus extends Descriptor { int getIgnoringAdvertisedBws();

/** + * Return a SHA-1 digest of the bridge authority's identity key, encoded as 40 + * upper-case hexadecimal characters. + * + * @since 2.11.0 + */ + String getFingerprint(); + + /** * Return status entries for each contained bridge, with map keys being * SHA-1 digests of SHA-1 digest of the bridges' public identity keys, * encoded as 40 upper-case hexadecimal characters. diff --git a/src/main/java/org/torproject/descriptor/RelayNetworkStatusVote.java b/src/main/java/org/torproject/descriptor/RelayNetworkStatusVote.java index 5ed31b3..a447a7a 100644 --- a/src/main/java/org/torproject/descriptor/RelayNetworkStatusVote.java +++ b/src/main/java/org/torproject/descriptor/RelayNetworkStatusVote.java @@ -361,6 +361,26 @@ public interface RelayNetworkStatusVote extends Descriptor { String getSharedRandCurrentValue();

/** + * Return the headers from the bandwidth file used to generate this vote, or + * null if the authority producing this vote is not configured with a + * bandwidth file or does not include the headers of the configured bandwidth + * file in its vote. + * + * @since 2.11.0 + */ + SortedMap<String, String> getBandwidthFileHeaders(); + + /** + * Return the SHA256 digest of the bandwidth file, encoded as 43 base64 + * characters without padding characters, or null if the authority producing + * this vote is not configured with a bandwidth file or does not include the + * SHA256 digest of the configured bandwidth file in its vote. + * + * @since 2.11.0 + */ + String getBandwidthFileDigestSha256Base64(); + + /** * Return the version of the directory key certificate used by this * authority, which must be 3 or higher. * diff --git a/src/main/java/org/torproject/descriptor/ServerDescriptor.java b/src/main/java/org/torproject/descriptor/ServerDescriptor.java index 2e0ecbd..ed585b0 100644 --- a/src/main/java/org/torproject/descriptor/ServerDescriptor.java +++ b/src/main/java/org/torproject/descriptor/ServerDescriptor.java @@ -245,6 +245,14 @@ public interface ServerDescriptor extends Descriptor { String getContact();

/** + * Return the method how a bridge requests to be distributed by BridgeDB, or + * {@code null} if no such request is contained in the descriptor. + * + * @since 2.11.0 + */ + String getBridgeDistributionRequest(); + + /** * Return nicknames, $-prefixed identity fingerprints, or tuples of the * format {@code $fingerprint=nickname} or {@code $fingerprint~nickname} * of servers contained in this server's family, or null if the diff --git a/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java b/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java index f9198a4..94df839 100644 --- a/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java @@ -66,6 +66,7 @@ public class BandwidthFileImpl extends DescriptorImpl implements BandwidthFile { this.parseRelayLine(line); } } + this.calculateDigestSha256Base64(); }

private void parseTimestampLine(String line) throws DescriptorParseException { @@ -257,6 +258,11 @@ public class BandwidthFileImpl extends DescriptorImpl implements BandwidthFile { additionalKeyValues.isEmpty() ? null : additionalKeyValues)); }

+ @Override + public String digestSha256Base64() { + return this.getDigestSha256Base64(); + } + private LocalDateTime timestamp;

@Override diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java index b9241a8..68b954f 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java @@ -10,8 +10,10 @@ import java.io.File; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.EnumSet; import java.util.Map; import java.util.Scanner; +import java.util.Set; import java.util.SortedMap; import java.util.TimeZone;

@@ -24,6 +26,10 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl throws DescriptorParseException { super(rawDescriptorBytes, offsetAndLength, descriptorFile, false); this.splitAndParseParts(false); + Set<Key> atMostOnceKeys = EnumSet.of(Key.PUBLISHED, Key.FLAG_THRESHOLDS, + Key.FINGERPRINT); + this.checkAtMostOnceKeys(atMostOnceKeys); + this.clearParsedKeys(); this.setPublishedMillisFromFileName(fileName); }

@@ -85,6 +91,9 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl case FLAG_THRESHOLDS: this.parseFlagThresholdsLine(line, parts); break; + case FINGERPRINT: + this.parseFingerprintLine(line, parts); + break; default: if (this.unrecognizedLines == null) { this.unrecognizedLines = new ArrayList<>(); @@ -151,6 +160,16 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl } }

+ private void parseFingerprintLine(String line, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length < 2) { + throw new DescriptorParseException("Illegal line '" + line + + "' in bridge network status."); + } + this.fingerprint = ParseHelper.parseTwentyByteHexString(line, + partsNoOpt[1]); + } + protected void parseDirSource(int offset, int length) throws DescriptorParseException { throw new DescriptorParseException("No directory source expected in " @@ -238,5 +257,12 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl public int getIgnoringAdvertisedBws() { return this.ignoringAdvertisedBws; } + + private String fingerprint; + + @Override + public String getFingerprint() { + return this.fingerprint; + } }

diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java index 4b380bc..f963a53 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java @@ -408,7 +408,10 @@ public abstract class DescriptorImpl implements Descriptor { if (null == this.digestSha256Base64) { String ascii = new String(this.rawDescriptorBytes, this.offset, this.length, StandardCharsets.US_ASCII); - int start = ascii.indexOf(startToken); + int start = 0; + if (null != startToken) { + start = ascii.indexOf(startToken); + } int end = -1; if (null == endToken) { end = ascii.length(); @@ -431,6 +434,10 @@ public abstract class DescriptorImpl implements Descriptor { this.calculateDigestSha256Base64(startToken, null); }

+ protected void calculateDigestSha256Base64() throws DescriptorParseException { + this.calculateDigestSha256Base64(null, null); + } + public String getDigestSha256Base64() { return this.digestSha256Base64; } diff --git a/src/main/java/org/torproject/descriptor/impl/Key.java b/src/main/java/org/torproject/descriptor/impl/Key.java index 10839dd..a6202e4 100644 --- a/src/main/java/org/torproject/descriptor/impl/Key.java +++ b/src/main/java/org/torproject/descriptor/impl/Key.java @@ -17,10 +17,13 @@ public enum Key { ACCEPT("accept"), ALLOW_SINGLE_HOP_EXITS("allow-single-hop-exits"), BANDWIDTH("bandwidth"), + BANDWIDTH_FILE_DIGEST("bandwidth-file-digest"), + BANDWIDTH_FILE_HEADERS("bandwidth-file-headers"), BANDWIDTH_WEIGHTS("bandwidth-weights"), BRIDGEDB_METRICS_END("bridgedb-metrics-end"), BRIDGEDB_METRICS_VERSION("bridgedb-metrics-version"), BRIDGEDB_METRIC_COUNT("bridgedb-metric-count"), + BRIDGE_DISTRIBUTION_REQUEST("bridge-distribution-request"), BRIDGE_IPS("bridge-ips"), BRIDGE_IP_TRANSPORTS("bridge-ip-transports"), BRIDGE_IP_VERSIONS("bridge-ip-versions"), diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index 7e8d816..e3fab9e 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -41,7 +41,8 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl Key.REQUIRED_CLIENT_PROTOCOLS, Key.REQUIRED_RELAY_PROTOCOLS, Key.FLAG_THRESHOLDS, Key.PARAMS, Key.CONTACT, Key.SHARED_RAND_PARTICIPATE, Key.SHARED_RAND_PREVIOUS_VALUE, - Key.SHARED_RAND_CURRENT_VALUE, Key.LEGACY_KEY, Key.DIR_KEY_CROSSCERT, + Key.SHARED_RAND_CURRENT_VALUE, Key.BANDWIDTH_FILE_HEADERS, + Key.BANDWIDTH_FILE_DIGEST, Key.LEGACY_KEY, Key.DIR_KEY_CROSSCERT, Key.DIR_ADDRESS, Key.DIRECTORY_FOOTER); this.checkAtMostOnceKeys(atMostOnceKeys); this.checkAtLeastOnceKeys(EnumSet.of(Key.DIRECTORY_SIGNATURE)); @@ -133,6 +134,12 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl case SHARED_RAND_CURRENT_VALUE: this.parseSharedRandCurrentValueLine(line, parts); break; + case BANDWIDTH_FILE_HEADERS: + this.parseBandwidthFileHeaders(line, parts); + break; + case BANDWIDTH_FILE_DIGEST: + this.parseBandwidthFileDigest(line, parts); + break; case DIR_KEY_CERTIFICATE_VERSION: this.parseDirKeyCertificateVersionLine(line, parts); break; @@ -479,6 +486,24 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl this.sharedRandCurrentValue = parts[2]; }

+ protected void parseBandwidthFileHeaders(String line, String[] parts) + throws DescriptorParseException { + this.bandwidthFileHeaders + = ParseHelper.parseKeyValueStringPairs(line, parts, 1); + } + + protected void parseBandwidthFileDigest(String line, String[] parts) + throws DescriptorParseException { + for (int i = 1; i < parts.length; i++) { + String part = parts[i]; + if (part.startsWith("sha256=")) { + /* 7 == "sha256=".length() */ + ParseHelper.verifyThirtyTwoByteBase64String(line, part.substring(7)); + this.bandwidthFileDigestSha256Base64 = part.substring(7); + } + } + } + private void parseDirKeyCertificateVersionLine(String line, String[] parts) throws DescriptorParseException { if (parts.length != 2) { @@ -662,6 +687,20 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl return this.sharedRandCurrentValue; }

+ private SortedMap<String, String> bandwidthFileHeaders; + + @Override + public SortedMap<String, String> getBandwidthFileHeaders() { + return this.bandwidthFileHeaders; + } + + private String bandwidthFileDigestSha256Base64; + + @Override + public String getBandwidthFileDigestSha256Base64() { + return this.bandwidthFileDigestSha256Base64; + } + private int dirKeyCertificateVersion;

@Override diff --git a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java index aac4bcc..261c897 100644 --- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -31,7 +31,7 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl Key.IPV6_POLICY, Key.NTOR_ONION_KEY, Key.ONION_KEY_CROSSCERT, Key.NTOR_ONION_KEY_CROSSCERT, Key.TUNNELLED_DIR_SERVER, Key.ROUTER_SIG_ED25519, Key.ROUTER_SIGNATURE, Key.ROUTER_DIGEST_SHA256, - Key.ROUTER_DIGEST); + Key.ROUTER_DIGEST, Key.BRIDGE_DISTRIBUTION_REQUEST);

private static final Set<Key> exactlyOnce = EnumSet.of( Key.ROUTER, Key.BANDWIDTH, Key.PUBLISHED); @@ -113,6 +113,9 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl case CONTACT: this.parseContactLine(lineNoOpt); break; + case BRIDGE_DISTRIBUTION_REQUEST: + this.parseBridgeDistributionRequestLine(line, partsNoOpt); + break; case FAMILY: this.parseFamilyLine(line, partsNoOpt); break; @@ -394,6 +397,14 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl } }

+ private void parseBridgeDistributionRequestLine(String line, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length < 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.bridgeDistributionRequest = partsNoOpt[1]; + } + private void parseFamilyLine(String line, String[] partsNoOpt) throws DescriptorParseException { String[] familyEntries = new String[partsNoOpt.length - 1]; @@ -790,6 +801,13 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl return this.contact; }

+ private String bridgeDistributionRequest = null; + + @Override + public String getBridgeDistributionRequest() { + return this.bridgeDistributionRequest; + } + private String[] familyEntries;

@Override diff --git a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java index e586650..eea15ed 100644 --- a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java +++ b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java @@ -9,7 +9,9 @@ import static org.junit.Assert.assertTrue; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.DescriptorParseException;

+import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException;

import java.util.ArrayList; import java.util.List; @@ -18,6 +20,9 @@ import java.util.List; * already tested in the consensus/vote-parsing tests. */ public class BridgeNetworkStatusTest {

+ @Rule + public ExpectedException thrown = ExpectedException.none(); + /* Helper class to build a bridge network status based on default data * and modifications requested by test methods. */ private static class StatusBuilder { @@ -57,6 +62,16 @@ public class BridgeNetworkStatusTest { return sb.buildStatus(); }

+ private String fingerprintLine + = "fingerprint BA44A889E64B93FAA2B114E02C2A279A8555C533"; + + private static BridgeNetworkStatus createWithFingerprintLine(String line) + throws DescriptorParseException { + StatusBuilder sb = new StatusBuilder(); + sb.fingerprintLine = line; + return sb.buildStatus(); + } + private List<String> statusEntries = new ArrayList<>();

private String unrecognizedHeaderLine = null; @@ -106,6 +121,9 @@ public class BridgeNetworkStatusTest { if (this.flagThresholdsLine != null) { sb.append(this.flagThresholdsLine).append("\n"); } + if (this.fingerprintLine != null) { + sb.append(this.fingerprintLine).append("\n"); + } if (this.unrecognizedHeaderLine != null) { sb.append(this.unrecognizedHeaderLine).append("\n"); } @@ -135,6 +153,8 @@ public class BridgeNetworkStatusTest { assertEquals(339000L, status.getGuardBandwidthExcludingExits()); assertEquals(1, status.getEnoughMtbfInfo()); assertEquals(0, status.getIgnoringAdvertisedBws()); + assertEquals("BA44A889E64B93FAA2B114E02C2A279A8555C533", + status.getFingerprint()); assertEquals(264, status.getStatusEntries().get( "001934C20E23E812C27592A5795B6636B7F32462").getBandwidth()); assertTrue(status.getUnrecognizedLines().isEmpty()); @@ -161,5 +181,16 @@ public class BridgeNetworkStatusTest { assertEquals(-1, status.getEnoughMtbfInfo()); assertEquals(-1, status.getIgnoringAdvertisedBws()); } + + @Test + public void testBridgeDistributionRequestGivenTwice() + throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage("Keyword 'fingerprint' is contained 2 times, but " + + "must be contained at most once."); + StatusBuilder.createWithFingerprintLine( + "fingerprint BA44A889E64B93FAA2B114E02C2A279A8555C533\n" + + "fingerprint BA44A889E64B93FAA2B114E02C2A279A8555C533"); + } }

diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 97456d6..f5fa52d 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -338,6 +338,15 @@ public class ServerDescriptorImplTest { return db.buildDescriptor(); }

+ private String bridgeDistributionRequestLine = null; + + private static ServerDescriptor createWithBridgeDistributionRequestLine( + String line) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.bridgeDistributionRequestLine = line; + return db.buildDescriptor(); + } + private byte[] buildDescriptorBytes() { StringBuilder sb = new StringBuilder(); if (this.routerLine != null) { @@ -385,6 +394,9 @@ public class ServerDescriptorImplTest { if (this.contactLine != null) { sb.append(this.contactLine).append("\n"); } + if (this.bridgeDistributionRequestLine != null) { + sb.append(this.bridgeDistributionRequestLine).append("\n"); + } if (this.familyLine != null) { sb.append(this.familyLine).append("\n"); } @@ -1973,5 +1985,23 @@ public class ServerDescriptorImplTest { assertNull(descriptor.getDigestSha1Hex()); assertNull(descriptor.getDigestSha256Base64()); } + + @Test + public void testBridgeDistributionRequestMoat() + throws DescriptorParseException { + ServerDescriptor descriptor = + DescriptorBuilder.createWithBridgeDistributionRequestLine( + "bridge-distribution-request moat"); + assertEquals("moat", descriptor.getBridgeDistributionRequest()); + } + + @Test + public void testBridgeDistributionRequestEmptySpace() + throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage("Illegal line 'bridge-distribution-request '."); + DescriptorBuilder.createWithBridgeDistributionRequestLine( + "bridge-distribution-request "); + } }