- tor-commits - lists.torproject.org

[metrics-lib/master] Parse Ed25519 and SHA-256 elements in descriptors.
by karsten＠torproject.org 21 Dec '15

21 Dec '15

commit 39a9c496effde2bd083df0095006ca299f9af2d5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 16 10:41:28 2015 +0100 Parse Ed25519 and SHA-256 elements in descriptors. More precisely, - support Ed25519 certificates and Ed25519 master keys as well as SHA-256 digests and Ed25519 signatures thereof in server descriptors and extra-info descriptors, - parse RSA-1024 signatures of SHA-1 digests of extra-info descriptors, - parse Ed25519 master keys in votes, and - parse Ed25519 and RSA-1024 identity digests in microdescriptors. This patch is based on metrics-db's bridge descriptor sanitizer. --- CHANGELOG.md | 5 + .../torproject/descriptor/ExtraInfoDescriptor.java | 26 +++ src/org/torproject/descriptor/Microdescriptor.java | 10 ++ .../torproject/descriptor/NetworkStatusEntry.java | 5 + .../torproject/descriptor/ServerDescriptor.java | 27 +++ .../descriptor/impl/ExtraInfoDescriptorImpl.java | 157 ++++++++++++++++- .../descriptor/impl/MicrodescriptorImpl.java | 29 +++- .../descriptor/impl/NetworkStatusEntryImpl.java | 19 +++ .../torproject/descriptor/impl/ParseHelper.java | 73 ++++++++ .../descriptor/impl/ServerDescriptorImpl.java | 137 ++++++++++++++- .../impl/ExtraInfoDescriptorImplTest.java | 176 +++++++++++++++++++- .../descriptor/impl/MicrodescriptorImplTest.java | 82 +++++++++ .../impl/RelayNetworkStatusVoteImplTest.java | 54 +++++- .../descriptor/impl/ServerDescriptorImplTest.java | 117 +++++++++++++ 14 files changed, 902 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5456795..1f2bc0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ existing types, ServerDescriptor and ExtraInfoDescriptor, are still usable and will not be deprecated, because applications may not care whether a relay or a bridge published a descriptor. + - Support Ed25519 certificates, Ed25519 master keys, SHA-256 + digests, and Ed25519 signatures thereof in server descriptors and + extra-info descriptors, and support Ed25519 master keys in votes. + - Include RSA-1024 signatures of SHA-1 digests of extra-info + descriptors, which were parsed and discarded before. # Changes in version 1.0.0 - 2015-12-05 diff --git a/src/org/torproject/descriptor/ExtraInfoDescriptor.java b/src/org/torproject/descriptor/ExtraInfoDescriptor.java index 380be00..1b978a4 100644 --- a/src/org/torproject/descriptor/ExtraInfoDescriptor.java +++ b/src/org/torproject/descriptor/ExtraInfoDescriptor.java @@ -12,6 +12,10 @@ public interface ExtraInfoDescriptor extends Descriptor { * extra-info descriptor in a server descriptor. */ public String getExtraInfoDigest(); + /* Return the base64-encoded SHA-256 descriptor digest that may be used + * to reference this extra-info descriptor in a server descriptor. */ + public String getExtraInfoDigestSha256(); + /* Return the relay's nickname. */ public String getNickname(); @@ -260,5 +264,27 @@ public interface ExtraInfoDescriptor extends Descriptor { /* Return the (possibly empty) list of transports supported by this * bridge. */ public List<String> getTransports(); + + /* Return the signature of the PKCS1-padded extra-info descriptor + * digest, or null if the descriptor doesn't contain a signature (which + * is the case in sanitized bridge descriptors). */ + public String getRouterSignature(); + + /* Return the base64-encoded Ed25519 certificate, or null if the + * descriptor doesn't contain one. */ + public String getIdentityEd25519(); + + /* Return the base64-encoded Ed25519 master key, which may either be + * parsed from the optional "master-key-ed25519" line or derived from + * the (likewise optional) Ed25519 certificate following the + * "identity-ed25519" line, or null if the descriptor contains neither + * Ed25519 master key nor Ed25519 certificate. */ + public String getMasterKeyEd25519(); + + /* Return the base64-encoded Ed25519 signature of a SHA-256 digest of + * the entire descriptor, from the first character up to and including + * the first space after the "router-sig-ed25519" string, prefixed with + * the string "Tor router descriptor signature v1". */ + public String getRouterSignatureEd25519(); } diff --git a/src/org/torproject/descriptor/Microdescriptor.java b/src/org/torproject/descriptor/Microdescriptor.java index 7715d35..3cebeb6 100644 --- a/src/org/torproject/descriptor/Microdescriptor.java +++ b/src/org/torproject/descriptor/Microdescriptor.java @@ -43,5 +43,15 @@ public interface Microdescriptor extends Descriptor { /* Return the port list of the IPv6 port summary or null if the * microdescriptor didn't contain an IPv6 port summary line. */ public String getIpv6PortList(); + + /* Return the optional, base64-encoded RSA-1024 identity that is only + * included to prevent collisions between microdescriptors, or null if + * no such identity is included. */ + public String getRsa1024Identity(); + + /* Return the optional, base64-encoded Ed25519 identity that is only + * included to prevent collisions between microdescriptors, or null if + * no such identity is included. */ + public String getEd25519Identity(); } diff --git a/src/org/torproject/descriptor/NetworkStatusEntry.java b/src/org/torproject/descriptor/NetworkStatusEntry.java index fb9163e..584f07b 100644 --- a/src/org/torproject/descriptor/NetworkStatusEntry.java +++ b/src/org/torproject/descriptor/NetworkStatusEntry.java @@ -75,5 +75,10 @@ public interface NetworkStatusEntry { /* Return the port list of the port summary or null if the status entry * didn't contain a port summary line. */ public String getPortList(); + + /* Return the relay's base64-encoded Ed25519 master key, "none" if the + * relay doesn't have an Ed25519 identity, or null if the status entry + * didn't contain this information. Only included in votes. */ + public String getMasterKeyEd25519(); } diff --git a/src/org/torproject/descriptor/ServerDescriptor.java b/src/org/torproject/descriptor/ServerDescriptor.java index 598b9b5..3266e9d 100644 --- a/src/org/torproject/descriptor/ServerDescriptor.java +++ b/src/org/torproject/descriptor/ServerDescriptor.java @@ -11,6 +11,10 @@ public interface ServerDescriptor extends Descriptor { * descriptor in a network status. */ public String getServerDescriptorDigest(); + /* Return the base64-encoded SHA-256 descriptor digest that may be used + * to reference this server descriptor in a network status. */ + public String getServerDescriptorDigestSha256(); + /* Return the relay's nickname. */ public String getNickname(); @@ -119,6 +123,12 @@ public interface ServerDescriptor extends Descriptor { * the relay did not upload a corresponding extra-info descriptor. */ public String getExtraInfoDigest(); + /* Return the base64-encoded SHA-256 digest of the extra-info descriptor + * referenced from this server descriptor, or null if the relay either + * did not upload a corresponding extra-info descriptor or did not refer + * to it using a SHA-256 digest. */ + public String getExtraInfoDigestSha256(); + /* Return the hidden service descriptor version(s) that this relay * stores and serves, or null if it doesn't store and serve any hidden * service descriptors. */ @@ -147,5 +157,22 @@ public interface ServerDescriptor extends Descriptor { /* Return the ntor onion key base64 string with padding omitted, or null * if the server descriptors didn't contain an ntor onion key line. */ public String getNtorOnionKey(); + + /* Return the base64-encoded Ed25519 certificate, or null if the + * descriptor doesn't contain one. */ + public String getIdentityEd25519(); + + /* Return the base64-encoded Ed25519 master key, which may either be + * parsed from the optional "master-key-ed25519" line or derived from + * the (likewise optional) Ed25519 certificate following the + * "identity-ed25519" line, or null if the descriptor contains neither + * Ed25519 master key nor Ed25519 certificate. */ + public String getMasterKeyEd25519(); + + /* Return the base64-encoded Ed25519 signature of a SHA-256 digest of + * the entire descriptor, from the first character up to and including + * the first space after the "router-sig-ed25519" string, prefixed with + * the string "Tor router descriptor signature v1". */ + public String getRouterSignatureEd25519(); } diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 61b8d13..4abace6 100644 --- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -31,6 +31,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl super(descriptorBytes, failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.calculateDigest(); + this.calculateDigestSha256(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "extra-info,published").split(","))); this.checkExactlyOnceKeywords(exactlyOnceKeywords); @@ -52,8 +53,9 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl Set<String> bridgeStatsKeywords = new HashSet<String>(Arrays.asList( "bridge-stats-end,bridge-stats-ips".split(","))); Set<String> atMostOnceKeywords = new HashSet<String>(Arrays.asList(( - "read-history,write-history,dirreq-read-history," - + "dirreq-write-history,geoip-db-digest,router-signature," + "identity-ed25519,master-key-ed25519,read-history,write-history," + + "dirreq-read-history,dirreq-write-history,geoip-db-digest," + + "router-sig-ed25519,router-signature,router-digest-sha256," + "router-digest").split(","))); atMostOnceKeywords.addAll(dirreqStatsKeywords); atMostOnceKeywords.addAll(entryStatsKeywords); @@ -75,7 +77,8 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseDescriptorBytes() throws DescriptorParseException { Scanner s = new Scanner(new String(this.rawDescriptorBytes)). useDelimiter("\n"); - boolean skipCrypto = false; + String nextCrypto = null; + List<String> cryptoLines = null; while (s.hasNext()) { String line = s.next(); String lineNoOpt = line.startsWith("opt ") ? @@ -162,15 +165,49 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl this.parseBridgeIpTransportsLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("transport")) { this.parseTransportLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("identity-ed25519")) { + this.parseIdentityEd25519Line(line, lineNoOpt, partsNoOpt); + nextCrypto = "identity-ed25519"; + } else if (keyword.equals("master-key-ed25519")) { + this.parseMasterKeyEd25519Line(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("router-sig-ed25519")) { + this.parseRouterSigEd25519Line(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("router-signature")) { this.parseRouterSignatureLine(line, lineNoOpt, partsNoOpt); + nextCrypto = "router-signature"; } else if (keyword.equals("router-digest")) { this.parseRouterDigestLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("router-digest-sha256")) { + this.parseRouterDigestSha256Line(line, lineNoOpt, partsNoOpt); } else if (line.startsWith("-----BEGIN")) { - skipCrypto = true; + cryptoLines = new ArrayList<String>(); + cryptoLines.add(line); } else if (line.startsWith("-----END")) { - skipCrypto = false; - } else if (!skipCrypto) { + cryptoLines.add(line); + StringBuilder sb = new StringBuilder(); + for (String cryptoLine : cryptoLines) { + sb.append("\n" + cryptoLine); + } + String cryptoString = sb.toString().substring(1); + if ("router-signature".equals(nextCrypto)) { + this.routerSignature = cryptoString; + } else if ("identity-ed25519".equals(nextCrypto)) { + this.identityEd25519 = cryptoString; + this.parseIdentityEd25519CryptoBlock(cryptoString); + } else if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized crypto " + + "block '" + cryptoString + "' in extra-info descriptor."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.addAll(cryptoLines); + } + cryptoLines = null; + nextCrypto = null; + } else if (cryptoLines != null) { + cryptoLines.add(line); + } else { ParseHelper.parseKeyword(line, partsNoOpt[0]); if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" @@ -610,7 +647,6 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl if (!lineNoOpt.equals("router-signature")) { throw new DescriptorParseException("Illegal line '" + line + "'."); } - /* Not parsing crypto parts (yet). */ } private void parseRouterDigestLine(String line, String lineNoOpt, @@ -622,6 +658,57 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl partsNoOpt[1]); } + private void parseIdentityEd25519Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 1) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + } + + private void parseIdentityEd25519CryptoBlock(String cryptoString) + throws DescriptorParseException { + String masterKeyEd25519FromIdentityEd25519 = + ParseHelper.parseMasterKeyEd25519FromIdentityEd25519CryptoBlock( + cryptoString); + if (this.masterKeyEd25519 != null && !this.masterKeyEd25519.equals( + masterKeyEd25519FromIdentityEd25519)) { + throw new DescriptorParseException("Mismatch between " + + "identity-ed25519 and master-key-ed25519."); + } + this.masterKeyEd25519 = masterKeyEd25519FromIdentityEd25519; + } + + private void parseMasterKeyEd25519Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + String masterKeyEd25519FromMasterKeyEd25519Line = partsNoOpt[1]; + if (this.masterKeyEd25519 != null && !masterKeyEd25519.equals( + masterKeyEd25519FromMasterKeyEd25519Line)) { + throw new DescriptorParseException("Mismatch between " + + "identity-ed25519 and master-key-ed25519."); + } + this.masterKeyEd25519 = masterKeyEd25519FromMasterKeyEd25519Line; + } + + private void parseRouterSigEd25519Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.routerSignatureEd25519 = partsNoOpt[1]; + } + + private void parseRouterDigestSha256Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[1]); + this.extraInfoDigestSha256 = partsNoOpt[1]; + } + private void calculateDigest() throws DescriptorParseException { if (this.extraInfoDigest != null) { /* We already learned the descriptor digest of this bridge @@ -653,11 +740,47 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl } } + private void calculateDigestSha256() throws DescriptorParseException { + if (this.extraInfoDigestSha256 != null) { + /* We already learned the descriptor digest of this bridge + * descriptor from a "router-digest-sha256" line. */ + return; + } + try { + String ascii = new String(this.getRawDescriptorBytes(), "US-ASCII"); + String startToken = "extra-info "; + String sigToken = "\n-----END SIGNATURE-----\n"; + int start = ascii.indexOf(startToken); + int sig = ascii.indexOf(sigToken) + sigToken.length(); + if (start >= 0 && sig >= 0 && sig > start) { + byte[] forDigest = new byte[sig - start]; + System.arraycopy(this.getRawDescriptorBytes(), start, forDigest, + 0, sig - start); + this.extraInfoDigestSha256 = DatatypeConverter.printBase64Binary( + MessageDigest.getInstance("SHA-256").digest(forDigest)). + replaceAll("=", ""); + } + } catch (UnsupportedEncodingException e) { + /* Handle below. */ + } catch (NoSuchAlgorithmException e) { + /* Handle below. */ + } + if (this.extraInfoDigestSha256 == null) { + throw new DescriptorParseException("Could not calculate extra-info " + + "descriptor SHA-256 digest."); + } + } + private String extraInfoDigest; public String getExtraInfoDigest() { return this.extraInfoDigest; } + private String extraInfoDigestSha256; + public String getExtraInfoDigestSha256() { + return this.extraInfoDigestSha256; + } + private String nickname; public String getNickname() { return this.nickname; @@ -933,5 +1056,25 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl public List<String> getTransports() { return new ArrayList<String>(this.transports); } + + private String routerSignature; + public String getRouterSignature() { + return this.routerSignature; + } + + private String identityEd25519; + public String getIdentityEd25519() { + return this.identityEd25519; + } + + private String masterKeyEd25519; + public String getMasterKeyEd25519() { + return this.masterKeyEd25519; + } + + private String routerSignatureEd25519; + public String getRouterSignatureEd25519() { + return this.routerSignatureEd25519; + } } diff --git a/src/org/torproject/descriptor/impl/MicrodescriptorImpl.java b/src/org/torproject/descriptor/impl/MicrodescriptorImpl.java index ffa5a0f..1987659 100644 --- a/src/org/torproject/descriptor/impl/MicrodescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/MicrodescriptorImpl.java @@ -48,7 +48,7 @@ public class MicrodescriptorImpl extends DescriptorImpl "onion-key".split(","))); this.checkExactlyOnceKeywords(exactlyOnceKeywords); Set<String> atMostOnceKeywords = new HashSet<String>(Arrays.asList(( - "ntor-onion-key,family,p,p6").split(","))); + "ntor-onion-key,family,p,p6,id").split(","))); this.checkAtMostOnceKeywords(atMostOnceKeywords); this.checkFirstKeyword("onion-key"); this.clearParsedKeywords(); @@ -80,6 +80,8 @@ public class MicrodescriptorImpl extends DescriptorImpl this.parsePLine(line, parts); } else if (keyword.equals("p6")) { this.parseP6Line(line, parts); + } else if (keyword.equals("id")) { + this.parseIdLine(line, parts); } else if (line.startsWith("-----BEGIN")) { crypto = new StringBuilder(); crypto.append(line + "\n"); @@ -196,6 +198,21 @@ public class MicrodescriptorImpl extends DescriptorImpl } } + private void parseIdLine(String line, String[] parts) + throws DescriptorParseException { + if (parts.length != 3) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } else if ("ed25519".equals(parts[1])) { + ParseHelper.parseThirtyTwoByteBase64String(line, parts[2]); + this.ed25519Identity = parts[2]; + } else if ("rsa1024".equals(parts[1])) { + ParseHelper.parseTwentyByteBase64String(line, parts[2]); + this.rsa1024Identity = parts[2]; + } else { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + } + private void calculateDigest() throws DescriptorParseException { try { String ascii = new String(this.getRawDescriptorBytes(), "US-ASCII"); @@ -265,5 +282,15 @@ public class MicrodescriptorImpl extends DescriptorImpl public String getIpv6PortList() { return this.ipv6PortList; } + + private String rsa1024Identity; + public String getRsa1024Identity() { + return this.rsa1024Identity; + } + + private String ed25519Identity; + public String getEd25519Identity() { + return this.ed25519Identity; + } } diff --git a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index dac6052..94575c6 100644 --- a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -91,6 +91,8 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { this.parsePLine(line, parts); } else if (keyword.equals("m")) { this.parseMLine(line, parts); + } else if (keyword.equals("id")) { + this.parseIdLine(line, parts); } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line + "' in status entry."); @@ -237,6 +239,18 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { } } + private void parseIdLine(String line, String[] parts) + throws DescriptorParseException { + if (parts.length != 3 || !"ed25519".equals(parts[1])) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } else if ("none".equals(parts[2])) { + this.masterKeyEd25519 = "none"; + } else { + ParseHelper.parseThirtyTwoByteBase64String(line, parts[2]); + this.masterKeyEd25519 = parts[2]; + } + } + private void clearAtMostOnceKeywords() { this.atMostOnceKeywords = null; } @@ -328,5 +342,10 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { public String getPortList() { return this.portList; } + + private String masterKeyEd25519; + public String getMasterKeyEd25519() { + return this.masterKeyEd25519; + } } diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java index 4e91e92..a354831 100644 --- a/src/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/org/torproject/descriptor/impl/ParseHelper.java @@ -434,5 +434,78 @@ public class ParseHelper { } return result; } + + public static String + parseMasterKeyEd25519FromIdentityEd25519CryptoBlock( + String identityEd25519CryptoBlock) throws DescriptorParseException { + String identityEd25519CryptoBlockNoNewlines = + identityEd25519CryptoBlock.replaceAll("\n", ""); + String beginEd25519CertLine = "-----BEGIN ED25519 CERT-----", + endEd25519CertLine = "-----END ED25519 CERT-----"; + if (!identityEd25519CryptoBlockNoNewlines.startsWith( + beginEd25519CertLine)) { + throw new DescriptorParseException("Illegal start of " + + "identity-ed25519 crypto block '" + identityEd25519CryptoBlock + + "'."); + } + if (!identityEd25519CryptoBlockNoNewlines.endsWith( + endEd25519CertLine)) { + throw new DescriptorParseException("Illegal end of " + + "identity-ed25519 crypto block '" + identityEd25519CryptoBlock + + "'."); + } + String identityEd25519Base64 = identityEd25519CryptoBlockNoNewlines. + substring(beginEd25519CertLine.length(), + identityEd25519CryptoBlock.length() + - endEd25519CertLine.length()).replaceAll("=", ""); + byte[] identityEd25519 = DatatypeConverter.parseBase64Binary( + identityEd25519Base64); + if (identityEd25519.length < 40) { + throw new DescriptorParseException("Invalid length of " + + "identity-ed25519 (in bytes): " + identityEd25519.length); + } else if (identityEd25519[0] != 0x01) { + throw new DescriptorParseException("Unknown version in " + + "identity-ed25519: " + identityEd25519[0]); + } else if (identityEd25519[1] != 0x04) { + throw new DescriptorParseException("Unknown cert type in " + + "identity-ed25519: " + identityEd25519[1]); + } else if (identityEd25519[6] != 0x01) { + throw new DescriptorParseException("Unknown certified key type in " + + "identity-ed25519: " + identityEd25519[1]); + } else if (identityEd25519[39] == 0x00) { + throw new DescriptorParseException("No extensions in " + + "identity-ed25519 (which would contain the encoded " + + "master-key-ed25519): " + identityEd25519[39]); + } else { + int extensionStart = 40; + for (int i = 0; i < (int) identityEd25519[39]; i++) { + if (identityEd25519.length < extensionStart + 4) { + throw new DescriptorParseException("Invalid extension with id " + + i + " in identity-ed25519."); + } + int extensionLength = identityEd25519[extensionStart]; + extensionLength <<= 8; + extensionLength += identityEd25519[extensionStart + 1]; + int extensionType = identityEd25519[extensionStart + 2]; + if (extensionLength == 32 && extensionType == 4) { + if (identityEd25519.length < extensionStart + 4 + 32) { + throw new DescriptorParseException("Invalid extension with " + + "id " + i + " in identity-ed25519."); + } + byte[] masterKeyEd25519 = new byte[32]; + System.arraycopy(identityEd25519, extensionStart + 4, + masterKeyEd25519, 0, masterKeyEd25519.length); + String masterKeyEd25519Base64 = DatatypeConverter. + printBase64Binary(masterKeyEd25519).replaceAll("=", ""); + String masterKeyEd25519Base64NoTrailingEqualSigns = + masterKeyEd25519Base64.replaceAll("=", ""); + return masterKeyEd25519Base64NoTrailingEqualSigns; + } + extensionStart += 4 + extensionLength; + } + } + throw new DescriptorParseException("Unable to locate " + + "master-key-ed25519 in identity-ed25519."); + } } diff --git a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java index 426f4d0..3dd6c40 100644 --- a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -28,15 +28,18 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl super(descriptorBytes, failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.calculateDigest(); + this.calculateDigestSha256(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList( "router,bandwidth,published".split(","))); this.checkExactlyOnceKeywords(exactlyOnceKeywords); Set<String> atMostOnceKeywords = new HashSet<String>(Arrays.asList(( - "platform,fingerprint,hibernating,uptime,contact,family," - + "read-history,write-history,eventdns,caches-extra-info," - + "extra-info-digest,hidden-service-dir,protocols," - + "allow-single-hop-exits,onion-key,signing-key,ipv6-policy," - + "ntor-onion-key,router-signature,router-digest").split(","))); + "identity-ed25519,master-key-ed25519,platform,fingerprint," + + "hibernating,uptime,contact,family,read-history,write-history," + + "eventdns,caches-extra-info,extra-info-digest," + + "hidden-service-dir,protocols,allow-single-hop-exits,onion-key," + + "signing-key,ipv6-policy,ntor-onion-key,router-sig-ed25519," + + "router-signature,router-digest-sha256,router-digest"). + split(","))); this.checkAtMostOnceKeywords(atMostOnceKeywords); this.checkFirstKeyword("router"); if (this.getKeywordCount("accept") == 0 && @@ -115,10 +118,19 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl this.parseDircacheportLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("router-digest")) { this.parseRouterDigestLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("router-digest-sha256")) { + this.parseRouterDigestSha256Line(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("ipv6-policy")) { this.parseIpv6PolicyLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("ntor-onion-key")) { this.parseNtorOnionKeyLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("identity-ed25519")) { + this.parseIdentityEd25519Line(line, lineNoOpt, partsNoOpt); + nextCrypto = "identity-ed25519"; + } else if (keyword.equals("master-key-ed25519")) { + this.parseMasterKeyEd25519Line(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("router-sig-ed25519")) { + this.parseRouterSigEd25519Line(line, lineNoOpt, partsNoOpt); } else if (line.startsWith("-----BEGIN")) { cryptoLines = new ArrayList<String>(); cryptoLines.add(line); @@ -135,6 +147,9 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl this.signingKey = cryptoString; } else if ("router-signature".equals(nextCrypto)) { this.routerSignature = cryptoString; + } else if ("identity-ed25519".equals(nextCrypto)) { + this.identityEd25519 = cryptoString; + this.parseIdentityEd25519CryptoBlock(cryptoString); } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized crypto " + "block '" + cryptoString + "' in server descriptor."); @@ -392,6 +407,10 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl } this.extraInfoDigest = ParseHelper.parseTwentyByteHexString(line, partsNoOpt[1]); + if (partsNoOpt.length >= 3) { + ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[2]); + this.extraInfoDigestSha256 = partsNoOpt[2]; + } } private void parseHiddenServiceDirLine(String line, String lineNoOpt, @@ -508,6 +527,57 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl this.ntorOnionKey = partsNoOpt[1].replaceAll("=", ""); } + private void parseIdentityEd25519Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 1) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + } + + private void parseIdentityEd25519CryptoBlock(String cryptoString) + throws DescriptorParseException { + String masterKeyEd25519FromIdentityEd25519 = + ParseHelper.parseMasterKeyEd25519FromIdentityEd25519CryptoBlock( + cryptoString); + if (this.masterKeyEd25519 != null && !this.masterKeyEd25519.equals( + masterKeyEd25519FromIdentityEd25519)) { + throw new DescriptorParseException("Mismatch between " + + "identity-ed25519 and master-key-ed25519."); + } + this.masterKeyEd25519 = masterKeyEd25519FromIdentityEd25519; + } + + private void parseMasterKeyEd25519Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + String masterKeyEd25519FromMasterKeyEd25519Line = partsNoOpt[1]; + if (this.masterKeyEd25519 != null && !masterKeyEd25519.equals( + masterKeyEd25519FromMasterKeyEd25519Line)) { + throw new DescriptorParseException("Mismatch between " + + "identity-ed25519 and master-key-ed25519."); + } + this.masterKeyEd25519 = masterKeyEd25519FromMasterKeyEd25519Line; + } + + private void parseRouterSigEd25519Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.routerSignatureEd25519 = partsNoOpt[1]; + } + + private void parseRouterDigestSha256Line(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[1]); + this.serverDescriptorDigestSha256 = partsNoOpt[1]; + } + private void calculateDigest() throws DescriptorParseException { if (this.serverDescriptorDigest != null) { /* We already learned the descriptor digest of this bridge @@ -539,11 +609,48 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl } } + private void calculateDigestSha256() throws DescriptorParseException { + if (this.serverDescriptorDigestSha256 != null) { + /* We already learned the descriptor digest of this bridge + * descriptor from a "router-digest-sha256" line. */ + return; + } + try { + String ascii = new String(this.getRawDescriptorBytes(), "US-ASCII"); + String startToken = "router "; + String sigToken = "\n-----END SIGNATURE-----\n"; + int start = ascii.indexOf(startToken); + int sig = ascii.indexOf(sigToken) + sigToken.length(); + if (start >= 0 && sig >= 0 && sig > start) { + byte[] forDigest = new byte[sig - start]; + System.arraycopy(this.getRawDescriptorBytes(), start, forDigest, + 0, sig - start); + this.serverDescriptorDigestSha256 = + DatatypeConverter.printBase64Binary( + MessageDigest.getInstance("SHA-256").digest(forDigest)). + replaceAll("=", ""); + } + } catch (UnsupportedEncodingException e) { + /* Handle below. */ + } catch (NoSuchAlgorithmException e) { + /* Handle below. */ + } + if (this.serverDescriptorDigestSha256 == null) { + throw new DescriptorParseException("Could not calculate server " + + "descriptor SHA-256 digest."); + } + } + private String serverDescriptorDigest; public String getServerDescriptorDigest() { return this.serverDescriptorDigest; } + private String serverDescriptorDigestSha256; + public String getServerDescriptorDigestSha256() { + return this.serverDescriptorDigestSha256; + } + private String nickname; public String getNickname() { return this.nickname; @@ -670,6 +777,11 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl return this.extraInfoDigest; } + private String extraInfoDigestSha256; + public String getExtraInfoDigestSha256() { + return this.extraInfoDigestSha256; + } + private Integer[] hiddenServiceDirVersions; public List<Integer> getHiddenServiceDirVersions() { return this.hiddenServiceDirVersions == null ? null : @@ -707,5 +819,20 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl public String getNtorOnionKey() { return this.ntorOnionKey; } + + private String identityEd25519; + public String getIdentityEd25519() { + return this.identityEd25519; + } + + private String masterKeyEd25519; + public String getMasterKeyEd25519() { + return this.masterKeyEd25519; + } + + private String routerSignatureEd25519; + public String getRouterSignatureEd25519() { + return this.routerSignatureEd25519; + } } diff --git a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index 4483af2..d70ac39 100644 --- a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.descriptor.impl; -import org.torproject.descriptor.DescriptorParseException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -16,7 +15,10 @@ import java.util.Map; import java.util.SortedMap; import org.junit.Test; +import org.torproject.descriptor.BridgeExtraInfoDescriptor; +import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.descriptor.RelayExtraInfoDescriptor; /* Test parsing of extra-info descriptors. */ public class ExtraInfoDescriptorImplTest { @@ -166,11 +168,28 @@ public class ExtraInfoDescriptorImplTest { db.routerSignatureLines = line; return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true); } + private String identityEd25519Lines = null, + masterKeyEd25519Line = null, routerSigEd25519Line = null; + private static ExtraInfoDescriptor createWithEd25519Lines( + String identityEd25519Lines, String masterKeyEd25519Line, + String routerSigEd25519Line) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.identityEd25519Lines = identityEd25519Lines; + db.masterKeyEd25519Line = masterKeyEd25519Line; + db.routerSigEd25519Line = routerSigEd25519Line; + return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true); + } private byte[] buildDescriptor() { StringBuilder sb = new StringBuilder(); if (this.extraInfoLine != null) { sb.append(this.extraInfoLine + "\n"); } + if (this.identityEd25519Lines != null) { + sb.append(this.identityEd25519Lines + "\n"); + } + if (this.masterKeyEd25519Line != null) { + sb.append(this.masterKeyEd25519Line + "\n"); + } if (this.publishedLine != null) { sb.append(this.publishedLine + "\n"); } @@ -230,6 +249,9 @@ public class ExtraInfoDescriptorImplTest { return null; } } + if (this.routerSigEd25519Line != null) { + sb.append(this.routerSigEd25519Line + "\n"); + } if (this.routerSignatureLines != null) { sb.append(this.routerSignatureLines + "\n"); } @@ -1430,5 +1452,157 @@ public class ExtraInfoDescriptorImplTest { unrecognizedLines.add(unrecognizedLine); assertEquals(unrecognizedLines, descriptor.getUnrecognizedLines()); } + + private static final String IDENTITY_ED25519_LINES = + "identity-ed25519\n" + + "-----BEGIN ED25519 CERT-----\n" + + "AQQABiX1AVGv5BuzJroQXbOh6vv1nbwc5rh2S13PyRFuLhTiifK4AQAgBACBCMwr" + + "\n4qgIlFDIzoC9ieJOtSkwrK+yXJPKlP8ojvgkx8cGKvhokOwA1eYDombzfwHcJ1" + + "EV\nbhEn/6g8i7wzO3LoqefIUrSAeEExOAOmm5mNmUIzL8EtnT6JHCr/sqUTUgA=" + + "\n" + + "-----END ED25519 CERT-----"; + + private static final String MASTER_KEY_ED25519_LINE = + "master-key-ed25519 gQjMK+KoCJRQyM6AvYniTrUpMKyvslyTypT/KI74JMc"; + + private static final String ROUTER_SIG_ED25519_LINE = + "router-sig-ed25519 y7WF9T2GFwkSDPZEhB55HgquIFOl5uXUFMYJPq3CXXUTKeJ" + + "kSrtaZUB5s34fWdHQNtl84mH4dVaFMunHnwgYAw"; + + @Test() + public void testEd25519() throws DescriptorParseException { + ExtraInfoDescriptor descriptor = + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE); + assertEquals(IDENTITY_ED25519_LINES.substring( + IDENTITY_ED25519_LINES.indexOf("\n") + 1), + descriptor.getIdentityEd25519()); + assertEquals(MASTER_KEY_ED25519_LINE.substring( + MASTER_KEY_ED25519_LINE.indexOf(" ") + 1), + descriptor.getMasterKeyEd25519()); + assertEquals(ROUTER_SIG_ED25519_LINE.substring( + ROUTER_SIG_ED25519_LINE.indexOf(" ") + 1), + descriptor.getRouterSignatureEd25519()); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519IdentityMasterKeyMismatch() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + "master-key-ed25519 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testEd25519IdentityMissing() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(null, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519IdentityDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES + "\n" + + IDENTITY_ED25519_LINES, MASTER_KEY_ED25519_LINE, + ROUTER_SIG_ED25519_LINE); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519IdentityEmptyCrypto() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines("identity-ed25519\n" + + "-----BEGIN ED25519 CERT-----\n-----END ED25519 CERT-----", + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testEd25519MasterKeyMissing() + throws DescriptorParseException { + ExtraInfoDescriptor descriptor = + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + null, ROUTER_SIG_ED25519_LINE); + assertEquals(MASTER_KEY_ED25519_LINE.substring( + MASTER_KEY_ED25519_LINE.indexOf(" ") + 1), + descriptor.getMasterKeyEd25519()); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519MasterKeyDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE + "\n" + MASTER_KEY_ED25519_LINE, + ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testEd25519RouterSigMissing() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, null); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519RouterSigDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE + "\n" + + ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testExtraInfoDigestSha256Relay() + throws DescriptorParseException { + byte[] descriptorBytes = ("extra-info Unnamed " + + "EA5B335055D2F03013FF030381F02B1C631EC723\n" + + "identity-ed25519\n" + + "-----BEGIN ED25519 CERT-----\n" + + "AQQABiZRAenzZorGtx6xapoEeaqcLLOk3uWwJXTvOVLluSXXbRSZAQAgBADLN5" + + "wp\nCEOrRbshSbj1NDAUgc6cxU65M/Vx1x+b5+EXbkQZ5uiyB4pphVF5kPPT1P" + + "SleYqM\n8j+tlKh2i6+Xr0xScSPpmtG00/D0MoRlT7ZdaaaT5iw1DWDQCZ8BHG" + + "lAZwU=\n" + + "-----END ED25519 CERT-----\n" + + "published 2015-12-01 04:38:12\n" + + "write-history 2015-12-01 01:40:37 (14400 s) 88704000,60825600," + + "61747200,76953600,61516800,59443200\n" + + "read-history 2015-12-01 01:40:37 (14400 s) 87321600,59443200," + + "59904000,74880000,60364800,58060800\n" + + "router-sig-ed25519 c6eUeJs/SVjun3JhmjByEeWdRDyunSMAnGVhx71JiRj" + + "YzR8x5IcPebylG7m10wiolFxinvw78UhrrGo9Sq5ZBw\n" + + "router-signature\n" + + "-----BEGIN SIGNATURE-----\n" + + "oC2qFHCDOKSRoIPR86jdRxEYia390Z4d8fT0yr/1mg4RQ7lHmxlzFT6QxCswdX" + + "Ry\nvGNGR0wARySgyE+YKKWYn/Hp547JhhWd9Oc7BuFMY0XMvl/HOo+B9VjW+l" + + "nv6UBE\niqxx3C3Iw0ymohvOenyCUa/7TmsT7eVotDO57uIoGEc=\n" + + "-----END SIGNATURE-----\n" + + "").getBytes(); + RelayExtraInfoDescriptor descriptor = + new RelayExtraInfoDescriptorImpl(descriptorBytes, true); + assertEquals("Pt1BtzfRwhYqGCDo8jjchS8nJP3ovrDyHGn+dqPbMgw", + descriptor.getExtraInfoDigestSha256()); + } + + @Test() + public void testExtraInfoDigestSha256Bridge() + throws DescriptorParseException { + byte[] descriptorBytes = ("extra-info idideditheconfig " + + "DC28749EC9E26E61DE492E46CD830379E9931B09\n" + + "master-key-ed25519 " + + "38FzmOIE6Mm85Ytx0MhFM6X9EuxWRUgb6HjyMGuO2AU\n" + + "published 2015-12-03 13:23:19\n" + + "write-history 2015-12-03 09:59:32 (14400 s) 53913600,52992000," + + "53222400,53222400,53452800,53222400\n" + + "read-history 2015-12-03 09:59:32 (14400 s) 61056000,60364800," + + "60364800,60134400,60595200,60364800\n" + + "geoip-db-digest 5BF366AD4A0572D82A1A0F6628AF8EF7725E3AB9\n" + + "geoip6-db-digest 212DE17D5A368DCAFA19B95F168BFFA101145A93\n" + + "router-digest-sha256 " + + "TvrqpjI7OmCtwGwair/NHUxg5ROVVQYz6/EDyXsDHR4\n" + + "router-digest 00B98F076B586272C3172B7F3DA29ADEE75F2ED8\n").getBytes(); + BridgeExtraInfoDescriptor descriptor = + new BridgeExtraInfoDescriptorImpl(descriptorBytes, true); + assertEquals("TvrqpjI7OmCtwGwair/NHUxg5ROVVQYz6/EDyXsDHR4", + descriptor.getExtraInfoDigestSha256()); + } } diff --git a/test/org/torproject/descriptor/impl/MicrodescriptorImplTest.java b/test/org/torproject/descriptor/impl/MicrodescriptorImplTest.java new file mode 100644 index 0000000..ab4b8c8 --- /dev/null +++ b/test/org/torproject/descriptor/impl/MicrodescriptorImplTest.java @@ -0,0 +1,82 @@ +package org.torproject.descriptor.impl; + +import org.junit.Test; +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.Microdescriptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class MicrodescriptorImplTest { + + /* Helper class to build a microdescriptor based on default data and + * modifications requested by test methods. */ + private static class DescriptorBuilder { + private String onionKeyLines = "onion-key\n" + + "-----BEGIN RSA PUBLIC KEY-----\n" + + "MIGJAoGBALNZ4pNsHHkl7a+kFWbBmPHNAepjjvuhjTr1TaMB3UKuCRaXJmS2Qr" + + "CW\nkTmINqdQUccwb3ghb7EBZfDtCUvjcwMSEsRRTVIZqVQsYj6m3n1CegOc4o" + + "UutXaZ\nfkyty5XOgV4Qucx9wokzTMCHlO0V0x9y0FwFsK5Nb6ugqfQLLQ6XAg" + + "MBAAE=\n" + + "-----END RSA PUBLIC KEY-----"; + private static Microdescriptor createWithDefaultLines() + throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + return new MicrodescriptorImpl(db.buildDescriptor(), true); + } + private String ntorOnionKeyLine = + "ntor-onion-key PXLa7IGE+TzPDMsM5j9rFnDa37rd6kfZa5QuzqqJukw="; + private String idLine = "id rsa1024 bvegfGxp8k7T9QFpjPTrPaJTa/8"; + private static Microdescriptor createWithIdLine(String line) + throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.idLine = line; + return new MicrodescriptorImpl(db.buildDescriptor(), true); + } + private byte[] buildDescriptor() { + StringBuilder sb = new StringBuilder(); + if (this.onionKeyLines != null) { + sb.append(this.onionKeyLines + "\n"); + } + if (this.ntorOnionKeyLine != null) { + sb.append(this.ntorOnionKeyLine + "\n"); + } + if (this.idLine != null) { + sb.append(this.idLine + "\n"); + } + return sb.toString().getBytes(); + } + } + + @Test() + public void testDefaults() throws DescriptorParseException { + DescriptorBuilder.createWithDefaultLines(); + } + + @Test(expected = DescriptorParseException.class) + public void testIdRsa1024TooShort() throws DescriptorParseException { + DescriptorBuilder.createWithIdLine("id rsa1024 AAAA"); + } + + @Test(expected = DescriptorParseException.class) + public void testIdRsa1024TooLong() throws DescriptorParseException { + DescriptorBuilder.createWithIdLine("id ed25519 AAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + } + + @Test(expected = DescriptorParseException.class) + public void testIdRsa512() throws DescriptorParseException { + DescriptorBuilder.createWithIdLine("id rsa512 " + + "bvegfGxp8k7T9QFpjPTrPaJTa/8"); + } + + @Test(expected = DescriptorParseException.class) + public void testIdEd25519Duplicate() throws DescriptorParseException { + DescriptorBuilder.createWithIdLine( + "id rsa1024 bvegfGxp8k7T9QFpjPTrPaJTa/8\n" + + "id rsa1024 bvegfGxp8k7T9QFpjPTrPaJTa/8"); + } +} diff --git a/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index a200dc4..46688d6 100644 --- a/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -267,7 +267,13 @@ public class RelayNetworkStatusVoteImplTest { vb.dirKeyCertificationLines = lines; return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } - private List<String> statusEntries = new ArrayList<String>(); + private List<String> statusEntries = null; + private static RelayNetworkStatusVote createWithStatusEntries( + List<String> statusEntries) throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.statusEntries = statusEntries; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); + } private String directoryFooterLine = "directory-footer"; private static RelayNetworkStatusVote createWithDirectoryFooterLine(String line) @@ -343,6 +349,10 @@ public class RelayNetworkStatusVoteImplTest { } private VoteBuilder() { + if (this.statusEntries != null) { + return; + } + this.statusEntries = new ArrayList<>(); this.statusEntries.add("r right2privassy3 " + "ADQ6gCT3DiFHKPDFr3rODBUI8HM lJY5Vf7kXec+VdkGW2flEsfkFC8 " + "2011-11-12 00:03:40 50.63.8.215 9023 0\n" @@ -1181,5 +1191,47 @@ public class RelayNetworkStatusVoteImplTest { unrecognizedLines.add(unrecognizedLine); assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); } + + @Test() + public void testIdEd25519MasterKey() + throws DescriptorParseException { + String masterKey25519 = "8RH34kO07Pp+XYwzdoATVyCibIvmbslUjRkAm7J4IA8"; + List<String> statusEntries = new ArrayList<>(); + statusEntries.add("r PDrelay1 AAFJ5u9xAqrKlpDW6N0pMhJLlKs " + + "bgJiI/la3e9u0K7cQ5pMSXhigHI 2015-12-01 04:54:30 95.215.44.189 " + + "8080 0\n" + + "id ed25519 " + masterKey25519); + RelayNetworkStatusVote vote = + VoteBuilder.createWithStatusEntries(statusEntries); + String fingerprint = vote.getStatusEntries().firstKey(); + assertEquals(masterKey25519, + vote.getStatusEntry(fingerprint).getMasterKeyEd25519()); + } + + @Test() + public void testIdEd25519None() + throws DescriptorParseException { + List<String> statusEntries = new ArrayList<>(); + statusEntries.add("r MathematicalApology AAPJIrV9nhfgTYQs0vsTghFaP2A " + + "uA7p0m68O8ILXsf3aLZUj0EvDNE 2015-12-01 18:01:49 172.99.69.177 " + + "443 9030\n" + + "id ed25519 none"); + RelayNetworkStatusVote vote = + VoteBuilder.createWithStatusEntries(statusEntries); + String fingerprint = vote.getStatusEntries().firstKey(); + assertEquals("none", + vote.getStatusEntry(fingerprint).getMasterKeyEd25519()); + } + + @Test(expected = DescriptorParseException.class) + public void testIdRsa1024None() + throws DescriptorParseException { + List<String> statusEntries = new ArrayList<>(); + statusEntries.add("r MathematicalApology AAPJIrV9nhfgTYQs0vsTghFaP2A " + + "uA7p0m68O8ILXsf3aLZUj0EvDNE 2015-12-01 18:01:49 172.99.69.177 " + + "443 9030\n" + + "id rsa1024 none"); + VoteBuilder.createWithStatusEntries(statusEntries); + } } diff --git a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index d2d03f3..a98df9f 100644 --- a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -228,11 +228,28 @@ public class ServerDescriptorImplTest { return new RelayServerDescriptorImpl(db.buildDescriptor(), failUnrecognizedDescriptorLines); } + private String identityEd25519Lines = null, + masterKeyEd25519Line = null, routerSigEd25519Line = null; + private static ServerDescriptor createWithEd25519Lines( + String identityEd25519Lines, String masterKeyEd25519Line, + String routerSigEd25519Line) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.identityEd25519Lines = identityEd25519Lines; + db.masterKeyEd25519Line = masterKeyEd25519Line; + db.routerSigEd25519Line = routerSigEd25519Line; + return new RelayServerDescriptorImpl(db.buildDescriptor(), true); + } private byte[] buildDescriptor() { StringBuilder sb = new StringBuilder(); if (this.routerLine != null) { sb.append(this.routerLine + "\n"); } + if (this.identityEd25519Lines != null) { + sb.append(this.identityEd25519Lines + "\n"); + } + if (this.masterKeyEd25519Line != null) { + sb.append(this.masterKeyEd25519Line + "\n"); + } if (this.bandwidthLine != null) { sb.append(this.bandwidthLine + "\n"); } @@ -313,6 +330,9 @@ public class ServerDescriptorImplTest { return null; } } + if (this.routerSigEd25519Line != null) { + sb.append(this.routerSigEd25519Line + "\n"); + } if (this.routerSignatureLines != null) { sb.append(this.routerSignatureLines + "\n"); } @@ -1364,5 +1384,102 @@ public class ServerDescriptorImplTest { createWithUnrecognizedLine(sb.toString().substring(1), false); assertEquals(unrecognizedLines, descriptor.getUnrecognizedLines()); } + + private static final String IDENTITY_ED25519_LINES = + "identity-ed25519\n" + + "-----BEGIN ED25519 CERT-----\n" + + "AQQABiX1AVGv5BuzJroQXbOh6vv1nbwc5rh2S13PyRFuLhTiifK4AQAgBACBCMwr" + + "\n4qgIlFDIzoC9ieJOtSkwrK+yXJPKlP8ojvgkx8cGKvhokOwA1eYDombzfwHcJ1" + + "EV\nbhEn/6g8i7wzO3LoqefIUrSAeEExOAOmm5mNmUIzL8EtnT6JHCr/sqUTUgA=" + + "\n" + + "-----END ED25519 CERT-----"; + + private static final String MASTER_KEY_ED25519_LINE = + "master-key-ed25519 gQjMK+KoCJRQyM6AvYniTrUpMKyvslyTypT/KI74JMc"; + + private static final String ROUTER_SIG_ED25519_LINE = + "router-sig-ed25519 y7WF9T2GFwkSDPZEhB55HgquIFOl5uXUFMYJPq3CXXUTKeJ" + + "kSrtaZUB5s34fWdHQNtl84mH4dVaFMunHnwgYAw"; + + @Test() + public void testEd25519() throws DescriptorParseException { + ServerDescriptor descriptor = + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE); + assertEquals(IDENTITY_ED25519_LINES.substring( + IDENTITY_ED25519_LINES.indexOf("\n") + 1), + descriptor.getIdentityEd25519()); + assertEquals(MASTER_KEY_ED25519_LINE.substring( + MASTER_KEY_ED25519_LINE.indexOf(" ") + 1), + descriptor.getMasterKeyEd25519()); + assertEquals(ROUTER_SIG_ED25519_LINE.substring( + ROUTER_SIG_ED25519_LINE.indexOf(" ") + 1), + descriptor.getRouterSignatureEd25519()); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519IdentityMasterKeyMismatch() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + "master-key-ed25519 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testEd25519IdentityMissing() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(null, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519IdentityDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES + "\n" + + IDENTITY_ED25519_LINES, MASTER_KEY_ED25519_LINE, + ROUTER_SIG_ED25519_LINE); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519IdentityEmptyCrypto() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines("identity-ed25519\n" + + "-----BEGIN ED25519 CERT-----\n-----END ED25519 CERT-----", + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testEd25519MasterKeyMissing() + throws DescriptorParseException { + ServerDescriptor descriptor = + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + null, ROUTER_SIG_ED25519_LINE); + assertEquals(MASTER_KEY_ED25519_LINE.substring( + MASTER_KEY_ED25519_LINE.indexOf(" ") + 1), + descriptor.getMasterKeyEd25519()); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519MasterKeyDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE + "\n" + MASTER_KEY_ED25519_LINE, + ROUTER_SIG_ED25519_LINE); + } + + @Test() + public void testEd25519RouterSigMissing() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, null); + } + + @Test(expected = DescriptorParseException.class) + public void testEd25519RouterSigDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, + MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE + "\n" + + ROUTER_SIG_ED25519_LINE); + } }

1 0

[metrics-lib/master] Parse hidserv-stats in extra-info descriptors.
by karsten＠torproject.org 21 Dec '15

21 Dec '15

commit f9762314f297d184a5c6e0b0f5209815c18a29bf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 16 12:06:41 2015 +0100 Parse hidserv-stats in extra-info descriptors. This patch is loosely based on metrics-web's hidserv module. --- CHANGELOG.md | 1 + .../torproject/descriptor/ExtraInfoDescriptor.java | 32 +++++ .../descriptor/impl/ExtraInfoDescriptorImpl.java | 81 ++++++++++++ .../torproject/descriptor/impl/ParseHelper.java | 34 ++++++ .../impl/ExtraInfoDescriptorImplTest.java | 129 ++++++++++++++++++++ 5 files changed, 277 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f2bc0d..bfda75e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ extra-info descriptors, and support Ed25519 master keys in votes. - Include RSA-1024 signatures of SHA-1 digests of extra-info descriptors, which were parsed and discarded before. + - Support hidden-service statistics in extra-info descriptors. # Changes in version 1.0.0 - 2015-12-05 diff --git a/src/org/torproject/descriptor/ExtraInfoDescriptor.java b/src/org/torproject/descriptor/ExtraInfoDescriptor.java index 1b978a4..ed0141d 100644 --- a/src/org/torproject/descriptor/ExtraInfoDescriptor.java +++ b/src/org/torproject/descriptor/ExtraInfoDescriptor.java @@ -3,6 +3,7 @@ package org.torproject.descriptor; import java.util.List; +import java.util.Map; import java.util.SortedMap; /* Contains a relay or bridge extra-info descriptor. */ @@ -265,6 +266,37 @@ public interface ExtraInfoDescriptor extends Descriptor { * bridge. */ public List<String> getTransports(); + /* Return the end of the included hidden-service statistics, or -1 if no + * hidden-service statistics are included. */ + public long getHidservStatsEndMillis(); + + /* Return the interval length of the included hidden-service statistics + * in seconds, or -1 if no hidden-service statistics are included. */ + public long getHidservStatsIntervalLength(); + + /* Return the approximate number of RELAY cells seen in either direction + * on a circuit after receiving and successfully processing a + * RENDEZVOUS1 cell, or null if no hidden-service statistics are + * included. */ + public Double getHidservRendRelayedCells(); + + /* Return the obfuscation parameters applied to the original measurement + * value of RELAY cells seen in either direction on a circuit after + * receiving and successfully processing a RENDEZVOUS1 cell, or null if + * no hidden-service statistics are included.. */ + public Map<String, Double> getHidservRendRelayedCellsParameters(); + + /* Return the approximate number of unique hidden-service identities + * seen in descriptors published to and accepted by this hidden-service + * directory, or null if no hidden-service statistics are included. */ + public Double getHidservDirOnionsSeen(); + + /* Return the obfuscation parameters applied to the original measurement + * value of unique hidden-service identities seen in descriptors + * published to and accepted by this hidden-service directory, or null + * if no hidden-service statistics are included. */ + public Map<String, Double> getHidservDirOnionsSeenParameters(); + /* Return the signature of the PKCS1-padded extra-info descriptor * digest, or null if the descriptor doesn't contain a signature (which * is the case in sanitized bridge descriptors). */ diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 4abace6..ef0c82c 100644 --- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -9,8 +9,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.SortedMap; @@ -165,6 +167,13 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl this.parseBridgeIpTransportsLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("transport")) { this.parseTransportLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("hidserv-stats-end")) { + this.parseHidservStatsEndLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("hidserv-rend-relayed-cells")) { + this.parseHidservRendRelayedCellsLine(line, lineNoOpt, + partsNoOpt); + } else if (keyword.equals("hidserv-dir-onions-seen")) { + this.parseHidservDirOnionsSeenLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("identity-ed25519")) { this.parseIdentityEd25519Line(line, lineNoOpt, partsNoOpt); nextCrypto = "identity-ed25519"; @@ -642,6 +651,46 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl this.transports.add(partsNoOpt[1]); } + private void parseHidservStatsEndLine(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + long[] parsedStatsEndData = this.parseStatsEndLine(line, partsNoOpt, + 5); + this.hidservStatsEndMillis = parsedStatsEndData[0]; + this.hidservStatsIntervalLength = parsedStatsEndData[1]; + } + + private void parseHidservRendRelayedCellsLine(String line, + String lineNoOpt, String[] partsNoOpt) + throws DescriptorParseException { + if (partsNoOpt.length < 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + try { + this.hidservRendRelayedCells = Double.parseDouble(partsNoOpt[1]); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.hidservRendRelayedCellsParameters = + ParseHelper.parseSpaceSeparatedStringKeyDoubleValueMap(line, + partsNoOpt, 2); + } + + private void parseHidservDirOnionsSeenLine(String line, + String lineNoOpt, String[] partsNoOpt) + throws DescriptorParseException { + if (partsNoOpt.length < 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + try { + this.hidservDirOnionsSeen = Double.parseDouble(partsNoOpt[1]); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.hidservDirOnionsSeenParameters = + ParseHelper.parseSpaceSeparatedStringKeyDoubleValueMap(line, + partsNoOpt, 2); + } + private void parseRouterSignatureLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { if (!lineNoOpt.equals("router-signature")) { @@ -1057,6 +1106,38 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl return new ArrayList<String>(this.transports); } + private long hidservStatsEndMillis = -1L; + public long getHidservStatsEndMillis() { + return this.hidservStatsEndMillis; + } + + private long hidservStatsIntervalLength = -1L; + public long getHidservStatsIntervalLength() { + return this.hidservStatsIntervalLength; + } + + private Double hidservRendRelayedCells; + public Double getHidservRendRelayedCells() { + return this.hidservRendRelayedCells; + } + + private Map<String, Double> hidservRendRelayedCellsParameters; + public Map<String, Double> getHidservRendRelayedCellsParameters() { + return this.hidservRendRelayedCellsParameters == null ? null : + new HashMap<>(this.hidservRendRelayedCellsParameters); + } + + private Double hidservDirOnionsSeen; + public Double getHidservDirOnionsSeen() { + return this.hidservDirOnionsSeen; + } + + private Map<String, Double> hidservDirOnionsSeenParameters; + public Map<String, Double> getHidservDirOnionsSeenParameters() { + return this.hidservDirOnionsSeenParameters == null ? null : + new HashMap<>(this.hidservDirOnionsSeenParameters); + } + private String routerSignature; public String getRouterSignature() { return this.routerSignature; diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java index a354831..15de5ee 100644 --- a/src/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/org/torproject/descriptor/impl/ParseHelper.java @@ -6,6 +6,7 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; import java.util.SortedMap; @@ -435,6 +436,39 @@ public class ParseHelper { return result; } + protected static Map<String, Double> + parseSpaceSeparatedStringKeyDoubleValueMap(String line, + String[] partsNoOpt, int startIndex) + throws DescriptorParseException { + Map<String, Double> result = new LinkedHashMap<>(); + if (partsNoOpt.length < startIndex) { + throw new DescriptorParseException("Line '" + line + "' does not " + + "contain a key-value list starting at index " + startIndex + + "."); + } + for (int i = startIndex; i < partsNoOpt.length; i++) { + String listElement = partsNoOpt[i]; + String[] keyAndValue = listElement.split("="); + String key = null; + Double value = null; + if (keyAndValue.length == 2) { + try { + value = Double.parseDouble(keyAndValue[1]); + key = keyAndValue[0]; + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (key == null) { + throw new DescriptorParseException("Line '" + line + "' contains " + + "an illegal key or value in list element '" + listElement + + "'."); + } + result.put(key, value); + } + return result; + } + public static String parseMasterKeyEd25519FromIdentityEd25519CryptoBlock( String identityEd25519CryptoBlock) throws DescriptorParseException { diff --git a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index d70ac39..55e0578 100644 --- a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -138,6 +138,13 @@ public class ExtraInfoDescriptorImplTest { db.bridgeStatsLines = lines; return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true); } + private String hidservStatsLines = null; + private static ExtraInfoDescriptor createWithHidservStatsLines( + String lines) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.hidservStatsLines = lines; + return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true); + } private String unrecognizedLine = null; private static ExtraInfoDescriptor createWithUnrecognizedLine( String line, boolean failUnrecognizedDescriptorLines) @@ -232,6 +239,9 @@ public class ExtraInfoDescriptorImplTest { if (this.bridgeStatsLines != null) { sb.append(this.bridgeStatsLines + "\n"); } + if (this.hidservStatsLines != null) { + sb.append(this.hidservStatsLines + "\n"); + } if (this.unrecognizedLine != null) { sb.append(this.unrecognizedLine + "\n"); } @@ -733,6 +743,62 @@ public class ExtraInfoDescriptorImplTest { } } + /* Helper class to build a set of hidserv-stats lines based on default + * data and modifications requested by test methods. */ + private static class HidservStatsBuilder { + private String hidservStatsEndLine = "hidserv-stats-end 2015-12-03 " + + "14:26:56 (86400 s)"; + private static ExtraInfoDescriptor createWithHidservStatsEndLine( + String line) throws DescriptorParseException { + HidservStatsBuilder hsb = new HidservStatsBuilder(); + hsb.hidservStatsEndLine = line; + return DescriptorBuilder.createWithHidservStatsLines( + hsb.buildHidservStatsLines()); + } + private String hidservRendRelayedCellsLine = + "hidserv-rend-relayed-cells 36474281 delta_f=2048 epsilon=0.30 " + + "bin_size=1024"; + private static ExtraInfoDescriptor + createWithHidservRendRelayedCellsLine(String line) + throws DescriptorParseException { + HidservStatsBuilder hsb = new HidservStatsBuilder(); + hsb.hidservRendRelayedCellsLine = line; + return DescriptorBuilder.createWithHidservStatsLines( + hsb.buildHidservStatsLines()); + } + private String hidservDirOnionsSeenLine = "hidserv-dir-onions-seen " + + "-3 delta_f=8 epsilon=0.30 bin_size=8"; + private static ExtraInfoDescriptor createWithHidservDirOnionsSeenLine( + String line) throws DescriptorParseException { + HidservStatsBuilder hsb = new HidservStatsBuilder(); + hsb.hidservDirOnionsSeenLine = line; + return DescriptorBuilder.createWithHidservStatsLines( + hsb.buildHidservStatsLines()); + } + private static ExtraInfoDescriptor createWithDefaultLines() + throws DescriptorParseException { + return DescriptorBuilder.createWithHidservStatsLines( + new HidservStatsBuilder().buildHidservStatsLines()); + } + private String buildHidservStatsLines() { + StringBuilder sb = new StringBuilder(); + if (this.hidservStatsEndLine != null) { + sb.append(this.hidservStatsEndLine + "\n"); + } + if (this.hidservRendRelayedCellsLine != null) { + sb.append(this.hidservRendRelayedCellsLine + "\n"); + } + if (this.hidservDirOnionsSeenLine != null) { + sb.append(this.hidservDirOnionsSeenLine + "\n"); + } + String lines = sb.toString(); + if (lines.endsWith("\n")) { + lines = lines.substring(0, lines.length() - 1); + } + return lines; + } + } + @Test() public void testSampleDescriptor() throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); @@ -1415,6 +1481,69 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testHidservStatsValid() throws DescriptorParseException { + ExtraInfoDescriptor descriptor = HidservStatsBuilder. + createWithDefaultLines(); + assertEquals(1449152816000L, descriptor.getHidservStatsEndMillis()); + assertEquals(86400L, descriptor.getHidservStatsIntervalLength()); + assertEquals(36474281.0, descriptor.getHidservRendRelayedCells(), + 0.0001); + Map<String, Double> params = + descriptor.getHidservRendRelayedCellsParameters(); + assertTrue(params.containsKey("delta_f")); + assertEquals(2048.0, params.remove("delta_f"), 0.0001); + assertTrue(params.containsKey("epsilon")); + assertEquals(0.3, params.remove("epsilon"), 0.0001); + assertTrue(params.containsKey("bin_size")); + assertEquals(1024.0, params.remove("bin_size"), 0.0001); + assertTrue(params.isEmpty()); + assertEquals(-3.0, descriptor.getHidservDirOnionsSeen(), 0.0001); + params = descriptor.getHidservDirOnionsSeenParameters(); + assertTrue(params.containsKey("delta_f")); + assertEquals(8.0, params.remove("delta_f"), 0.0001); + assertTrue(params.containsKey("epsilon")); + assertEquals(0.3, params.remove("epsilon"), 0.0001); + assertTrue(params.containsKey("bin_size")); + assertEquals(8.0, params.remove("bin_size"), 0.0001); + assertTrue(params.isEmpty()); + } + + @Test() + public void testHidservStatsEndLineMissing() + throws DescriptorParseException { + ExtraInfoDescriptor descriptor = + HidservStatsBuilder.createWithHidservStatsEndLine(null); + assertEquals(-1L, descriptor.getHidservStatsEndMillis()); + assertEquals(-1L, descriptor.getHidservStatsIntervalLength()); + } + + @Test() + public void testHidservRendRelayedCellsNoParams() + throws DescriptorParseException { + ExtraInfoDescriptor descriptor = + HidservStatsBuilder.createWithHidservRendRelayedCellsLine( + "hidserv-rend-relayed-cells 36474281"); + assertEquals(36474281.0, descriptor.getHidservRendRelayedCells(), + 0.0001); + assertTrue( + descriptor.getHidservRendRelayedCellsParameters().isEmpty()); + } + + @Test(expected = DescriptorParseException.class) + public void testHidservDirOnionsSeenCommaSeparatedParams() + throws DescriptorParseException { + HidservStatsBuilder.createWithHidservDirOnionsSeenLine( + "hidserv-dir-onions-seen -3 delta_f=8,epsilon=0.30,bin_size=8"); + } + + @Test(expected = DescriptorParseException.class) + public void testHidservDirOnionsSeenNoDoubleParams() + throws DescriptorParseException { + HidservStatsBuilder.createWithHidservDirOnionsSeenLine( + "hidserv-dir-onions-seen -3 delta_f=A epsilon=B bin_size=C"); + } + + @Test() public void testRouterSignatureOpt() throws DescriptorParseException { DescriptorBuilder.createWithRouterSignatureLines("opt "

1 0

[metrics-lib/master] Support (ntor-)onion-key crosscerts.
by karsten＠torproject.org 21 Dec '15

21 Dec '15

commit f4fca35135482979f85f7891b075b46a194ef37e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 16 16:09:17 2015 +0100 Support (ntor-)onion-key crosscerts. --- CHANGELOG.md | 2 + .../torproject/descriptor/ServerDescriptor.java | 17 +++++ .../descriptor/impl/ServerDescriptorImpl.java | 50 +++++++++++++- .../descriptor/impl/ServerDescriptorImplTest.java | 73 ++++++++++++++++++++ 4 files changed, 139 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfda75e..73e9a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ - Include RSA-1024 signatures of SHA-1 digests of extra-info descriptors, which were parsed and discarded before. - Support hidden-service statistics in extra-info descriptors. + - Support onion-key and ntor-onion-key cross certificates in server + descriptors. # Changes in version 1.0.0 - 2015-12-05 diff --git a/src/org/torproject/descriptor/ServerDescriptor.java b/src/org/torproject/descriptor/ServerDescriptor.java index 3266e9d..49eaa92 100644 --- a/src/org/torproject/descriptor/ServerDescriptor.java +++ b/src/org/torproject/descriptor/ServerDescriptor.java @@ -174,5 +174,22 @@ public interface ServerDescriptor extends Descriptor { * the first space after the "router-sig-ed25519" string, prefixed with * the string "Tor router descriptor signature v1". */ public String getRouterSignatureEd25519(); + + /* Return an RSA signature, generated using the onion-key, that proves + * that the party creating the descriptor had control over the secret + * key corresponding to the onion-key, or null if the descriptor does + * not contain such a signature. */ + public String getOnionKeyCrosscert(); + + /* Return an Ed25519 signature, generated using the ntor-onion-key, that + * proves that the party creating the descriptor had control over the + * secret key corresponding to the ntor-onion-key, or null if the + * descriptor does not contain such a signature. */ + public String getNtorOnionKeyCrosscert(); + + /* Return the sign of the Ed25519 public key corresponding to the ntor + * onion key as 0 or 1, or -1 if the descriptor does not contain this + * information. */ + public int getNtorOnionKeyCrosscertSign(); } diff --git a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java index 3dd6c40..1484866 100644 --- a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -37,9 +37,9 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl + "hibernating,uptime,contact,family,read-history,write-history," + "eventdns,caches-extra-info,extra-info-digest," + "hidden-service-dir,protocols,allow-single-hop-exits,onion-key," - + "signing-key,ipv6-policy,ntor-onion-key,router-sig-ed25519," - + "router-signature,router-digest-sha256,router-digest"). - split(","))); + + "signing-key,ipv6-policy,ntor-onion-key,onion-key-crosscert," + + "ntor-onion-key-crosscert,router-sig-ed25519,router-signature," + + "router-digest-sha256,router-digest").split(","))); this.checkAtMostOnceKeywords(atMostOnceKeywords); this.checkFirstKeyword("router"); if (this.getKeywordCount("accept") == 0 && @@ -131,6 +131,12 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl this.parseMasterKeyEd25519Line(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("router-sig-ed25519")) { this.parseRouterSigEd25519Line(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("onion-key-crosscert")) { + this.parseOnionKeyCrosscert(line, lineNoOpt, partsNoOpt); + nextCrypto = "onion-key-crosscert"; + } else if (keyword.equals("ntor-onion-key-crosscert")) { + this.parseNtorOnionKeyCrosscert(line, lineNoOpt, partsNoOpt); + nextCrypto = "ntor-onion-key-crosscert"; } else if (line.startsWith("-----BEGIN")) { cryptoLines = new ArrayList<String>(); cryptoLines.add(line); @@ -150,6 +156,10 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl } else if ("identity-ed25519".equals(nextCrypto)) { this.identityEd25519 = cryptoString; this.parseIdentityEd25519CryptoBlock(cryptoString); + } else if ("onion-key-crosscert".equals(nextCrypto)) { + this.onionKeyCrosscert = cryptoString; + } else if ("ntor-onion-key-crosscert".equals(nextCrypto)) { + this.ntorOnionKeyCrosscert = cryptoString; } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized crypto " + "block '" + cryptoString + "' in server descriptor."); @@ -534,6 +544,25 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl } } + private void parseOnionKeyCrosscert(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 1) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + } + + private void parseNtorOnionKeyCrosscert(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + if (partsNoOpt.length != 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + try { + this.ntorOnionKeyCrosscertSign = Integer.parseInt(partsNoOpt[1]); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + } + private void parseIdentityEd25519CryptoBlock(String cryptoString) throws DescriptorParseException { String masterKeyEd25519FromIdentityEd25519 = @@ -834,5 +863,20 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl public String getRouterSignatureEd25519() { return this.routerSignatureEd25519; } + + private String onionKeyCrosscert; + public String getOnionKeyCrosscert() { + return this.onionKeyCrosscert; + } + + private String ntorOnionKeyCrosscert; + public String getNtorOnionKeyCrosscert() { + return this.ntorOnionKeyCrosscert; + } + + private int ntorOnionKeyCrosscertSign = -1; + public int getNtorOnionKeyCrosscertSign() { + return ntorOnionKeyCrosscertSign; + } } diff --git a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index a98df9f..56f1419 100644 --- a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -104,6 +104,20 @@ public class ServerDescriptorImplTest { db.signingKeyLines = lines; return new RelayServerDescriptorImpl(db.buildDescriptor(), true); } + private String onionKeyCrosscertLines = null; + private static ServerDescriptor createWithOnionKeyCrosscertLines( + String lines) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.onionKeyCrosscertLines = lines; + return new RelayServerDescriptorImpl(db.buildDescriptor(), true); + } + private String ntorOnionKeyCrosscertLines = null; + private static ServerDescriptor createWithNtorOnionKeyCrosscertLines( + String lines) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.ntorOnionKeyCrosscertLines = lines; + return new RelayServerDescriptorImpl(db.buildDescriptor(), true); + } private String exitPolicyLines = "reject *:*"; private static ServerDescriptor createWithExitPolicyLines( String lines) throws DescriptorParseException { @@ -274,6 +288,12 @@ public class ServerDescriptorImplTest { if (this.signingKeyLines != null) { sb.append(this.signingKeyLines + "\n"); } + if (this.onionKeyCrosscertLines != null) { + sb.append(this.onionKeyCrosscertLines + "\n"); + } + if (this.ntorOnionKeyCrosscertLines != null) { + sb.append(this.ntorOnionKeyCrosscertLines + "\n"); + } if (this.exitPolicyLines != null) { sb.append(this.exitPolicyLines + "\n"); } @@ -1481,5 +1501,58 @@ public class ServerDescriptorImplTest { MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE + "\n" + ROUTER_SIG_ED25519_LINE); } + + private static final String ONION_KEY_CROSSCERT_LINES = + "onion-key-crosscert\n" + + "-----BEGIN CROSSCERT-----\n" + + "gVWpiNgG2FekW1uonr4KKoqykjr4bqUBKGZfu6s9rvsV1TThnquZNP6ZhX2IPdQA" + + "\nlfKtzFggGu/4BiJ5oTSDj2sK2DMjY3rjrMQZ3I/wJ25yhc9gxjqYqUYO9MmJwA" + + "Lp\nfYkqp/t4WchJpyva/4hK8vITsI6eT2BfY/DWMy/suIE=\n" + + "-----END CROSSCERT-----"; + + private static final String NTOR_ONION_KEY_CROSSCERT_LINES = + "ntor-onion-key-crosscert 1\n" + + "-----BEGIN ED25519 CERT-----\n" + + "AQoABiUeAdauu1MxYGMmGLTCPaoes0RvW7udeLc1t8LZ4P3CDo5bAN4nrRfbCfOt" + + "\nz2Nwqn8tER1a+Ry6Vs+ilMZA55Rag4+f6Zdb1fmHWknCxbQlLHpqHACMtemPda" + + "Ka\nErPtMuiEqAc=\n" + + "-----END ED25519 CERT-----"; + + @Test() + public void testOnionKeyCrosscert() throws DescriptorParseException { + ServerDescriptor descriptor = + DescriptorBuilder.createWithOnionKeyCrosscertLines( + ONION_KEY_CROSSCERT_LINES); + assertEquals(ONION_KEY_CROSSCERT_LINES.substring( + ONION_KEY_CROSSCERT_LINES.indexOf("\n") + 1), + descriptor.getOnionKeyCrosscert()); + } + + @Test(expected = DescriptorParseException.class) + public void testOnionKeyCrosscertDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithOnionKeyCrosscertLines( + ONION_KEY_CROSSCERT_LINES + "\n" + ONION_KEY_CROSSCERT_LINES); + } + + @Test() + public void testNtorOnionKeyCrosscert() + throws DescriptorParseException { + ServerDescriptor descriptor = + DescriptorBuilder.createWithNtorOnionKeyCrosscertLines( + NTOR_ONION_KEY_CROSSCERT_LINES); + assertEquals(NTOR_ONION_KEY_CROSSCERT_LINES.substring( + NTOR_ONION_KEY_CROSSCERT_LINES.indexOf("\n") + 1), + descriptor.getNtorOnionKeyCrosscert()); + assertEquals(1, descriptor.getNtorOnionKeyCrosscertSign()); + } + + @Test(expected = DescriptorParseException.class) + public void testNtorOnionKeyCrosscertDuplicate() + throws DescriptorParseException { + DescriptorBuilder.createWithOnionKeyCrosscertLines( + NTOR_ONION_KEY_CROSSCERT_LINES + "\n" + + NTOR_ONION_KEY_CROSSCERT_LINES); + } }

1 0

[tor-browser-bundle/hardened-builds] Bug 17875: Discourage editing of torrc-defaults.
by gk＠torproject.org 21 Dec '15

21 Dec '15

commit ac11a0c9cd84d0287ae85fb7357e55535e11f6a9 Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Mon Dec 21 12:03:13 2015 -0500 Bug 17875: Discourage editing of torrc-defaults. --- Bundle-Data/linux/Data/Tor/torrc-defaults | 6 ++++++ Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults | 6 ++++++ Bundle-Data/windows/Data/Tor/torrc-defaults | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/Bundle-Data/linux/Data/Tor/torrc-defaults b/Bundle-Data/linux/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/linux/Data/Tor/torrc-defaults +++ b/Bundle-Data/linux/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity] diff --git a/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults b/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults +++ b/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity] diff --git a/Bundle-Data/windows/Data/Tor/torrc-defaults b/Bundle-Data/windows/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/windows/Data/Tor/torrc-defaults +++ b/Bundle-Data/windows/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity]

1 0

[tor-browser-bundle/maint-5.0] Bug 17875: Discourage editing of torrc-defaults.
by gk＠torproject.org 21 Dec '15

21 Dec '15

commit 41b71527e0d64a3b14290a82c96014e663f8b93d Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Mon Dec 21 12:03:13 2015 -0500 Bug 17875: Discourage editing of torrc-defaults. --- Bundle-Data/linux/Data/Tor/torrc-defaults | 6 ++++++ Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults | 6 ++++++ Bundle-Data/windows/Data/Tor/torrc-defaults | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/Bundle-Data/linux/Data/Tor/torrc-defaults b/Bundle-Data/linux/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/linux/Data/Tor/torrc-defaults +++ b/Bundle-Data/linux/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity] diff --git a/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults b/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults +++ b/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity] diff --git a/Bundle-Data/windows/Data/Tor/torrc-defaults b/Bundle-Data/windows/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/windows/Data/Tor/torrc-defaults +++ b/Bundle-Data/windows/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity]

1 0

[tor-browser-bundle/master] Bug 17875: Discourage editing of torrc-defaults.
by gk＠torproject.org 21 Dec '15

21 Dec '15

commit 940d6ea21668e99619185ef325be39223ffd93ac Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Mon Dec 21 12:03:13 2015 -0500 Bug 17875: Discourage editing of torrc-defaults. --- Bundle-Data/linux/Data/Tor/torrc-defaults | 6 ++++++ Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults | 6 ++++++ Bundle-Data/windows/Data/Tor/torrc-defaults | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/Bundle-Data/linux/Data/Tor/torrc-defaults b/Bundle-Data/linux/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/linux/Data/Tor/torrc-defaults +++ b/Bundle-Data/linux/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity] diff --git a/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults b/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults +++ b/Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity] diff --git a/Bundle-Data/windows/Data/Tor/torrc-defaults b/Bundle-Data/windows/Data/Tor/torrc-defaults index 4b91aae..0a87e5f 100644 --- a/Bundle-Data/windows/Data/Tor/torrc-defaults +++ b/Bundle-Data/windows/Data/Tor/torrc-defaults @@ -1,3 +1,9 @@ +# torrc-defaults for Tor Browser +# +# This file is distributed with Tor Browser and should not be modified (it +# may be overwritten during the next Tor Browser update). To customize your +# Tor configuration, shut down Tor Browser and edit the torrc file. +# # If non-zero, try to write to disk less frequently than we would otherwise. AvoidDiskWrites 1 # Where to send logging messages. Format is minSeverity[-maxSeverity]

1 0

[tor/master] whoops; really fix the 32-bit builds
by nickm＠torproject.org 21 Dec '15

21 Dec '15

commit f2a5df252f3ba84f2bc98862fc35f9020ce78fd5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Dec 21 13:10:10 2015 -0500 whoops; really fix the 32-bit builds --- src/or/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/config.c b/src/or/config.c index 53f2c76..e3d6653 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4102,7 +4102,7 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, if (total_mem == 0) { if (get_total_system_memory(&total_mem) < 0) { total_mem = options->MaxMemInQueues >= SIZE_MAX ? - SIZE_MAX : options->MaxMemInQueues; + SIZE_MAX : (size_t)options->MaxMemInQueues; } } if (options->DirCache) {

1 0

[tor/master] Fix a couple of jenkins issues from 12538.
by nickm＠torproject.org 21 Dec '15

21 Dec '15

commit 8ede8d411a29450a746efdfb950c916b0058b497 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Dec 21 12:32:20 2015 -0500 Fix a couple of jenkins issues from 12538. --- src/or/config.c | 8 ++++++-- src/test/include.am | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index ce7adba..53f2c76 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4097,9 +4097,13 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, char **msg) { *msg = NULL; + /* XXX We should possibly be looking at MaxMemInQueues here + * unconditionally. Or we should believe total_mem unconditionally. */ if (total_mem == 0) { - if (get_total_system_memory(&total_mem) < 0) - total_mem = options->MaxMemInQueues; + if (get_total_system_memory(&total_mem) < 0) { + total_mem = options->MaxMemInQueues >= SIZE_MAX ? + SIZE_MAX : options->MaxMemInQueues; + } } if (options->DirCache) { if (total_mem < DIRCACHE_MIN_BANDWIDTH) { diff --git a/src/test/include.am b/src/test/include.am index 44a2d86..786fb77 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -193,6 +193,7 @@ noinst_HEADERS+= \ src/test/rend_test_helpers.h \ src/test/test.h \ src/test/test_helpers.h \ + src/test/test_dir_common.h \ src/test/test_descriptors.inc \ src/test/example_extrainfo.inc \ src/test/failing_routerdescs.inc \

1 0

[tor/master] Add new DirCache configuration option
by nickm＠torproject.org 21 Dec '15

21 Dec '15

commit 997f779a7f05540e5f564b4d121706c4a7069fb2 Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Sun Feb 8 06:51:51 2015 +0000 Add new DirCache configuration option This will give relay operators the ability of disabling the caching of directory data. In general, this should not be necessary, but on some lower-resource systems it may beneficial. --- doc/tor.1.txt | 6 +++ src/or/config.c | 61 +++++++++++++++++++++++++ src/or/config.h | 2 + src/or/or.h | 4 ++ src/or/router.c | 4 +- src/test/test_options.c | 115 ++++++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 186 insertions(+), 6 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index f173a97..021353d 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1987,6 +1987,12 @@ if DirPort is non-zero): except that port specifiers are ignored. Any address not matched by some entry in the policy is accepted. +[[DirCache]] **DirCache** **0**|**1**:: + When this option is set, Tor caches all current directory documents and + accepts client requests for them. Setting DirPort is not required for this, + because clients connect via the ORPort by default. Setting either DirPort + or BridgeRelay and setting DirCache to 0 is not supported. (Default: 1) + DIRECTORY AUTHORITY SERVER OPTIONS ---------------------------------- diff --git a/src/or/config.c b/src/or/config.c index 9ec47d2..f9cab9f 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -222,6 +222,7 @@ static config_var_t option_vars_[] = { V(DirPortFrontPage, FILENAME, NULL), VAR("DirReqStatistics", BOOL, DirReqStatistics_option, "1"), VAR("DirAuthority", LINELIST, DirAuthorities, NULL), + V(DirCache, BOOL, "1"), V(DirAuthorityFallbackRate, DOUBLE, "1.0"), V(DisableAllSwap, BOOL, "0"), V(DisableDebuggerAttachment, BOOL, "1"), @@ -3457,6 +3458,24 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("AccountingRule must be 'sum' or 'max'"); } + if (options->DirPort_set && !options->DirCache) { + REJECT("DirPort configured but DirCache disabled. DirPort requires " + "DirCache."); + } + + if (options->BridgeRelay && !options->DirCache) { + REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires " + "DirCache."); + } + + if (server_mode(options)) { + char *msg = NULL; + if (have_enough_mem_for_dircache(options, 0, &msg)) { + log_warn(LD_CONFIG, "%s", msg); + tor_free(msg); + } + } + if (options->HTTPProxy) { /* parse it now */ if (tor_addr_port_lookup(options->HTTPProxy, &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0) @@ -4065,6 +4084,48 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess) } } +/* If we have less than 300 MB suggest disabling dircache */ +#define DIRCACHE_MIN_MB_BANDWIDTH 300 +#define DIRCACHE_MIN_BANDWIDTH (DIRCACHE_MIN_MB_BANDWIDTH*ONE_MEGABYTE) +#define STRINGIFY(val) #val + +/** Create a warning message for emitting if we are a dircache but may not have + * enough system memory, or if we are not a dircache but probably should be. + * Return -1 when a message is returned in *msg*, else return 0. */ +STATIC int +have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, + char **msg) +{ + *msg = NULL; + if (total_mem == 0) { + if (get_total_system_memory(&total_mem) < 0) + total_mem = options->MaxMemInQueues; + } + if (options->DirCache) { + if (total_mem < DIRCACHE_MIN_BANDWIDTH) { + if (options->BridgeRelay) { + *msg = strdup("Running a Bridge with less than " + STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is " + "not recommended."); + } else { + *msg = strdup("Being a directory cache (default) with less than " + STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is " + "not recommended and may consume most of the available " + "resources, consider disabling this functionality by " + "setting the DirCache option to 0."); + } + } + } else { + if (total_mem >= DIRCACHE_MIN_BANDWIDTH) { + *msg = strdup("DirCache is disabled and we are configured as a " + "relay. This may disqualify us from becoming a guard in the " + "future."); + } + } + return *msg == NULL ? 0 : -1; +} +#undef STRINGIFY + /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL * equal strings. */ static int diff --git a/src/or/config.h b/src/or/config.h index bfdd169..6e08f9d 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -158,6 +158,8 @@ STATIC int parse_dir_authority_line(const char *line, dirinfo_type_t required_type, int validate_only); STATIC int parse_dir_fallback_line(const char *line, int validate_only); +STATIC int have_enough_mem_for_dircache(const or_options_t *options, + size_t total_mem, char **msg); #endif #endif diff --git a/src/or/or.h b/src/or/or.h index 3cb1e7d..89c5398 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3969,6 +3969,10 @@ typedef struct { /** Should we fetch our dir info at the start of the consensus period? */ int FetchDirInfoExtraEarly; + int DirCache; /**< Cache all directory documents and accept requests via + * tunnelled dir conns from clients. If 1, enabled (default); + * If 0, disabled. */ + char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual * MAPADDRESS requests for IPv4 addresses */ char *VirtualAddrNetworkIPv6; /**< Address and mask to hand out for virtual diff --git a/src/or/router.c b/src/or/router.c index 1927cfd..5e4f855 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1176,7 +1176,9 @@ router_should_be_directory_server(const or_options_t *options, int dir_port) int dir_server_mode(const or_options_t *options) { - return (server_mode(options) || options->DirPort_set) && + if (!options->DirCache) + return 0; + return (server_mode(options) || options->DirPort_set) && router_should_be_directory_server(options, 0); } diff --git a/src/test/test_options.c b/src/test/test_options.c index a8ebadb..7e47f33 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -69,22 +69,29 @@ clear_log_messages(void) messages = NULL; } +#define setup_options(opt,dflt) \ + do { \ + opt = options_new(); \ + opt->command = CMD_RUN_TOR; \ + options_init(opt); \ + \ + dflt = config_dup(&options_format, opt); \ + clear_log_messages(); \ + } while (0) + static void test_options_validate_impl(const char *configuration, const char *expect_errmsg, int expect_log_severity, const char *expect_log) { - or_options_t *opt = options_new(); + or_options_t *opt=NULL; or_options_t *dflt; config_line_t *cl=NULL; char *msg=NULL; int r; - opt->command = CMD_RUN_TOR; - options_init(opt); - dflt = config_dup(&options_format, opt); - clear_log_messages(); + setup_options(opt, dflt); r = config_get_lines(configuration, &cl, 1); tt_int_op(r, OP_EQ, 0); @@ -159,12 +166,110 @@ test_options_validate(void *arg) "ServerTransportOptions did not parse", LOG_WARN, "\"slingsnappy\" is not a k=v"); + WANT_ERR("DirPort 8080\nDirCache 0", + "DirPort configured but DirCache disabled."); + WANT_ERR("BridgeRelay 1\nDirCache 0", + "We're a bridge but DirCache is disabled."); + clear_log_messages(); return; } +#define MEGABYTEIFY(mb) (U64_LITERAL(mb) << 20) +static void +test_have_enough_mem_for_dircache(void *arg) +{ + (void)arg; + or_options_t *opt=NULL; + or_options_t *dflt; + config_line_t *cl=NULL; + char *msg=NULL;; + int r; + const char *configuration = "ORPort 8080\nDirCache 1", *expect_errmsg; + + setup_options(opt, dflt); + setup_log_callback(); + (void)dflt; + + r = config_get_lines(configuration, &cl, 1); + tt_int_op(r, OP_EQ, 0); + + r = config_assign(&options_format, opt, cl, 0, 0, &msg); + tt_int_op(r, OP_EQ, 0); + + /* 300 MB RAM available, DirCache enabled */ + r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg); + tt_int_op(r, OP_EQ, 0); + tt_assert(!msg); + + /* 200 MB RAM available, DirCache enabled */ + r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg); + tt_int_op(r, OP_EQ, -1); + expect_errmsg = "Being a directory cache (default) with less than "; + if (!strstr(msg, expect_errmsg)) { + TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.", + expect_errmsg, configuration, msg)); + } + tor_free(msg); + + configuration = "ORPort 8080\nDirCache 1\nBridgeRelay 1"; + r = config_get_lines(configuration, &cl, 1); + tt_int_op(r, OP_EQ, 0); + + r = config_assign(&options_format, opt, cl, 0, 0, &msg); + tt_int_op(r, OP_EQ, 0); + + /* 300 MB RAM available, DirCache enabled, Bridge */ + r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg); + tt_int_op(r, OP_EQ, 0); + tt_assert(!msg); + + /* 200 MB RAM available, DirCache enabled, Bridge */ + r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg); + tt_int_op(r, OP_EQ, -1); + expect_errmsg = "Running a Bridge with less than "; + if (!strstr(msg, expect_errmsg)) { + TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.", + expect_errmsg, configuration, msg)); + } + tor_free(msg); + + configuration = "ORPort 8080\nDirCache 0"; + r = config_get_lines(configuration, &cl, 1); + tt_int_op(r, OP_EQ, 0); + + r = config_assign(&options_format, opt, cl, 0, 0, &msg); + tt_int_op(r, OP_EQ, 0); + + /* 200 MB RAM available, DirCache disabled */ + r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg); + tt_int_op(r, OP_EQ, 0); + tt_assert(!msg); + + /* 300 MB RAM available, DirCache disabled */ + r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg); + tt_int_op(r, OP_EQ, -1); + expect_errmsg = "DirCache is disabled and we are configured as a "; + if (!strstr(msg, expect_errmsg)) { + TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.", + expect_errmsg, configuration, msg)); + } + tor_free(msg); + + clear_log_messages(); + + done: + if (msg) + tor_free(msg); + tor_free(dflt); + tor_free(opt); + tor_free(cl); + return; +} + struct testcase_t options_tests[] = { { "validate", test_options_validate, TT_FORK, NULL, NULL }, + { "mem_dircache", test_have_enough_mem_for_dircache, TT_FORK, NULL, NULL }, END_OF_TESTCASES };

1 0

[tor/master] Authorities must set a router's V2Dir flag if it supports tunnelled reqs
by nickm＠torproject.org 21 Dec '15

21 Dec '15

commit 467d0919d26977322a9404a9f0c426ac67c475fb Author: Matthew Finkel <Matthew.Finkel(a)gmail.com> Date: Tue Oct 28 17:12:52 2014 +0000 Authorities must set a router's V2Dir flag if it supports tunnelled reqs Partial implementation of prop 237, ticket 12538 --- src/or/dirserv.c | 3 ++- src/or/or.h | 8 ++++++++ src/or/routerparse.c | 15 +++++++++++++-- src/test/test_dir.c | 17 ++++++++++------- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 39563c3..467c6e2 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1921,7 +1921,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_hs_dir?" HSDir":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", - (rs->dir_port!=0)?" V2Dir":"", + rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); /* length of "opt v \n" */ @@ -2185,6 +2185,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname)); rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; + rs->is_v2_dir = ri->supports_tunnelled_dir_requests; if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { diff --git a/src/or/or.h b/src/or/or.h index e621fe9..fe59124 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2147,6 +2147,11 @@ typedef struct { * tests for it. */ unsigned int needs_retest_if_added:1; + /** True iff this router included "tunnelled-dir-server" in its descriptor, + * implies it accepts tunnelled directory requests, or it advertised + * dir_port > 0. */ + unsigned int supports_tunnelled_dir_requests:1; + /** Tor can use this router for general positions in circuits; we got it * from a directory server as usual, or we're an authority and a server * uploaded it. */ @@ -2224,6 +2229,9 @@ typedef struct routerstatus_t { * an exit node. */ unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden * service directory. */ + unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort + * or it claims to accept tunnelled dir requests. + */ /** True iff we know version info for this router. (i.e., a "v" entry was * included.) We'll replace all these with a big tor_version_t or a char[] * if the number of traits we care about ever becomes incredibly big. */ diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 3f794ad..fafba96 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -35,8 +35,9 @@ /****************************************************************************/ /** Enumeration of possible token types. The ones starting with K_ correspond - * to directory 'keywords'. ERR_ is an error in the tokenizing process, EOF_ - * is an end-of-file marker, and NIL_ is used to encode not-a-token. + * to directory 'keywords'. A_ is for an annotation, R or C is related to + * hidden services, ERR_ is an error in the tokenizing process, EOF_ is an + * end-of-file marker, and NIL_ is used to encode not-a-token. */ typedef enum { K_ACCEPT = 0, @@ -125,6 +126,7 @@ typedef enum { K_DIR_KEY_CERTIFICATION, K_DIR_KEY_CROSSCERT, K_DIR_ADDRESS, + K_DIR_TUNNELLED, K_VOTE_STATUS, K_VALID_AFTER, @@ -318,6 +320,7 @@ static token_rule_t routerdesc_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T1( "bandwidth", K_BANDWIDTH, GE(3), NO_OBJ ), A01("@purpose", A_PURPOSE, GE(1), NO_OBJ ), + T01("tunnelled-dir-server",K_DIR_TUNNELLED, NO_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -1609,6 +1612,12 @@ router_parse_entry_from_string(const char *s, const char *end, router->wants_to_be_hs_dir = 1; } + /* This router accepts tunnelled directory requests via begindir if it has + * an open dirport or it included "tunnelled-dir-server". */ + if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) { + router->supports_tunnelled_dir_requests = 1; + } + tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); #ifdef COUNT_DISTINCT_DIGESTS @@ -2294,6 +2303,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_unnamed = 1; } else if (!strcmp(tok->args[i], "HSDir")) { rs->is_hs_dir = 1; + } else if (!strcmp(tok->args[i], "V2Dir")) { + rs->is_v2_dir = 1; } } } diff --git a/src/test/test_dir.c b/src/test/test_dir.c index ce639b6..ab802dc 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -1579,8 +1579,9 @@ gen_routerstatus_for_v3ns(int idx, time_t now) rs->addr = 0x99008801; rs->or_port = 443; rs->dir_port = 8000; - /* all flags but running cleared */ + /* all flags but running and v2dir cleared */ rs->is_flagged_running = 1; + rs->is_v2_dir = 1; break; case 1: /* Generate the second routerstatus. */ @@ -1598,7 +1599,7 @@ gen_routerstatus_for_v3ns(int idx, time_t now) tor_addr_copy(&rs->ipv6_addr, &addr_ipv6); rs->ipv6_orport = 4711; rs->is_exit = rs->is_stable = rs->is_fast = rs->is_flagged_running = - rs->is_valid = rs->is_possible_guard = 1; + rs->is_valid = rs->is_possible_guard = rs->is_v2_dir = 1; break; case 2: /* Generate the third routerstatus. */ @@ -1613,7 +1614,7 @@ gen_routerstatus_for_v3ns(int idx, time_t now) rs->or_port = 400; rs->dir_port = 9999; rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_valid = + rs->is_flagged_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; break; case 3: @@ -1746,11 +1747,11 @@ test_vrs_for_v3ns(vote_routerstatus_t *vrs, int voter, time_t now) tt_assert(tor_addr_eq(&rs->ipv6_addr, &addr_ipv6)); tt_int_op(rs->ipv6_orport,OP_EQ, 4711); if (voter == 1) { - /* all except "authority" (1) and "v2dir" (64) */ - tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(190)); + /* all except "authority" (1) */ + tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(254)); } else { - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) - v2dir(256) */ - tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(718)); + /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ + tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(974)); } } else if (tor_memeq(rs->identity_digest, "\x33\x33\x33\x33\x33\x33\x33\x33\x33\x33" @@ -1845,6 +1846,7 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now) tt_assert(rs->is_stable); tt_assert(rs->is_flagged_running); tt_assert(rs->is_valid); + tt_assert(rs->is_v2_dir); tt_assert(!rs->is_named); /* XXXX check version */ } else { @@ -2965,6 +2967,7 @@ test_dir_fmt_control_ns(void *arg) rs.is_fast = 1; rs.is_flagged_running = 1; rs.has_bandwidth = 1; + rs.is_v2_dir = 1; rs.bandwidth_kb = 1000; s = networkstatus_getinfo_helper_single(&rs);

1 0