commit f9762314f297d184a5c6e0b0f5209815c18a29bf Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Dec 16 12:06:41 2015 +0100 Parse hidserv-stats in extra-info descriptors. This patch is loosely based on metrics-web's hidserv module. --- CHANGELOG.md | 1 + .../torproject/descriptor/ExtraInfoDescriptor.java | 32 +++++ .../descriptor/impl/ExtraInfoDescriptorImpl.java | 81 ++++++++++++ .../torproject/descriptor/impl/ParseHelper.java | 34 ++++++ .../impl/ExtraInfoDescriptorImplTest.java | 129 ++++++++++++++++++++ 5 files changed, 277 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f2bc0d..bfda75e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ extra-info descriptors, and support Ed25519 master keys in votes. - Include RSA-1024 signatures of SHA-1 digests of extra-info descriptors, which were parsed and discarded before. + - Support hidden-service statistics in extra-info descriptors. # Changes in version 1.0.0 - 2015-12-05 diff --git a/src/org/torproject/descriptor/ExtraInfoDescriptor.java b/src/org/torproject/descriptor/ExtraInfoDescriptor.java index 1b978a4..ed0141d 100644 --- a/src/org/torproject/descriptor/ExtraInfoDescriptor.java +++ b/src/org/torproject/descriptor/ExtraInfoDescriptor.java @@ -3,6 +3,7 @@ package org.torproject.descriptor; import java.util.List; +import java.util.Map; import java.util.SortedMap; /* Contains a relay or bridge extra-info descriptor. */ @@ -265,6 +266,37 @@ public interface ExtraInfoDescriptor extends Descriptor { * bridge. */ public List<String> getTransports(); + /* Return the end of the included hidden-service statistics, or -1 if no + * hidden-service statistics are included. */ + public long getHidservStatsEndMillis(); + + /* Return the interval length of the included hidden-service statistics + * in seconds, or -1 if no hidden-service statistics are included. */ + public long getHidservStatsIntervalLength(); + + /* Return the approximate number of RELAY cells seen in either direction + * on a circuit after receiving and successfully processing a + * RENDEZVOUS1 cell, or null if no hidden-service statistics are + * included. */ + public Double getHidservRendRelayedCells(); + + /* Return the obfuscation parameters applied to the original measurement + * value of RELAY cells seen in either direction on a circuit after + * receiving and successfully processing a RENDEZVOUS1 cell, or null if + * no hidden-service statistics are included.. */ + public Map<String, Double> getHidservRendRelayedCellsParameters(); + + /* Return the approximate number of unique hidden-service identities + * seen in descriptors published to and accepted by this hidden-service + * directory, or null if no hidden-service statistics are included. */ + public Double getHidservDirOnionsSeen(); + + /* Return the obfuscation parameters applied to the original measurement + * value of unique hidden-service identities seen in descriptors + * published to and accepted by this hidden-service directory, or null + * if no hidden-service statistics are included. */ + public Map<String, Double> getHidservDirOnionsSeenParameters(); + /* Return the signature of the PKCS1-padded extra-info descriptor * digest, or null if the descriptor doesn't contain a signature (which * is the case in sanitized bridge descriptors). */ diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 4abace6..ef0c82c 100644 --- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -9,8 +9,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.SortedMap; @@ -165,6 +167,13 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl this.parseBridgeIpTransportsLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("transport")) { this.parseTransportLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("hidserv-stats-end")) { + this.parseHidservStatsEndLine(line, lineNoOpt, partsNoOpt); + } else if (keyword.equals("hidserv-rend-relayed-cells")) { + this.parseHidservRendRelayedCellsLine(line, lineNoOpt, + partsNoOpt); + } else if (keyword.equals("hidserv-dir-onions-seen")) { + this.parseHidservDirOnionsSeenLine(line, lineNoOpt, partsNoOpt); } else if (keyword.equals("identity-ed25519")) { this.parseIdentityEd25519Line(line, lineNoOpt, partsNoOpt); nextCrypto = "identity-ed25519"; @@ -642,6 +651,46 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl this.transports.add(partsNoOpt[1]); } + private void parseHidservStatsEndLine(String line, String lineNoOpt, + String[] partsNoOpt) throws DescriptorParseException { + long[] parsedStatsEndData = this.parseStatsEndLine(line, partsNoOpt, + 5); + this.hidservStatsEndMillis = parsedStatsEndData[0]; + this.hidservStatsIntervalLength = parsedStatsEndData[1]; + } + + private void parseHidservRendRelayedCellsLine(String line, + String lineNoOpt, String[] partsNoOpt) + throws DescriptorParseException { + if (partsNoOpt.length < 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + try { + this.hidservRendRelayedCells = Double.parseDouble(partsNoOpt[1]); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.hidservRendRelayedCellsParameters = + ParseHelper.parseSpaceSeparatedStringKeyDoubleValueMap(line, + partsNoOpt, 2); + } + + private void parseHidservDirOnionsSeenLine(String line, + String lineNoOpt, String[] partsNoOpt) + throws DescriptorParseException { + if (partsNoOpt.length < 2) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + try { + this.hidservDirOnionsSeen = Double.parseDouble(partsNoOpt[1]); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.hidservDirOnionsSeenParameters = + ParseHelper.parseSpaceSeparatedStringKeyDoubleValueMap(line, + partsNoOpt, 2); + } + private void parseRouterSignatureLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { if (!lineNoOpt.equals("router-signature")) { @@ -1057,6 +1106,38 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl return new ArrayList<String>(this.transports); } + private long hidservStatsEndMillis = -1L; + public long getHidservStatsEndMillis() { + return this.hidservStatsEndMillis; + } + + private long hidservStatsIntervalLength = -1L; + public long getHidservStatsIntervalLength() { + return this.hidservStatsIntervalLength; + } + + private Double hidservRendRelayedCells; + public Double getHidservRendRelayedCells() { + return this.hidservRendRelayedCells; + } + + private Map<String, Double> hidservRendRelayedCellsParameters; + public Map<String, Double> getHidservRendRelayedCellsParameters() { + return this.hidservRendRelayedCellsParameters == null ? null : + new HashMap<>(this.hidservRendRelayedCellsParameters); + } + + private Double hidservDirOnionsSeen; + public Double getHidservDirOnionsSeen() { + return this.hidservDirOnionsSeen; + } + + private Map<String, Double> hidservDirOnionsSeenParameters; + public Map<String, Double> getHidservDirOnionsSeenParameters() { + return this.hidservDirOnionsSeenParameters == null ? null : + new HashMap<>(this.hidservDirOnionsSeenParameters); + } + private String routerSignature; public String getRouterSignature() { return this.routerSignature; diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java index a354831..15de5ee 100644 --- a/src/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/org/torproject/descriptor/impl/ParseHelper.java @@ -6,6 +6,7 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; import java.util.SortedMap; @@ -435,6 +436,39 @@ public class ParseHelper { return result; } + protected static Map<String, Double> + parseSpaceSeparatedStringKeyDoubleValueMap(String line, + String[] partsNoOpt, int startIndex) + throws DescriptorParseException { + Map<String, Double> result = new LinkedHashMap<>(); + if (partsNoOpt.length < startIndex) { + throw new DescriptorParseException("Line '" + line + "' does not " + + "contain a key-value list starting at index " + startIndex + + "."); + } + for (int i = startIndex; i < partsNoOpt.length; i++) { + String listElement = partsNoOpt[i]; + String[] keyAndValue = listElement.split("="); + String key = null; + Double value = null; + if (keyAndValue.length == 2) { + try { + value = Double.parseDouble(keyAndValue[1]); + key = keyAndValue[0]; + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (key == null) { + throw new DescriptorParseException("Line '" + line + "' contains " + + "an illegal key or value in list element '" + listElement + + "'."); + } + result.put(key, value); + } + return result; + } + public static String parseMasterKeyEd25519FromIdentityEd25519CryptoBlock( String identityEd25519CryptoBlock) throws DescriptorParseException { diff --git a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index d70ac39..55e0578 100644 --- a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -138,6 +138,13 @@ public class ExtraInfoDescriptorImplTest { db.bridgeStatsLines = lines; return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true); } + private String hidservStatsLines = null; + private static ExtraInfoDescriptor createWithHidservStatsLines( + String lines) throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.hidservStatsLines = lines; + return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true); + } private String unrecognizedLine = null; private static ExtraInfoDescriptor createWithUnrecognizedLine( String line, boolean failUnrecognizedDescriptorLines) @@ -232,6 +239,9 @@ public class ExtraInfoDescriptorImplTest { if (this.bridgeStatsLines != null) { sb.append(this.bridgeStatsLines + "\n"); } + if (this.hidservStatsLines != null) { + sb.append(this.hidservStatsLines + "\n"); + } if (this.unrecognizedLine != null) { sb.append(this.unrecognizedLine + "\n"); } @@ -733,6 +743,62 @@ public class ExtraInfoDescriptorImplTest { } } + /* Helper class to build a set of hidserv-stats lines based on default + * data and modifications requested by test methods. */ + private static class HidservStatsBuilder { + private String hidservStatsEndLine = "hidserv-stats-end 2015-12-03 " + + "14:26:56 (86400 s)"; + private static ExtraInfoDescriptor createWithHidservStatsEndLine( + String line) throws DescriptorParseException { + HidservStatsBuilder hsb = new HidservStatsBuilder(); + hsb.hidservStatsEndLine = line; + return DescriptorBuilder.createWithHidservStatsLines( + hsb.buildHidservStatsLines()); + } + private String hidservRendRelayedCellsLine = + "hidserv-rend-relayed-cells 36474281 delta_f=2048 epsilon=0.30 " + + "bin_size=1024"; + private static ExtraInfoDescriptor + createWithHidservRendRelayedCellsLine(String line) + throws DescriptorParseException { + HidservStatsBuilder hsb = new HidservStatsBuilder(); + hsb.hidservRendRelayedCellsLine = line; + return DescriptorBuilder.createWithHidservStatsLines( + hsb.buildHidservStatsLines()); + } + private String hidservDirOnionsSeenLine = "hidserv-dir-onions-seen " + + "-3 delta_f=8 epsilon=0.30 bin_size=8"; + private static ExtraInfoDescriptor createWithHidservDirOnionsSeenLine( + String line) throws DescriptorParseException { + HidservStatsBuilder hsb = new HidservStatsBuilder(); + hsb.hidservDirOnionsSeenLine = line; + return DescriptorBuilder.createWithHidservStatsLines( + hsb.buildHidservStatsLines()); + } + private static ExtraInfoDescriptor createWithDefaultLines() + throws DescriptorParseException { + return DescriptorBuilder.createWithHidservStatsLines( + new HidservStatsBuilder().buildHidservStatsLines()); + } + private String buildHidservStatsLines() { + StringBuilder sb = new StringBuilder(); + if (this.hidservStatsEndLine != null) { + sb.append(this.hidservStatsEndLine + "\n"); + } + if (this.hidservRendRelayedCellsLine != null) { + sb.append(this.hidservRendRelayedCellsLine + "\n"); + } + if (this.hidservDirOnionsSeenLine != null) { + sb.append(this.hidservDirOnionsSeenLine + "\n"); + } + String lines = sb.toString(); + if (lines.endsWith("\n")) { + lines = lines.substring(0, lines.length() - 1); + } + return lines; + } + } + @Test() public void testSampleDescriptor() throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); @@ -1415,6 +1481,69 @@ public class ExtraInfoDescriptorImplTest { } @Test() + public void testHidservStatsValid() throws DescriptorParseException { + ExtraInfoDescriptor descriptor = HidservStatsBuilder. + createWithDefaultLines(); + assertEquals(1449152816000L, descriptor.getHidservStatsEndMillis()); + assertEquals(86400L, descriptor.getHidservStatsIntervalLength()); + assertEquals(36474281.0, descriptor.getHidservRendRelayedCells(), + 0.0001); + Map<String, Double> params = + descriptor.getHidservRendRelayedCellsParameters(); + assertTrue(params.containsKey("delta_f")); + assertEquals(2048.0, params.remove("delta_f"), 0.0001); + assertTrue(params.containsKey("epsilon")); + assertEquals(0.3, params.remove("epsilon"), 0.0001); + assertTrue(params.containsKey("bin_size")); + assertEquals(1024.0, params.remove("bin_size"), 0.0001); + assertTrue(params.isEmpty()); + assertEquals(-3.0, descriptor.getHidservDirOnionsSeen(), 0.0001); + params = descriptor.getHidservDirOnionsSeenParameters(); + assertTrue(params.containsKey("delta_f")); + assertEquals(8.0, params.remove("delta_f"), 0.0001); + assertTrue(params.containsKey("epsilon")); + assertEquals(0.3, params.remove("epsilon"), 0.0001); + assertTrue(params.containsKey("bin_size")); + assertEquals(8.0, params.remove("bin_size"), 0.0001); + assertTrue(params.isEmpty()); + } + + @Test() + public void testHidservStatsEndLineMissing() + throws DescriptorParseException { + ExtraInfoDescriptor descriptor = + HidservStatsBuilder.createWithHidservStatsEndLine(null); + assertEquals(-1L, descriptor.getHidservStatsEndMillis()); + assertEquals(-1L, descriptor.getHidservStatsIntervalLength()); + } + + @Test() + public void testHidservRendRelayedCellsNoParams() + throws DescriptorParseException { + ExtraInfoDescriptor descriptor = + HidservStatsBuilder.createWithHidservRendRelayedCellsLine( + "hidserv-rend-relayed-cells 36474281"); + assertEquals(36474281.0, descriptor.getHidservRendRelayedCells(), + 0.0001); + assertTrue( + descriptor.getHidservRendRelayedCellsParameters().isEmpty()); + } + + @Test(expected = DescriptorParseException.class) + public void testHidservDirOnionsSeenCommaSeparatedParams() + throws DescriptorParseException { + HidservStatsBuilder.createWithHidservDirOnionsSeenLine( + "hidserv-dir-onions-seen -3 delta_f=8,epsilon=0.30,bin_size=8"); + } + + @Test(expected = DescriptorParseException.class) + public void testHidservDirOnionsSeenNoDoubleParams() + throws DescriptorParseException { + HidservStatsBuilder.createWithHidservDirOnionsSeenLine( + "hidserv-dir-onions-seen -3 delta_f=A epsilon=B bin_size=C"); + } + + @Test() public void testRouterSignatureOpt() throws DescriptorParseException { DescriptorBuilder.createWithRouterSignatureLines("opt "