commit dc946fe1c8b50386f60f29a5073046eb2c0b27ad Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Aug 13 15:01:45 2019 +0200

Add new SnowflakeStats descriptor type.

Implements #29461. --- CHANGELOG.md | 1 + .../org/torproject/descriptor/SnowflakeStats.java | 86 ++++++++++ .../descriptor/impl/DescriptorParserImpl.java | 5 + .../java/org/torproject/descriptor/impl/Key.java | 6 + .../torproject/descriptor/impl/ParseHelper.java | 21 +++ .../descriptor/impl/SnowflakeStatsImpl.java | 185 +++++++++++++++++++++ .../org/torproject/descriptor/DescriptorTest.java | 7 +- .../descriptor/impl/SnowflakeStatsImplTest.java | 137 +++++++++++++++ src/test/resources/snowflake/example_metrics.log | 12 ++ 9 files changed, 459 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index b804ceb..2fdc45b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ versions resolved by Ivy are the same as in Debian stretch with few exceptions. - Remove Cobertura from the build process. + - Add new SnowflakeStats descriptor type.

# Changes in version 2.6.2 - 2019-05-29 diff --git a/src/main/java/org/torproject/descriptor/SnowflakeStats.java b/src/main/java/org/torproject/descriptor/SnowflakeStats.java new file mode 100644 index 0000000..379a0f7 --- /dev/null +++ b/src/main/java/org/torproject/descriptor/SnowflakeStats.java @@ -0,0 +1,86 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Optional; +import java.util.SortedMap; + +/** + * Contain aggregated information about snowflake proxies and snowflake clients. + * + * @since 2.7.0 + */ +public interface SnowflakeStats extends Descriptor { + + /** + * Return the end of the included measurement interval. + * + * @return End of the included measurement interval. + * @since 2.7.0 + */ + LocalDateTime snowflakeStatsEnd(); + + /** + * Return the length of the included measurement interval. + * + * @return Length of the included measurement interval. + * @since 2.7.0 + */ + Duration snowflakeStatsIntervalLength(); + + /** + * Return a list of mappings from two-letter country codes to the number of + * unique IP addresses of snowflake proxies that have polled. + * + * @return List of mappings from two-letter country codes to the number of + * unique IP addresses of snowflake proxies that have polled. + * @since 2.7.0 + */ + Optional<SortedMap<String, Long>> snowflakeIps(); + + /** + * Return a count of the total number of unique IP addresses of snowflake + * proxies that have polled. + * + * @return Count of the total number of unique IP addresses of snowflake + * proxies that have polled. + * @since 2.7.0 + */ + Optional<Long> snowflakeIpsTotal(); + + /** + * Return a count of the number of times a proxy has polled but received no + * client offer, rounded up to the nearest multiple of 8. + * + * @return Count of the number of times a proxy has polled but received no + * client offer, rounded up to the nearest multiple of 8. + * @since 2.7.0 + */ + Optional<Long> snowflakeIdleCount(); + + /** + * Return a count of the number of times a client has requested a proxy from + * the broker but no proxies were available, rounded up to the nearest + * multiple of 8. + * + * @return Count of the number of times a client has requested a proxy from + * the broker but no proxies were available, rounded up to the nearest + * multiple of 8. + * @since 2.7.0 + */ + Optional<Long> clientDeniedCount(); + + /** + * Return a count of the number of times a client successfully received a + * proxy from the broker, rounded up to the nearest multiple of 8. + * + * @return Count of the number of times a client successfully received a proxy + * from the broker, rounded up to the nearest multiple of 8. + * @since 2.7.0 + */ + Optional<Long> clientSnowflakeMatchCount(); +} + diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java index ef9da56..9b620cb 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -130,6 +130,11 @@ public class DescriptorParserImpl implements DescriptorParser { } else if (firstLines.startsWith("@type torperf 1.")) { return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes, sourceFile); + } else if (firstLines.startsWith("@type snowflake-stats 1.") + || firstLines.startsWith(Key.SNOWFLAKE_STATS_END.keyword + SP) + || firstLines.contains(NL + Key.SNOWFLAKE_STATS_END.keyword + SP)) { + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, sourceFile, + Key.SNOWFLAKE_STATS_END, SnowflakeStatsImpl.class); } else if (fileName.contains(LogDescriptorImpl.MARKER)) { return LogDescriptorImpl.parse(rawDescriptorBytes, sourceFile, fileName); } else if (firstLines.startsWith("@type bandwidth-file 1.") diff --git a/src/main/java/org/torproject/descriptor/impl/Key.java b/src/main/java/org/torproject/descriptor/impl/Key.java index f7b613f..cf6ed09 100644 --- a/src/main/java/org/torproject/descriptor/impl/Key.java +++ b/src/main/java/org/torproject/descriptor/impl/Key.java @@ -29,6 +29,8 @@ public enum Key { CELL_QUEUED_CELLS("cell-queued-cells"), CELL_STATS_END("cell-stats-end"), CELL_TIME_IN_QUEUE("cell-time-in-queue"), + CLIENT_DENIED_COUNT("client-denied-count"), + CLIENT_SNOWFLAKE_MATCH_COUNT("client-snowflake-match-count"), CLIENT_VERSIONS("client-versions"), CONN_BI_DIRECT("conn-bi-direct"), CONSENSUS_METHOD("consensus-method"), @@ -132,6 +134,10 @@ public enum Key { SHARED_RAND_PREVIOUS_VALUE("shared-rand-previous-value"), SIGNED_DIRECTORY("signed-directory"), SIGNING_KEY("signing-key"), + SNOWFLAKE_IDLE_COUNT("snowflake-idle-count"), + SNOWFLAKE_IPS("snowflake-ips"), + SNOWFLAKE_IPS_TOTAL("snowflake-ips-total"), + SNOWFLAKE_STATS_END("snowflake-stats-end"), TRANSPORT("transport"), TUNNELLED_DIR_SERVER("tunnelled-dir-server"), UPTIME("uptime"), diff --git a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java index dcb365e..ba45ff6 100644 --- a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java @@ -11,6 +11,10 @@ import org.apache.commons.codec.binary.Hex; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -101,6 +105,16 @@ public class ParseHelper { } }

+ static Duration parseDuration(String line, String secondsString) + throws DescriptorParseException { + long parsedSeconds = parseSeconds(line, secondsString); + if (parsedSeconds <= 0L) { + throw new DescriptorParseException("Duration must be positive in line '" + + line + "'."); + } + return Duration.ofSeconds(parsedSeconds); + } + protected static String parseExitPattern(String line, String exitPattern) throws DescriptorParseException { if (!exitPattern.contains(":")) { @@ -188,6 +202,13 @@ public class ParseHelper { return result; }

+ static LocalDateTime parseLocalDateTime(String line, String[] parts, + int dateIndex, int timeIndex) throws DescriptorParseException { + return LocalDateTime.ofInstant(Instant.ofEpochMilli( + parseTimestampAtIndex(line, parts, dateIndex, timeIndex)), + ZoneOffset.UTC); + } + protected static long parseDateAtIndex(String line, String[] parts, int dateIndex) throws DescriptorParseException { if (dateIndex >= parts.length) { diff --git a/src/main/java/org/torproject/descriptor/impl/SnowflakeStatsImpl.java b/src/main/java/org/torproject/descriptor/impl/SnowflakeStatsImpl.java new file mode 100644 index 0000000..2f46bbe --- /dev/null +++ b/src/main/java/org/torproject/descriptor/impl/SnowflakeStatsImpl.java @@ -0,0 +1,185 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.impl; + +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.SnowflakeStats; + +import java.io.File; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.Optional; +import java.util.Scanner; +import java.util.Set; +import java.util.SortedMap; + +public class SnowflakeStatsImpl extends DescriptorImpl + implements SnowflakeStats { + + private static final Set<Key> atMostOnce = EnumSet.of( + Key.SNOWFLAKE_IPS, Key.SNOWFLAKE_IPS_TOTAL, Key.SNOWFLAKE_IDLE_COUNT, + Key.CLIENT_DENIED_COUNT, Key.CLIENT_SNOWFLAKE_MATCH_COUNT); + + private static final Set<Key> exactlyOnce = EnumSet.of( + Key.SNOWFLAKE_STATS_END); + + SnowflakeStatsImpl(byte[] rawDescriptorBytes, int[] offsetAndLength, + File descriptorFile) throws DescriptorParseException { + super(rawDescriptorBytes, offsetAndLength, descriptorFile, false); + this.parseDescriptorBytes(); + this.checkExactlyOnceKeys(exactlyOnce); + this.checkAtMostOnceKeys(atMostOnce); + this.checkFirstKey(Key.SNOWFLAKE_STATS_END); + this.clearParsedKeys(); + } + + SnowflakeStatsImpl(byte[] rawDescriptorBytes, File descriptorFile) + throws DescriptorParseException { + this(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length }, + descriptorFile); + } + + private void parseDescriptorBytes() throws DescriptorParseException { + Scanner scanner = this.newScanner().useDelimiter(NL); + while (scanner.hasNext()) { + String line = scanner.next(); + if (line.startsWith("@")) { + continue; + } + String[] parts = line.split("[ \t]+"); + Key key = Key.get(parts[0]); + switch (key) { + case SNOWFLAKE_STATS_END: + this.parseSnowflakeStatsEnd(line, parts); + break; + case SNOWFLAKE_IPS: + this.parseSnowflakeIps(line, parts); + break; + case SNOWFLAKE_IPS_TOTAL: + this.parseSnowflakeIpsTotal(line, parts); + break; + case SNOWFLAKE_IDLE_COUNT: + this.parseSnowflakeIdleCount(line, parts); + break; + case CLIENT_DENIED_COUNT: + this.parseClientDeniedCount(line, parts); + break; + case CLIENT_SNOWFLAKE_MATCH_COUNT: + this.parseClientSnowflakeMatchCount(line, parts); + break; + case INVALID: + default: + ParseHelper.parseKeyword(line, parts[0]); + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<>(); + } + this.unrecognizedLines.add(line); + } + } + } + + private void parseSnowflakeStatsEnd(String line, String[] parts) + throws DescriptorParseException { + if (parts.length < 5 || parts[3].length() < 2 || !parts[3].startsWith("(") + || !parts[4].equals("s)")) { + throw new DescriptorParseException("Illegal line '" + line + "'."); + } + this.snowflakeStatsEnd = ParseHelper.parseLocalDateTime(line, parts, + 1, 2); + this.snowflakeStatsIntervalLength = ParseHelper.parseDuration(line, + parts[3].substring(1)); + } + + private void parseSnowflakeIps(String line, String[] parts) + throws DescriptorParseException { + this.snowflakeIps = ParseHelper.parseCommaSeparatedKeyLongValueList(line, + parts, 1, 2); + } + + private void parseSnowflakeIpsTotal(String line, String[] parts) + throws DescriptorParseException { + this.snowflakeIpsTotal = parseLong(line, parts, 1); + } + + private void parseSnowflakeIdleCount(String line, String[] parts) + throws DescriptorParseException { + this.snowflakeIdleCount = parseLong(line, parts, 1); + } + + private void parseClientDeniedCount(String line, String[] parts) + throws DescriptorParseException { + this.clientDeniedCount = parseLong(line, parts, 1); + } + + private void parseClientSnowflakeMatchCount(String line, String[] parts) + throws DescriptorParseException { + this.clientSnowflakeMatchCount = parseLong(line, parts, 1); + } + + private static Long parseLong(String line, String[] parts, int index) + throws DescriptorParseException { + if (index >= parts.length) { + throw new DescriptorParseException(String.format( + "Line '%s' does not contain a long value at index %d.", line, index)); + } + try { + return Long.parseLong(parts[index]); + } catch (NumberFormatException e) { + throw new DescriptorParseException(String.format( + "Unable to parse long value '%s' in line '%s'.", parts[index], line)); + } + } + + private LocalDateTime snowflakeStatsEnd; + + @Override + public LocalDateTime snowflakeStatsEnd() { + return this.snowflakeStatsEnd; + } + + private Duration snowflakeStatsIntervalLength; + + @Override + public Duration snowflakeStatsIntervalLength() { + return this.snowflakeStatsIntervalLength; + } + + private SortedMap<String, Long> snowflakeIps; + + @Override + public Optional<SortedMap<String, Long>> snowflakeIps() { + return Optional.ofNullable(this.snowflakeIps); + } + + private Long snowflakeIpsTotal; + + @Override + public Optional<Long> snowflakeIpsTotal() { + return Optional.ofNullable(this.snowflakeIpsTotal); + } + + private Long snowflakeIdleCount; + + @Override + public Optional<Long> snowflakeIdleCount() { + return Optional.ofNullable(this.snowflakeIdleCount); + } + + private Long clientDeniedCount; + + @Override + public Optional<Long> clientDeniedCount() { + return Optional.ofNullable(this.clientDeniedCount); + } + + private Long clientSnowflakeMatchCount; + + @Override + public Optional<Long> clientSnowflakeMatchCount() { + return Optional.ofNullable(this.clientSnowflakeMatchCount); + } +} + diff --git a/src/test/java/org/torproject/descriptor/DescriptorTest.java b/src/test/java/org/torproject/descriptor/DescriptorTest.java index 9e32ac9..719d16b 100644 --- a/src/test/java/org/torproject/descriptor/DescriptorTest.java +++ b/src/test/java/org/torproject/descriptor/DescriptorTest.java @@ -80,7 +80,12 @@ public class DescriptorTest { {"bridge/2017-07-17-17-09-00-server-descriptors", BridgeServerDescriptor.class, new String[] {"@type bridge-server-descriptor 1.2"}, - 13} + 13}, + + {"snowflake/example_metrics.log", + SnowflakeStats.class, + new String[0], + 2} }); }

diff --git a/src/test/java/org/torproject/descriptor/impl/SnowflakeStatsImplTest.java b/src/test/java/org/torproject/descriptor/impl/SnowflakeStatsImplTest.java new file mode 100644 index 0000000..6d0e50b --- /dev/null +++ b/src/test/java/org/torproject/descriptor/impl/SnowflakeStatsImplTest.java @@ -0,0 +1,137 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.SnowflakeStats; + +import org.hamcrest.Matchers; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.time.Duration; +import java.time.LocalDateTime; + +public class SnowflakeStatsImplTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + /** + * Example from example_metrics.log attached to #29461. + */ + private static final String[] exampleMetricsLog = new String[] { + "snowflake-stats-end 2019-08-07 19:52:11 (86400 s)", + "snowflake-ips VN=5,NL=26,AU=30,GT=2,NO=5,EG=3,NI=1,AT=22,FR=42,CA=44," + + "ZA=3,PL=20,RU=10,HR=1,CN=1,RO=4,??=7,TH=7,UA=5,DZ=5,HU=5,CH=15," + + "AE=1,PH=6,RS=3,BR=20,IT=8,KR=13,HK=7,GR=5,GB=41,DK=4,CZ=7,IE=4," + + "PT=7,TR=2,NP=2,BA=1,BE=2,IN=45,SE=23,CL=3,IL=3,FI=7,MX=6,CO=1," + + "PK=4,ID=9,IR=7,JO=2,CR=2,US=265,DE=92,LV=1,MY=8,AR=5,NZ=10,BG=2," + + "UY=1,TW=5,SI=3,LU=2,GE=2,BN=1,JP=15,ES=9,SG=7,EC=1", + "snowflake-ips-total 937", + "snowflake-idle-count 660976", + "client-denied-count 0", + "client-snowflake-match-count 864" }; + + @Test + public void testExampleMetricsLog() throws DescriptorParseException { + SnowflakeStats snowflakeStats = new SnowflakeStatsImpl( + new TestDescriptorBuilder(exampleMetricsLog).build(), null); + assertEquals(LocalDateTime.of(2019, 8, 7, 19, 52, 11), + snowflakeStats.snowflakeStatsEnd()); + assertEquals(Duration.ofDays(1L), + snowflakeStats.snowflakeStatsIntervalLength()); + assertTrue(snowflakeStats.snowflakeIps().isPresent()); + assertEquals(68, snowflakeStats.snowflakeIps().get().size()); + assertTrue(snowflakeStats.snowflakeIpsTotal().isPresent()); + assertEquals((Long) 937L, snowflakeStats.snowflakeIpsTotal().get()); + assertTrue(snowflakeStats.snowflakeIdleCount().isPresent()); + assertEquals((Long) 660976L, snowflakeStats.snowflakeIdleCount().get()); + assertTrue(snowflakeStats.clientDeniedCount().isPresent()); + assertEquals((Long) 0L, snowflakeStats.clientDeniedCount().get()); + assertTrue(snowflakeStats.clientSnowflakeMatchCount().isPresent()); + assertEquals((Long) 864L, snowflakeStats.clientSnowflakeMatchCount().get()); + } + + @Test + public void testMinimalSnowflakeStats() throws DescriptorParseException { + SnowflakeStats snowflakeStats = new SnowflakeStatsImpl( + new TestDescriptorBuilder(exampleMetricsLog[0]).build(), null); + assertEquals(LocalDateTime.of(2019, 8, 7, 19, 52, 11), + snowflakeStats.snowflakeStatsEnd()); + assertEquals(Duration.ofDays(1L), + snowflakeStats.snowflakeStatsIntervalLength()); + assertFalse(snowflakeStats.snowflakeIps().isPresent()); + assertFalse(snowflakeStats.snowflakeIpsTotal().isPresent()); + assertFalse(snowflakeStats.snowflakeIdleCount().isPresent()); + assertFalse(snowflakeStats.clientDeniedCount().isPresent()); + assertFalse(snowflakeStats.clientSnowflakeMatchCount().isPresent()); + } + + @Test + public void testEmptyLine() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Blank lines are not allowed.")); + new SnowflakeStatsImpl(new TestDescriptorBuilder(exampleMetricsLog) + .appendLines("") + .build(), null); + } + + @Test + public void testDuplicateLine() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "must be contained at most once.")); + new SnowflakeStatsImpl(new TestDescriptorBuilder( + exampleMetricsLog[0], exampleMetricsLog[1], exampleMetricsLog[1]) + .build(), null); + } + + @Test + public void testEmptyList() throws DescriptorParseException { + SnowflakeStats snowflakeStats = new SnowflakeStatsImpl( + new TestDescriptorBuilder(exampleMetricsLog[0], "snowflake-ips ") + .build(), null); + assertEquals(LocalDateTime.of(2019, 8, 7, 19, 52, 11), + snowflakeStats.snowflakeStatsEnd()); + assertEquals(Duration.ofDays(1L), + snowflakeStats.snowflakeStatsIntervalLength()); + assertTrue(snowflakeStats.snowflakeIps().isPresent()); + assertTrue(snowflakeStats.snowflakeIps().get().isEmpty()); + } + + @Test + public void testNoValue() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "does not contain a long value at index 1.")); + new SnowflakeStatsImpl(new TestDescriptorBuilder( + exampleMetricsLog[0], "snowflake-ips-total").build(), null); + } + + @Test + public void testNotANumber() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Unable to parse long value")); + new SnowflakeStatsImpl(new TestDescriptorBuilder( + exampleMetricsLog[0], "snowflake-ips-total NaN").build(), null); + } + + @Test + public void testNonPositiveIntervalLength() throws DescriptorParseException { + this.thrown.expect(DescriptorParseException.class); + this.thrown.expectMessage(Matchers.containsString( + "Duration must be positive")); + new SnowflakeStatsImpl(new TestDescriptorBuilder( + "snowflake-stats-end 2019-08-07 19:52:11 (0 s)").build(), null); + } +} + diff --git a/src/test/resources/snowflake/example_metrics.log b/src/test/resources/snowflake/example_metrics.log new file mode 100644 index 0000000..cfc7beb --- /dev/null +++ b/src/test/resources/snowflake/example_metrics.log @@ -0,0 +1,12 @@ +snowflake-stats-end 2019-08-07 19:52:11 (86400 s) +snowflake-ips VN=5,NL=26,AU=30,GT=2,NO=5,EG=3,NI=1,AT=22,FR=42,CA=44,ZA=3,PL=20,RU=10,HR=1,CN=1,RO=4,??=7,TH=7,UA=5,DZ=5,HU=5,CH=15,AE=1,PH=6,RS=3,BR=20,IT=8,KR=13,HK=7,GR=5,GB=41,DK=4,CZ=7,IE=4,PT=7,TR=2,NP=2,BA=1,BE=2,IN=45,SE=23,CL=3,IL=3,FI=7,MX=6,CO=1,PK=4,ID=9,IR=7,JO=2,CR=2,US=265,DE=92,LV=1,MY=8,AR=5,NZ=10,BG=2,UY=1,TW=5,SI=3,LU=2,GE=2,BN=1,JP=15,ES=9,SG=7,EC=1 +snowflake-ips-total 937 +snowflake-idle-count 660976 +client-denied-count 0 +client-snowflake-match-count 864 +snowflake-stats-end 2019-08-08 19:52:11 (86400 s) +snowflake-ips IE=7,IN=31,ES=17,MY=7,NO=7,IR=6,NL=36,ZA=2,GT=1,PK=5,US=284,SE=21,UY=1,AR=8,VN=6,RS=3,GB=37,CZ=12,NZ=7,CO=2,PH=6,RO=5,AT=24,GE=1,BE=5,EE=1,TR=5,CL=5,CA=59,PT=9,MX=76,IL=3,BG=2,BA=1,HU=9,JO=2,PL=16,GR=5,KR=13,EG=2,TW=10,ID=14,FI=9,DK=3,IT=11,TH=3,DE=118,SI=4,CH=19,UA=5,AU=32,NG=3,AE=1,RU=17,NI=1,JP=18,SD=1,LU=2,FR=80,BR=12,CR=6,CN=16,DZ=4,SG=13,NP=2,??=7,HK=12,HR=5,BN=1 +snowflake-ips-total 1178 +snowflake-idle-count 979344 +client-denied-count 0 +client-snowflake-match-count 392