commit 60a066a0b08bbe621ed5ce9d0184ac168435bfa4 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Jul 8 12:53:53 2013 +0200 Fast exits read/write more than MAX_INT KiB per day. For example, see "other" entry in: exit-kibibytes-read 80=505190490,182=25102395,443=61873906, 6881=47999666,8989=8657674,17173=7910494,21762=9138992, 45682=5154543,50500=6086469,51413=62394452,other=2282907805 --- .../torproject/descriptor/ExtraInfoDescriptor.java | 6 +- .../descriptor/impl/ExtraInfoDescriptorImpl.java | 101 +++++++++++--------- .../torproject/descriptor/impl/ParseHelper.java | 43 ++++++++- .../impl/ExtraInfoDescriptorImplTest.java | 15 ++- 4 files changed, 111 insertions(+), 54 deletions(-) diff --git a/src/org/torproject/descriptor/ExtraInfoDescriptor.java b/src/org/torproject/descriptor/ExtraInfoDescriptor.java index 1478a4a..d19c590 100644 --- a/src/org/torproject/descriptor/ExtraInfoDescriptor.java +++ b/src/org/torproject/descriptor/ExtraInfoDescriptor.java @@ -205,18 +205,18 @@ public interface ExtraInfoDescriptor extends Descriptor { /* Return statistics on KiB written by port with map keys being ports * (or "other") and map values being KiB rounded up to the next full * KiB, or null if no exit statistics are included. */ - public SortedMap<String, Integer> getExitKibibytesWritten(); + public SortedMap<String, Long> getExitKibibytesWritten(); /* Return statistics on KiB read by port with map keys being ports (or * "other") and map values being KiB rounded up to the next full KiB, or * null if no exit statistics are included. */ - public SortedMap<String, Integer> getExitKibibytesRead(); + public SortedMap<String, Long> getExitKibibytesRead(); /* Return statistics on opened exit streams with map keys being ports * (or "other") and map values being the number of opened streams, * rounded up to the nearest multiple of 4, or null if no exit * statistics are included. */ - public SortedMap<String, Integer> getExitStreamsOpened(); + public SortedMap<String, Long> getExitStreamsOpened(); /* Return the start of the included geoip statistics, or -1 if no geoip * statistics are included. */ diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index f5bf8ed..c67a040 100644 --- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -258,8 +258,9 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseGeoipClientOriginsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.geoipClientOrigins = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 2); + this.geoipClientOrigins = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, + partsNoOpt, 1, 2); } private void parseDirreqStatsEndLine(String line, String lineNoOpt, @@ -290,25 +291,27 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseDirreqV2IpsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV2Ips = ParseHelper.parseCommaSeparatedKeyValueList(line, - partsNoOpt, 1, 2); + this.dirreqV2Ips = ParseHelper.parseCommaSeparatedKeyIntegerValueList( + line, partsNoOpt, 1, 2); } private void parseDirreqV3IpsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV3Ips = ParseHelper.parseCommaSeparatedKeyValueList(line, - partsNoOpt, 1, 2); + this.dirreqV3Ips = ParseHelper.parseCommaSeparatedKeyIntegerValueList( + line, partsNoOpt, 1, 2); } private void parseDirreqV2ReqsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV2Reqs = ParseHelper.parseCommaSeparatedKeyValueList(line, + this.dirreqV2Reqs = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, partsNoOpt, 1, 2); } private void parseDirreqV3ReqsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV3Reqs = ParseHelper.parseCommaSeparatedKeyValueList(line, + this.dirreqV3Reqs = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, partsNoOpt, 1, 2); } @@ -343,38 +346,44 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseDirreqV2RespLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV2Resp = ParseHelper.parseCommaSeparatedKeyValueList(line, + this.dirreqV2Resp = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, partsNoOpt, 1, 0); } private void parseDirreqV3RespLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV3Resp = ParseHelper.parseCommaSeparatedKeyValueList(line, + this.dirreqV3Resp = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, partsNoOpt, 1, 0); } private void parseDirreqV2DirectDlLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV2DirectDl = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 0); + this.dirreqV2DirectDl = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, + partsNoOpt, 1, 0); } private void parseDirreqV3DirectDlLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV3DirectDl = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 0); + this.dirreqV3DirectDl = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, + partsNoOpt, 1, 0); } private void parseDirreqV2TunneledDlLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV2TunneledDl = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 0); + this.dirreqV2TunneledDl = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, + partsNoOpt, 1, 0); } private void parseDirreqV3TunneledDlLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.dirreqV3TunneledDl = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 0); + this.dirreqV3TunneledDl = + ParseHelper.parseCommaSeparatedKeyIntegerValueList( + line,partsNoOpt, 1, 0); } private void parseDirreqReadHistoryLine(String line, String lineNoOpt, @@ -399,8 +408,8 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseEntryIpsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.entryIps = ParseHelper.parseCommaSeparatedKeyValueList(line, - partsNoOpt, 1, 2); + this.entryIps = ParseHelper.parseCommaSeparatedKeyIntegerValueList( + line, partsNoOpt, 1, 2); } private void parseCellStatsEndLine(String line, String lineNoOpt, @@ -488,7 +497,7 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { this.exitKibibytesWritten = this.sortByPorts(ParseHelper. - parseCommaSeparatedKeyValueList(line, partsNoOpt, 1, 0)); + parseCommaSeparatedKeyLongValueList(line, partsNoOpt, 1, 0)); this.verifyPorts(line, this.exitKibibytesWritten.keySet()); this.verifyBytesOrStreams(line, this.exitKibibytesWritten.values()); } @@ -496,7 +505,7 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseExitKibibytesReadLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { this.exitKibibytesRead = this.sortByPorts(ParseHelper. - parseCommaSeparatedKeyValueList(line, partsNoOpt, 1, 0)); + parseCommaSeparatedKeyLongValueList(line, partsNoOpt, 1, 0)); this.verifyPorts(line, this.exitKibibytesRead.keySet()); this.verifyBytesOrStreams(line, this.exitKibibytesRead.values()); } @@ -504,15 +513,15 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseExitStreamsOpenedLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { this.exitStreamsOpened = this.sortByPorts(ParseHelper. - parseCommaSeparatedKeyValueList(line, partsNoOpt, 1, 0)); + parseCommaSeparatedKeyLongValueList(line, partsNoOpt, 1, 0)); this.verifyPorts(line, this.exitStreamsOpened.keySet()); this.verifyBytesOrStreams(line, this.exitStreamsOpened.values()); } - private SortedMap<String, Integer> sortByPorts( - SortedMap<String, Integer> naturalOrder) { - SortedMap<String, Integer> byPortNumber = - new TreeMap<String, Integer>(new Comparator<String>() { + private SortedMap<String, Long> sortByPorts( + SortedMap<String, Long> naturalOrder) { + SortedMap<String, Long> byPortNumber = + new TreeMap<String, Long>(new Comparator<String>() { public int compare(String arg0, String arg1) { int port0 = 0, port1 = 0; try { @@ -557,11 +566,10 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl } private void verifyBytesOrStreams(String line, - Collection<Integer> bytesOrStreams) - throws DescriptorParseException { + Collection<Long> bytesOrStreams) throws DescriptorParseException { boolean valid = true; - for (int s : bytesOrStreams) { - if (s < 0) { + for (long s : bytesOrStreams) { + if (s < 0L) { valid = false; break; } @@ -582,20 +590,23 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl private void parseBridgeStatsIpsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.bridgeIps = ParseHelper.parseCommaSeparatedKeyValueList(line, + this.bridgeIps = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, partsNoOpt, 1, 2); } private void parseBridgeIpVersionsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.bridgeIpVersions = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 2); + this.bridgeIpVersions = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, + partsNoOpt, 1, 2); } private void parseBridgeIpTransportsLine(String line, String lineNoOpt, String[] partsNoOpt) throws DescriptorParseException { - this.bridgeIpTransports = ParseHelper.parseCommaSeparatedKeyValueList( - line, partsNoOpt, 1, 0); + this.bridgeIpTransports = + ParseHelper.parseCommaSeparatedKeyIntegerValueList(line, + partsNoOpt, 1, 0); } private void parseTransportLine(String line, String lineNoOpt, @@ -869,22 +880,22 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl return this.exitStatsIntervalLength; } - private SortedMap<String, Integer> exitKibibytesWritten; - public SortedMap<String, Integer> getExitKibibytesWritten() { + private SortedMap<String, Long> exitKibibytesWritten; + public SortedMap<String, Long> getExitKibibytesWritten() { return this.exitKibibytesWritten == null ? null : - new TreeMap<String, Integer>(this.exitKibibytesWritten); + new TreeMap<String, Long>(this.exitKibibytesWritten); } - private SortedMap<String, Integer> exitKibibytesRead; - public SortedMap<String, Integer> getExitKibibytesRead() { + private SortedMap<String, Long> exitKibibytesRead; + public SortedMap<String, Long> getExitKibibytesRead() { return this.exitKibibytesRead == null ? null : - new TreeMap<String, Integer>(this.exitKibibytesRead); + new TreeMap<String, Long>(this.exitKibibytesRead); } - private SortedMap<String, Integer> exitStreamsOpened; - public SortedMap<String, Integer> getExitStreamsOpened() { + private SortedMap<String, Long> exitStreamsOpened; + public SortedMap<String, Long> getExitStreamsOpened() { return this.exitStreamsOpened == null ? null : - new TreeMap<String, Integer>(this.exitStreamsOpened); + new TreeMap<String, Long>(this.exitStreamsOpened); } private long geoipStartTimeMillis = -1L; diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java index 4fd776e..ebb8bc6 100644 --- a/src/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/org/torproject/descriptor/impl/ParseHelper.java @@ -253,8 +253,9 @@ public class ParseHelper { } public static SortedMap<String, Integer> - parseCommaSeparatedKeyValueList(String line, String[] partsNoOpt, - int index, int keyLength) throws DescriptorParseException { + parseCommaSeparatedKeyIntegerValueList(String line, + String[] partsNoOpt, int index, int keyLength) + throws DescriptorParseException { SortedMap<String, Integer> result = new TreeMap<String, Integer>(); if (partsNoOpt.length < index) { throw new DescriptorParseException("Line '" + line + "' does not " @@ -289,6 +290,44 @@ public class ParseHelper { return result; } + public static SortedMap<String, Long> + parseCommaSeparatedKeyLongValueList(String line, + String[] partsNoOpt, int index, int keyLength) + throws DescriptorParseException { + SortedMap<String, Long> result = new TreeMap<String, Long>(); + if (partsNoOpt.length < index) { + throw new DescriptorParseException("Line '" + line + "' does not " + + "contain a key-value list at index " + index + "."); + } else if (partsNoOpt.length > index + 1 ) { + throw new DescriptorParseException("Line '" + line + "' contains " + + "unrecognized values beyond the expected key-value list at " + + "index " + index + "."); + } else if (partsNoOpt.length > index) { + String[] listElements = partsNoOpt[index].split(",", -1); + for (String listElement : listElements) { + String[] keyAndValue = listElement.split("="); + String key = null; + long value = -1; + if (keyAndValue.length == 2 && (keyLength == 0 || + keyAndValue[0].length() == keyLength)) { + try { + value = Long.parseLong(keyAndValue[1]); + key = keyAndValue[0]; + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (key == null) { + throw new DescriptorParseException("Line '" + line + "' " + + "contains an illegal key or value in list element '" + + listElement + "'."); + } + result.put(key, value); + } + } + return result; + } + public static List<Integer> parseCommaSeparatedIntegerValueList( String line, String[] partsNoOpt, int index) throws DescriptorParseException { diff --git a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index 11b07b2..4aa8a44 100644 --- a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -1234,7 +1234,7 @@ public class ExtraInfoDescriptorImplTest { int[] writtenValues = new int[] { 74647, 31370, 20577, 23, 12, 1111, 4, 11, 6, 3365, 2592 }; int i = 0; - for (Map.Entry<String, Integer> e : + for (Map.Entry<String, Long> e : descriptor.getExitKibibytesWritten().entrySet()) { assertEquals(ports[i], e.getKey()); assertEquals(writtenValues[i++], e.getValue().intValue()); @@ -1242,7 +1242,7 @@ public class ExtraInfoDescriptorImplTest { int[] readValues = new int[] { 35562, 1254256, 110279, 9396, 1911, 648, 1188, 1427, 1824, 14, 3054 }; i = 0; - for (Map.Entry<String, Integer> e : + for (Map.Entry<String, Long> e : descriptor.getExitKibibytesRead().entrySet()) { assertEquals(ports[i], e.getKey()); assertEquals(readValues[i++], e.getValue().intValue()); @@ -1250,7 +1250,7 @@ public class ExtraInfoDescriptorImplTest { int[] streamsValues = new int[] { 369748, 64212, 151660, 4, 4, 4, 4, 4, 4, 4, 1212 }; i = 0; - for (Map.Entry<String, Integer> e : + for (Map.Entry<String, Long> e : descriptor.getExitStreamsOpened().entrySet()) { assertEquals(ports[i], e.getKey()); assertEquals(streamsValues[i++], e.getValue().intValue()); @@ -1285,7 +1285,14 @@ public class ExtraInfoDescriptorImplTest { "exit-kibibytes-read 25=-35562"); } - @Test(expected = DescriptorParseException.class) + @Test() + public void testExitStatsReadTooLarge() + throws DescriptorParseException { + ExitStatsBuilder.createWithExitKibibytesReadLine( + "exit-kibibytes-read other=2282907805"); + } + + @Test() public void testExitStatsStreamsTooLarge() throws DescriptorParseException { ExitStatsBuilder.createWithExitStreamsOpenedLine(