[metrics-lib/master] Fast exits read/write more than MAX_INT KiB per day. - tor-commits

8 Jul 2013

commit 60a066a0b08bbe621ed5ce9d0184ac168435bfa4
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Mon Jul 8 12:53:53 2013 +0200
Fast exits read/write more than MAX_INT KiB per day.
For example, see "other" entry in:
exit-kibibytes-read 80=505190490,182=25102395,443=61873906,
        6881=47999666,8989=8657674,17173=7910494,21762=9138992,
        45682=5154543,50500=6086469,51413=62394452,other=2282907805
---
 .../torproject/descriptor/ExtraInfoDescriptor.java |    6 +-
 .../descriptor/impl/ExtraInfoDescriptorImpl.java   |  101 +++++++++++---------
 .../torproject/descriptor/impl/ParseHelper.java    |   43 ++++++++-
 .../impl/ExtraInfoDescriptorImplTest.java          |   15 ++-
 4 files changed, 111 insertions(+), 54 deletions(-)

diff --git a/src/org/torproject/descriptor/ExtraInfoDescriptor.java b/src/org/torproject/descriptor/ExtraInfoDescriptor.java
index 1478a4a..d19c590 100644
--- a/src/org/torproject/descriptor/ExtraInfoDescriptor.java
+++ b/src/org/torproject/descriptor/ExtraInfoDescriptor.java
@@ -205,18 +205,18 @@ public interface ExtraInfoDescriptor extends Descriptor {
   /* Return statistics on KiB written by port with map keys being ports
    * (or "other") and map values being KiB rounded up to the next full
    * KiB, or null if no exit statistics are included. */
-  public SortedMap<String, Integer> getExitKibibytesWritten();
+  public SortedMap<String, Long> getExitKibibytesWritten();
/* Return statistics on KiB read by port with map keys being ports (or
    * "other") and map values being KiB rounded up to the next full KiB, or
    * null if no exit statistics are included. */
-  public SortedMap<String, Integer> getExitKibibytesRead();
+  public SortedMap<String, Long> getExitKibibytesRead();
/* Return statistics on opened exit streams with map keys being ports
    * (or "other") and map values being the number of opened streams,
    * rounded up to the nearest multiple of 4, or null if no exit
    * statistics are included. */
-  public SortedMap<String, Integer> getExitStreamsOpened();
+  public SortedMap<String, Long> getExitStreamsOpened();
/* Return the start of the included geoip statistics, or -1 if no geoip
    * statistics are included. */
diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
index f5bf8ed..c67a040 100644
--- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
+++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
@@ -258,8 +258,9 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
private void parseGeoipClientOriginsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.geoipClientOrigins = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 2);
+    this.geoipClientOrigins =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
+        partsNoOpt, 1, 2);
   }
private void parseDirreqStatsEndLine(String line, String lineNoOpt,
@@ -290,25 +291,27 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
private void parseDirreqV2IpsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV2Ips = ParseHelper.parseCommaSeparatedKeyValueList(line,
-        partsNoOpt, 1, 2);
+    this.dirreqV2Ips = ParseHelper.parseCommaSeparatedKeyIntegerValueList(
+        line, partsNoOpt, 1, 2);
   }
private void parseDirreqV3IpsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV3Ips = ParseHelper.parseCommaSeparatedKeyValueList(line,
-        partsNoOpt, 1, 2);
+    this.dirreqV3Ips = ParseHelper.parseCommaSeparatedKeyIntegerValueList(
+        line, partsNoOpt, 1, 2);
   }
private void parseDirreqV2ReqsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV2Reqs = ParseHelper.parseCommaSeparatedKeyValueList(line,
+    this.dirreqV2Reqs =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
         partsNoOpt, 1, 2);
   }
private void parseDirreqV3ReqsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV3Reqs = ParseHelper.parseCommaSeparatedKeyValueList(line,
+    this.dirreqV3Reqs =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
         partsNoOpt, 1, 2);
   }
@@ -343,38 +346,44 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
private void parseDirreqV2RespLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV2Resp = ParseHelper.parseCommaSeparatedKeyValueList(line,
+    this.dirreqV2Resp =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
         partsNoOpt, 1, 0);
   }
private void parseDirreqV3RespLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV3Resp = ParseHelper.parseCommaSeparatedKeyValueList(line,
+    this.dirreqV3Resp =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
         partsNoOpt, 1, 0);
   }
private void parseDirreqV2DirectDlLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV2DirectDl = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 0);
+    this.dirreqV2DirectDl =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
+        partsNoOpt, 1, 0);
   }
private void parseDirreqV3DirectDlLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV3DirectDl = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 0);
+    this.dirreqV3DirectDl =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
+        partsNoOpt, 1, 0);
   }
private void parseDirreqV2TunneledDlLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV2TunneledDl = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 0);
+    this.dirreqV2TunneledDl =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
+        partsNoOpt, 1, 0);
   }
private void parseDirreqV3TunneledDlLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.dirreqV3TunneledDl = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 0);
+    this.dirreqV3TunneledDl =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(
+        line,partsNoOpt, 1, 0);
   }
private void parseDirreqReadHistoryLine(String line, String lineNoOpt,
@@ -399,8 +408,8 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
private void parseEntryIpsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.entryIps = ParseHelper.parseCommaSeparatedKeyValueList(line,
-        partsNoOpt, 1, 2);
+    this.entryIps = ParseHelper.parseCommaSeparatedKeyIntegerValueList(
+        line, partsNoOpt, 1, 2);
   }
private void parseCellStatsEndLine(String line, String lineNoOpt,
@@ -488,7 +497,7 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
       String lineNoOpt, String[] partsNoOpt)
       throws DescriptorParseException {
     this.exitKibibytesWritten = this.sortByPorts(ParseHelper.
-        parseCommaSeparatedKeyValueList(line, partsNoOpt, 1, 0));
+        parseCommaSeparatedKeyLongValueList(line, partsNoOpt, 1, 0));
     this.verifyPorts(line, this.exitKibibytesWritten.keySet());
     this.verifyBytesOrStreams(line, this.exitKibibytesWritten.values());
   }
@@ -496,7 +505,7 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
   private void parseExitKibibytesReadLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
     this.exitKibibytesRead = this.sortByPorts(ParseHelper.
-        parseCommaSeparatedKeyValueList(line, partsNoOpt, 1, 0));
+        parseCommaSeparatedKeyLongValueList(line, partsNoOpt, 1, 0));
     this.verifyPorts(line, this.exitKibibytesRead.keySet());
     this.verifyBytesOrStreams(line, this.exitKibibytesRead.values());
   }
@@ -504,15 +513,15 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
   private void parseExitStreamsOpenedLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
     this.exitStreamsOpened = this.sortByPorts(ParseHelper.
-        parseCommaSeparatedKeyValueList(line, partsNoOpt, 1, 0));
+        parseCommaSeparatedKeyLongValueList(line, partsNoOpt, 1, 0));
     this.verifyPorts(line, this.exitStreamsOpened.keySet());
     this.verifyBytesOrStreams(line, this.exitStreamsOpened.values());
   }
-  private SortedMap<String, Integer> sortByPorts(
-      SortedMap<String, Integer> naturalOrder) {
-    SortedMap<String, Integer> byPortNumber =
-        new TreeMap<String, Integer>(new Comparator<String>() {
+  private SortedMap<String, Long> sortByPorts(
+      SortedMap<String, Long> naturalOrder) {
+    SortedMap<String, Long> byPortNumber =
+        new TreeMap<String, Long>(new Comparator<String>() {
           public int compare(String arg0, String arg1) {
             int port0 = 0, port1 = 0;
             try {
@@ -557,11 +566,10 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
   }
private void verifyBytesOrStreams(String line,
-      Collection<Integer> bytesOrStreams)
-      throws DescriptorParseException {
+      Collection<Long> bytesOrStreams) throws DescriptorParseException {
     boolean valid = true;
-    for (int s : bytesOrStreams) {
-      if (s < 0) {
+    for (long s : bytesOrStreams) {
+      if (s < 0L) {
         valid = false;
         break;
       }
@@ -582,20 +590,23 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
private void parseBridgeStatsIpsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.bridgeIps = ParseHelper.parseCommaSeparatedKeyValueList(line,
+    this.bridgeIps =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
         partsNoOpt, 1, 2);
   }
private void parseBridgeIpVersionsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.bridgeIpVersions = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 2);
+    this.bridgeIpVersions =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
+        partsNoOpt, 1, 2);
   }
private void parseBridgeIpTransportsLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
-    this.bridgeIpTransports = ParseHelper.parseCommaSeparatedKeyValueList(
-        line, partsNoOpt, 1, 0);
+    this.bridgeIpTransports =
+        ParseHelper.parseCommaSeparatedKeyIntegerValueList(line,
+        partsNoOpt, 1, 0);
   }
private void parseTransportLine(String line, String lineNoOpt,
@@ -869,22 +880,22 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
     return this.exitStatsIntervalLength;
   }
-  private SortedMap<String, Integer> exitKibibytesWritten;
-  public SortedMap<String, Integer> getExitKibibytesWritten() {
+  private SortedMap<String, Long> exitKibibytesWritten;
+  public SortedMap<String, Long> getExitKibibytesWritten() {
     return this.exitKibibytesWritten == null ? null :
-        new TreeMap<String, Integer>(this.exitKibibytesWritten);
+        new TreeMap<String, Long>(this.exitKibibytesWritten);
   }
-  private SortedMap<String, Integer> exitKibibytesRead;
-  public SortedMap<String, Integer> getExitKibibytesRead() {
+  private SortedMap<String, Long> exitKibibytesRead;
+  public SortedMap<String, Long> getExitKibibytesRead() {
     return this.exitKibibytesRead == null ? null :
-        new TreeMap<String, Integer>(this.exitKibibytesRead);
+        new TreeMap<String, Long>(this.exitKibibytesRead);
   }
-  private SortedMap<String, Integer> exitStreamsOpened;
-  public SortedMap<String, Integer> getExitStreamsOpened() {
+  private SortedMap<String, Long> exitStreamsOpened;
+  public SortedMap<String, Long> getExitStreamsOpened() {
     return this.exitStreamsOpened == null ? null :
-        new TreeMap<String, Integer>(this.exitStreamsOpened);
+        new TreeMap<String, Long>(this.exitStreamsOpened);
   }
private long geoipStartTimeMillis = -1L;
diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java
index 4fd776e..ebb8bc6 100644
--- a/src/org/torproject/descriptor/impl/ParseHelper.java
+++ b/src/org/torproject/descriptor/impl/ParseHelper.java
@@ -253,8 +253,9 @@ public class ParseHelper {
   }
public static SortedMap<String, Integer>
-      parseCommaSeparatedKeyValueList(String line, String[] partsNoOpt,
-      int index, int keyLength) throws DescriptorParseException {
+      parseCommaSeparatedKeyIntegerValueList(String line,
+      String[] partsNoOpt, int index, int keyLength)
+      throws DescriptorParseException {
     SortedMap<String, Integer> result = new TreeMap<String, Integer>();
     if (partsNoOpt.length < index) {
       throw new DescriptorParseException("Line '" + line + "' does not "
@@ -289,6 +290,44 @@ public class ParseHelper {
     return result;
   }
+  public static SortedMap<String, Long>
+      parseCommaSeparatedKeyLongValueList(String line,
+      String[] partsNoOpt, int index, int keyLength)
+      throws DescriptorParseException {
+    SortedMap<String, Long> result = new TreeMap<String, Long>();
+    if (partsNoOpt.length < index) {
+      throw new DescriptorParseException("Line '" + line + "' does not "
+          + "contain a key-value list at index " + index + ".");
+    } else if (partsNoOpt.length > index + 1 ) {
+      throw new DescriptorParseException("Line '" + line + "' contains "
+          + "unrecognized values beyond the expected key-value list at "
+          + "index " + index + ".");
+    } else if (partsNoOpt.length > index) {
+      String[] listElements = partsNoOpt[index].split(",", -1);
+      for (String listElement : listElements) {
+        String[] keyAndValue = listElement.split("=");
+        String key = null;
+        long value = -1;
+        if (keyAndValue.length == 2 && (keyLength == 0 ||
+            keyAndValue[0].length() == keyLength)) {
+          try {
+            value = Long.parseLong(keyAndValue[1]);
+            key = keyAndValue[0];
+          } catch (NumberFormatException e) {
+            /* Handle below. */
+          }
+        }
+        if (key == null) {
+          throw new DescriptorParseException("Line '" + line + "' "
+              + "contains an illegal key or value in list element '"
+              + listElement + "'.");
+        }
+        result.put(key, value);
+      }
+    }
+    return result;
+  }
+
   public static List<Integer> parseCommaSeparatedIntegerValueList(
       String line, String[] partsNoOpt, int index)
       throws DescriptorParseException {
diff --git a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
index 11b07b2..4aa8a44 100644
--- a/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
+++ b/test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
@@ -1234,7 +1234,7 @@ public class ExtraInfoDescriptorImplTest {
     int[] writtenValues = new int[] { 74647, 31370, 20577, 23, 12, 1111,
         4, 11, 6, 3365, 2592 };
     int i = 0;
-    for (Map.Entry<String, Integer> e :
+    for (Map.Entry<String, Long> e :
         descriptor.getExitKibibytesWritten().entrySet()) {
       assertEquals(ports[i], e.getKey());
       assertEquals(writtenValues[i++], e.getValue().intValue());
@@ -1242,7 +1242,7 @@ public class ExtraInfoDescriptorImplTest {
     int[] readValues = new int[] { 35562, 1254256, 110279, 9396, 1911,
         648, 1188, 1427, 1824, 14, 3054 };
     i = 0;
-    for (Map.Entry<String, Integer> e :
+    for (Map.Entry<String, Long> e :
         descriptor.getExitKibibytesRead().entrySet()) {
       assertEquals(ports[i], e.getKey());
       assertEquals(readValues[i++], e.getValue().intValue());
@@ -1250,7 +1250,7 @@ public class ExtraInfoDescriptorImplTest {
     int[] streamsValues = new int[] { 369748, 64212, 151660, 4, 4, 4, 4,
         4, 4, 4, 1212 };
     i = 0;
-    for (Map.Entry<String, Integer> e :
+    for (Map.Entry<String, Long> e :
         descriptor.getExitStreamsOpened().entrySet()) {
       assertEquals(ports[i], e.getKey());
       assertEquals(streamsValues[i++], e.getValue().intValue());
@@ -1285,7 +1285,14 @@ public class ExtraInfoDescriptorImplTest {
         "exit-kibibytes-read 25=-35562");
   }
-  @Test(expected = DescriptorParseException.class)
+  @Test()
+  public void testExitStatsReadTooLarge()
+      throws DescriptorParseException {
+    ExitStatsBuilder.createWithExitKibibytesReadLine(
+        "exit-kibibytes-read other=2282907805");
+  }
+
+  @Test()
   public void testExitStatsStreamsTooLarge()
       throws DescriptorParseException {
     ExitStatsBuilder.createWithExitStreamsOpenedLine(

    