[metrics-lib/master] Avoid parsing descriptor contents to Maps. - tor-commits

18 Jun 2014

commit 73e5a6989df19923775978428cd9bb21e0a96dc4
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Wed Jun 18 11:38:07 2014 +0200
Avoid parsing descriptor contents to Maps.
Extra-info descriptors contain lots of comma-separated key=value lists
    that we store in SortedMap instances.  But those occupy a lot of memory,
    and it's not certain that we'll ever want to use the contained keys or
    values.
New approach: when parsing a descriptor, use regular expressions to check
    if lines are valid, and delay parsing into maps until needed.
---
 .../descriptor/impl/ExtraInfoDescriptorImpl.java   |   90 ++++++++++----------
 .../torproject/descriptor/impl/ParseHelper.java    |   59 ++++++++-----
 2 files changed, 81 insertions(+), 68 deletions(-)

diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
index 13fdfa8..836551a 100644
--- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
+++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
@@ -712,28 +712,28 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
     return this.dirreqStatsIntervalLength;
   }
-  private SortedMap<String, Integer> dirreqV2Ips;
+  private String dirreqV2Ips;
   public SortedMap<String, Integer> getDirreqV2Ips() {
-    return this.dirreqV2Ips == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV2Ips);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV2Ips);
   }
-  private SortedMap<String, Integer> dirreqV3Ips;
+  private String dirreqV3Ips;
   public SortedMap<String, Integer> getDirreqV3Ips() {
-    return this.dirreqV3Ips == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV3Ips);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV3Ips);
   }
-  private SortedMap<String, Integer> dirreqV2Reqs;
+  private String dirreqV2Reqs;
   public SortedMap<String, Integer> getDirreqV2Reqs() {
-    return this.dirreqV2Reqs == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV2Reqs);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV2Reqs);
   }
-  private SortedMap<String, Integer> dirreqV3Reqs;
+  private String dirreqV3Reqs;
   public SortedMap<String, Integer> getDirreqV3Reqs() {
-    return this.dirreqV3Reqs == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV3Reqs);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV3Reqs);
   }
private double dirreqV2Share = -1.0;
@@ -746,40 +746,40 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
     return this.dirreqV3Share;
   }
-  private SortedMap<String, Integer> dirreqV2Resp;
+  private String dirreqV2Resp;
   public SortedMap<String, Integer> getDirreqV2Resp() {
-    return this.dirreqV2Resp == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV2Resp);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV2Resp);
   }
-  private SortedMap<String, Integer> dirreqV3Resp;
+  private String dirreqV3Resp;
   public SortedMap<String, Integer> getDirreqV3Resp() {
-    return this.dirreqV3Resp == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV3Resp);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV3Resp);
   }
-  private SortedMap<String, Integer> dirreqV2DirectDl;
+  private String dirreqV2DirectDl;
   public SortedMap<String, Integer> getDirreqV2DirectDl() {
-    return this.dirreqV2DirectDl == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV2DirectDl);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV2DirectDl);
   }
-  private SortedMap<String, Integer> dirreqV3DirectDl;
+  private String dirreqV3DirectDl;
   public SortedMap<String, Integer> getDirreqV3DirectDl() {
-    return this.dirreqV3DirectDl == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV3DirectDl);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV3DirectDl);
   }
-  private SortedMap<String, Integer> dirreqV2TunneledDl;
+  private String dirreqV2TunneledDl;
   public SortedMap<String, Integer> getDirreqV2TunneledDl() {
-    return this.dirreqV2TunneledDl == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV2TunneledDl);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV2TunneledDl);
   }
-  private SortedMap<String, Integer> dirreqV3TunneledDl;
+  private String dirreqV3TunneledDl;
   public SortedMap<String, Integer> getDirreqV3TunneledDl() {
-    return this.dirreqV3TunneledDl == null ? null :
-        new TreeMap<String, Integer>(this.dirreqV3TunneledDl);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.dirreqV3TunneledDl);
   }
private BandwidthHistory dirreqReadHistory;
@@ -802,10 +802,10 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
     return this.entryStatsIntervalLength;
   }
-  private SortedMap<String, Integer> entryIps;
+  private String entryIps;
   public SortedMap<String, Integer> getEntryIps() {
-    return this.entryIps == null ? null :
-        new TreeMap<String, Integer>(this.entryIps);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.entryIps);
   }
private long cellStatsEndMillis = -1L;
@@ -904,10 +904,10 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
     return this.geoipStartTimeMillis;
   }
-  private SortedMap<String, Integer> geoipClientOrigins;
+  private String geoipClientOrigins;
   public SortedMap<String, Integer> getGeoipClientOrigins() {
-    return this.geoipClientOrigins == null ? null :
-        new TreeMap<String, Integer>(this.geoipClientOrigins);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.geoipClientOrigins);
   }
private long bridgeStatsEndMillis = -1L;
@@ -920,22 +920,22 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl
     return this.bridgeStatsIntervalLength;
   }
-  private SortedMap<String, Integer> bridgeIps;
+  private String bridgeIps;
   public SortedMap<String, Integer> getBridgeIps() {
-    return this.bridgeIps == null ? null :
-        new TreeMap<String, Integer>(this.bridgeIps);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.bridgeIps);
   }
-  private SortedMap<String, Integer> bridgeIpVersions;
+  private String bridgeIpVersions;
   public SortedMap<String, Integer> getBridgeIpVersions() {
-    return this.bridgeIpVersions == null ? null :
-        new TreeMap<String, Integer>(this.bridgeIpVersions);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.bridgeIpVersions);
   }
-  private SortedMap<String, Integer> bridgeIpTransports;
+  private String bridgeIpTransports;
   public SortedMap<String, Integer> getBridgeIpTransports() {
-    return this.bridgeIpTransports == null ? null :
-      new TreeMap<String, Integer>(this.bridgeIpTransports);
+    return ParseHelper.convertCommaSeparatedKeyIntegerValueList(
+        this.bridgeIpTransports);
   }
private List<String> transports = new ArrayList<String>();
diff --git a/src/org/torproject/descriptor/impl/ParseHelper.java b/src/org/torproject/descriptor/impl/ParseHelper.java
index 4fbe34a..048225c 100644
--- a/src/org/torproject/descriptor/impl/ParseHelper.java
+++ b/src/org/torproject/descriptor/impl/ParseHelper.java
@@ -286,11 +286,13 @@ public class ParseHelper {
         toUpperCase();
   }
-  public static SortedMap<String, Integer>
-      parseCommaSeparatedKeyIntegerValueList(String line,
-      String[] partsNoOpt, int index, int keyLength)
+  private static Map<Integer, Pattern>
+      commaSeparatedKeyValueListPatterns =
+      new HashMap<Integer, Pattern>();
+  public static String parseCommaSeparatedKeyIntegerValueList(
+      String line, String[] partsNoOpt, int index, int keyLength)
       throws DescriptorParseException {
-    SortedMap<String, Integer> result = new TreeMap<String, Integer>();
+    String result = "";
     if (partsNoOpt.length < index) {
       throw new DescriptorParseException("Line '" + line + "' does not "
           + "contain a key-value list at index " + index + ".");
@@ -299,26 +301,37 @@ public class ParseHelper {
           + "unrecognized values beyond the expected key-value list at "
           + "index " + index + ".");
     } else if (partsNoOpt.length > index) {
-      String[] listElements = partsNoOpt[index].split(",", -1);
-      for (String listElement : listElements) {
-        String[] keyAndValue = listElement.split("=");
-        String key = null;
-        int value = -1;
-        if (keyAndValue.length == 2 && (keyLength == 0 ||
-            keyAndValue[0].length() == keyLength)) {
-          try {
-            value = Integer.parseInt(keyAndValue[1]);
-            key = keyAndValue[0];
-          } catch (NumberFormatException e) {
-            /* Handle below. */
-          }
-        }
-        if (key == null) {
-          throw new DescriptorParseException("Line '" + line + "' "
-              + "contains an illegal key or value in list element '"
-              + listElement + "'.");
+      if (!commaSeparatedKeyValueListPatterns.containsKey(keyLength)) {
+        String keyPattern = "[0-9a-zA-Z?<>-]"
+            + (keyLength == 0 ? "+" : "{" + keyLength + "}");
+        String valuePattern = "\-?[0-9]{1,9}";
+        String patternString = String.format("^%s=%s(,%s=%s)*$",
+            keyPattern, valuePattern, keyPattern, valuePattern);
+        commaSeparatedKeyValueListPatterns.put(keyLength,
+            Pattern.compile(patternString));
+      }
+      Pattern pattern = commaSeparatedKeyValueListPatterns.get(
+          keyLength);
+      if (pattern.matcher(partsNoOpt[index]).matches()) {
+        result = partsNoOpt[index];
+      } else {
+        throw new DescriptorParseException("Line '" + line + "' "
+            + "contains an illegal key or value.");
+      }
+    }
+    return result;
+  }
+
+  public static SortedMap<String, Integer>
+      convertCommaSeparatedKeyIntegerValueList(String validatedString) {
+    SortedMap<String, Integer> result = null;
+    if (validatedString != null) {
+      result = new TreeMap<String, Integer>();
+      if (validatedString.contains("=")) {
+        for (String listElement : validatedString.split(",", -1)) {
+          String[] keyAndValue = listElement.split("=");
+          result.put(keyAndValue[0], Integer.parseInt(keyAndValue[1]));
         }
-        result.put(key, value);
       }
     }
     return result;

    