[onionoo/master] Add fields parameter to reduce response size. - tor-commits

24 Jul 2013

commit 1c1dfa4867f7694339afb0b79ded21ca47e3492a
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Tue Jul 23 00:13:25 2013 +0200
Add fields parameter to reduce response size.
---
 src/org/torproject/onionoo/ResourceServlet.java |   62 +++++++++++++++++++----
 web/index.html                                  |   18 +++++++
 2 files changed, 69 insertions(+), 11 deletions(-)

diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
index 47d4a93..0487ee0 100644
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ b/src/org/torproject/onionoo/ResourceServlet.java
@@ -383,7 +383,8 @@ public class ResourceServlet extends HttpServlet {
      * parameters. */
     Set<String> knownParameters = new HashSet<String>(Arrays.asList((
         "type,running,search,lookup,country,as,flag,first_seen_days,"
-        + "last_seen_days,contact,order,limit,offset").split(",")));
+        + "last_seen_days,contact,order,limit,offset,fields").
+        split(",")));
     for (String parameterKey : parameterMap.keySet()) {
       if (!knownParameters.contains(parameterKey)) {
         response.sendError(HttpServletResponse.SC_BAD_REQUEST);
@@ -589,13 +590,24 @@ public class ResourceServlet extends HttpServlet {
       }
     }
+    /* Possibly include only a subset of fields in the response
+     * document. */
+    String[] fields = null;
+    if (parameterMap.containsKey("fields")) {
+      fields = this.parseFieldsParameter(parameterMap.get("fields"));
+      if (fields == null) {
+        response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+        return;
+      }
+    }
+
     /* Set response headers and write the response. */
     response.setHeader("Access-Control-Allow-Origin", "*");
     response.setContentType("application/json");
     response.setCharacterEncoding("utf-8");
     PrintWriter pw = response.getWriter();
-    this.writeRelays(orderedRelays, pw, resourceType);
-    this.writeBridges(orderedBridges, pw, resourceType);
+    this.writeRelays(orderedRelays, pw, resourceType, fields);
+    this.writeBridges(orderedBridges, pw, resourceType, fields);
     pw.flush();
     pw.close();
   }
@@ -694,6 +706,15 @@ public class ResourceServlet extends HttpServlet {
     return parameter.split(" ");
   }
+  private static Pattern fieldsParameterPattern =
+      Pattern.compile("^[0-9a-zA-Z_,]*$");
+  private String[] parseFieldsParameter(String parameter) {
+    if (!fieldsParameterPattern.matcher(parameter).matches()) {
+      return null;
+    }
+    return parameter.split(",");
+  }
+
   private void filterByType(Map<String, String> filteredRelays,
       Map<String, String> filteredBridges, boolean relaysRequested) {
     if (relaysRequested) {
@@ -922,7 +943,7 @@ public class ResourceServlet extends HttpServlet {
   }
private void writeRelays(List<String> relays, PrintWriter pw,
-      String resourceType) {
+      String resourceType, String[] fields) {
     pw.write("{"relays_published":"" + this.relaysPublishedString
         + "",\n"relays":[");
     int written = 0;
@@ -931,7 +952,7 @@ public class ResourceServlet extends HttpServlet {
         /* TODO This is a workaround for a bug; line shouldn't be null. */
         continue;
       }
-      String lines = this.getFromSummaryLine(line, resourceType);
+      String lines = this.getFromSummaryLine(line, resourceType, fields);
       if (lines.length() > 0) {
         pw.print((written++ > 0 ? ",\n" : "\n") + lines);
       }
@@ -940,7 +961,7 @@ public class ResourceServlet extends HttpServlet {
   }
private void writeBridges(List<String> bridges, PrintWriter pw,
-      String resourceType) {
+      String resourceType, String[] fields) {
     pw.write(""bridges_published":"" + this.bridgesPublishedString
         + "",\n"bridges":[");
     int written = 0;
@@ -949,7 +970,7 @@ public class ResourceServlet extends HttpServlet {
         /* TODO This is a workaround for a bug; line shouldn't be null. */
         continue;
       }
-      String lines = this.getFromSummaryLine(line, resourceType);
+      String lines = this.getFromSummaryLine(line, resourceType, fields);
       if (lines.length() > 0) {
         pw.print((written++ > 0 ? ",\n" : "\n") + lines);
       }
@@ -958,11 +979,11 @@ public class ResourceServlet extends HttpServlet {
   }
private String getFromSummaryLine(String summaryLine,
-      String resourceType) {
+      String resourceType, String[] fields) {
     if (resourceType.equals("summary")) {
       return this.writeSummaryLine(summaryLine);
     } else if (resourceType.equals("details")) {
-      return this.writeDetailsLines(summaryLine);
+      return this.writeDetailsLines(summaryLine, fields);
     } else if (resourceType.equals("bandwidth")) {
       return this.writeBandwidthLines(summaryLine);
     } else if (resourceType.equals("weights")) {
@@ -977,7 +998,7 @@ public class ResourceServlet extends HttpServlet {
         summaryLine.length() - 1) : summaryLine);
   }
-  private String writeDetailsLines(String summaryLine) {
+  private String writeDetailsLines(String summaryLine, String[] fields) {
     String fingerprint = null;
     if (summaryLine.contains(""f":"")) {
       fingerprint = summaryLine.substring(summaryLine.indexOf(
@@ -1000,12 +1021,27 @@ public class ResourceServlet extends HttpServlet {
         /* Skip version line. */
         s.nextLine();
       }
+      boolean includeLine = true;
       while (s.hasNextLine()) {
         String line = s.nextLine();
         if (line.equals("}")) {
           sb.append("}\n");
           break;
-        } else if (!line.startsWith(""desc_published":")) {
+        } else if (line.startsWith(""desc_published":")) {
+          continue;
+        } else if (fields != null) {
+          if (line.startsWith(""")) {
+            includeLine = false;
+            for (String field : fields) {
+              if (line.startsWith(""" + field + "":")) {
+                sb.append(line + "\n");
+                includeLine = true;
+              }
+            }
+          } else if (includeLine) {
+            sb.append(line + "\n");
+          }
+        } else {
           sb.append(line + "\n");
         }
       }
@@ -1015,6 +1051,10 @@ public class ResourceServlet extends HttpServlet {
         detailsLines = detailsLines.substring(0,
             detailsLines.length() - 1);
       }
+      if (detailsLines.endsWith(",\n}")) {
+        detailsLines = detailsLines.substring(0,
+            detailsLines.length() - 3) + "\n}";
+      }
       return detailsLines;
     } else {
       // TODO We should probably log that we didn't find a details
diff --git a/web/index.html b/web/index.html
index 2655975..abc124e 100755
--- a/web/index.html
+++ b/web/index.html
@@ -699,6 +699,24 @@ Comparisons are case-insensitive.
 <font color="blue">Added on July 19, 2013.</font>
 </td></tr>
 </table>
+<p>Response documents can be reduced in size by requesting only a subset
+of contained fields.</p>
+<table border="0" cellpadding="4" cellspacing="0" summary="">
+<colgroup>
+<col width="150">
+<col width="850">
+</colgroup>
+<tr><td><b><font color="blue">fields</font></b></td><td>Comma-separated
+list of fields that will be
+included in the result.
+So far, only top-level fields in relay or bridge objects of details
+documents can be specified, e.g., <b>nickname,hashed_fingerprint</b>.
+If the fields parameter is provided, all other fields which are not
+contained in the provided list will be removed from the result.
+Field names are case-insensitive.
+<font color="blue">Added on July 23, 2013.</font>
+</td></tr>
+</table>
 <p>Relay and/or bridge documents in the response can be ordered and
 limited by providing further parameters.
 If the same parameter is specified more than once, only the first

    