[tor-commits] [metrics-web/master] Export graph data as JSON objects.

karsten at torproject.org karsten at torproject.org
Tue Mar 6 14:49:35 UTC 2012


commit b889394b3cfbbcb8a038d7a285f39c9b68c283cd
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Tue Mar 6 15:23:23 2012 +0100

    Export graph data as JSON objects.
---
 etc/web.xml                                        |   11 +
 src/org/torproject/ernie/web/GraphDataServlet.java |  238 ++++++++++++++++++++
 2 files changed, 249 insertions(+), 0 deletions(-)

diff --git a/etc/web.xml b/etc/web.xml
index 2c6752b..fea3df6 100644
--- a/etc/web.xml
+++ b/etc/web.xml
@@ -393,6 +393,17 @@
     <url-pattern>/custom-graph.html</url-pattern>
   </servlet-mapping>
 
+  <servlet>
+    <servlet-name>GraphData</servlet-name>
+    <servlet-class>
+      org.torproject.ernie.web.GraphDataServlet
+    </servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>GraphData</servlet-name>
+    <url-pattern>/graphs/*</url-pattern>
+  </servlet-mapping>
+
   <welcome-file-list>
     <welcome-file>index.html</welcome-file>
   </welcome-file-list>
diff --git a/src/org/torproject/ernie/web/GraphDataServlet.java b/src/org/torproject/ernie/web/GraphDataServlet.java
new file mode 100644
index 0000000..ec3c93b
--- /dev/null
+++ b/src/org/torproject/ernie/web/GraphDataServlet.java
@@ -0,0 +1,238 @@
+package org.torproject.ernie.web;
+
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import java.util.logging.*;
+
+import javax.servlet.*;
+import javax.servlet.http.*;
+
+/**
+ * Servlet that reads an HTTP request for a JSON-formatted graph data
+ * document, asks the RObjectGenerator to generate the CSV file behind it,
+ * converts it to a JSON object, and returns it to the client.
+ */
+public class GraphDataServlet extends HttpServlet {
+
+  private RObjectGenerator rObjectGenerator;
+
+  /* Available graph data files. */
+  private SortedMap<String, String> availableGraphDataFiles;
+
+  /* Variable columns in CSV files that are in long form, not wide. */
+  private SortedMap<String, String> variableColumns;
+
+  /* Value columns in CSV files if only specific value columns shall be
+   * included in results. */
+  private SortedMap<String, String> valueColumns;
+
+  private Logger logger;
+
+  public void init() {
+
+    /* Initialize logger. */
+    this.logger = Logger.getLogger(GraphDataServlet.class.toString());
+
+    /* Initialize map of available graph data files and corresponding CSV
+     * files. */
+    this.availableGraphDataFiles = new TreeMap<String, String>();
+    this.availableGraphDataFiles.put("relays", "networksize");
+    this.availableGraphDataFiles.put("bridges", "networksize");
+    this.availableGraphDataFiles.put("relays-by-country",
+        "relaycountries");
+    this.availableGraphDataFiles.put("relays-by-flags", "relayflags");
+    this.availableGraphDataFiles.put("relays-by-version", "versions");
+    this.availableGraphDataFiles.put("relays-by-platform", "platforms");
+    this.availableGraphDataFiles.put("relay-bandwidth", "bandwidth");
+    this.availableGraphDataFiles.put("relay-dir-bandwidth", "dirbytes");
+    this.availableGraphDataFiles.put("relay-bandwidth-history-by-flags",
+        "bwhist-flags");
+    this.availableGraphDataFiles.put("direct-users-by-country",
+        "direct-users");
+    this.availableGraphDataFiles.put("bridge-users-by-country",
+        "bridge-users");
+    this.availableGraphDataFiles.put("gettor", "gettor");
+    this.availableGraphDataFiles.put("torperf", "torperf");
+
+    /* Initialize map of graphs with specific variable columns. */
+    this.variableColumns = new TreeMap<String, String>();
+    this.variableColumns.put("relays-by-country", "country");
+    this.variableColumns.put("relay-bandwidth-history-by-flags",
+        "isexit,isguard");
+    this.variableColumns.put("torperf", "source");
+
+    /* Initialize map of graphs with specific value columns. */
+    this.valueColumns = new TreeMap<String, String>();
+    this.valueColumns.put("relays", "relays");
+    this.valueColumns.put("bridges", "bridges");
+
+    /* Get a reference to the R object generator that we need to generate
+     * CSV files. */
+    this.rObjectGenerator = (RObjectGenerator) getServletContext().
+        getAttribute("RObjectGenerator");
+  }
+
+  public void doGet(HttpServletRequest request,
+      HttpServletResponse response) throws IOException,
+      ServletException {
+
+    /* Find out which JSON file was requested and make sure we know this
+     * JSON file type. */
+    String requestedJsonFile = request.getRequestURI();
+    if (requestedJsonFile.contains("/")) {
+      requestedJsonFile = requestedJsonFile.substring(requestedJsonFile.
+          lastIndexOf("/") + 1);
+    }
+    if (!availableGraphDataFiles.containsKey(requestedJsonFile)) {
+      logger.info("Did not recognize requested .csv file from request "
+          + "URI: '" + request.getRequestURI() + "'. Responding with 404 "
+          + "Not Found.");
+      response.sendError(HttpServletResponse.SC_NOT_FOUND);
+      return;
+    }
+    String requestedCsvFile = this.availableGraphDataFiles.get(
+        requestedJsonFile);
+    logger.fine("CSV file '" + requestedCsvFile + ".csv' requested.");
+
+    /* Prepare filename and R query string. */
+    String rQuery = "export_" + requestedCsvFile.replaceAll("-", "_")
+        + "(path = '%s')";
+    String csvFilename = requestedCsvFile + ".csv";
+
+    /* Request CSV file from R object generator, which asks Rserve to
+     * generate it. */
+    String csvFileContent = this.rObjectGenerator.generateCsv(rQuery,
+        csvFilename);
+
+    /* Make sure that we have a CSV to convert into JSON. */
+    if (csvFileContent == null) {
+      response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      return;
+    }
+
+    /* Convert CSV to JSON format. */
+    String jsonString = null;
+    try {
+      BufferedReader br = new BufferedReader(new StringReader(
+          csvFileContent));
+      String line;
+      String[] columns = null;
+      int dateCol = -1;
+      SortedSet<Integer> variableCols = new TreeSet<Integer>();
+      SortedSet<Integer> valueCols = new TreeSet<Integer>();
+      if ((line = br.readLine()) != null) {
+        columns = line.split(",");
+        for (int i = 0; i < columns.length; i++) {
+          if (columns[i].equals("date")) {
+            dateCol = i;
+          } else if (this.variableColumns.containsKey(requestedJsonFile)
+              && this.variableColumns.get(requestedJsonFile).contains(
+              columns[i])) {
+            variableCols.add(i);
+          } else if (!this.valueColumns.containsKey(requestedJsonFile) ||
+              this.valueColumns.get(requestedJsonFile).contains(
+              columns[i])) {
+            valueCols.add(i);
+          }
+        }
+      }
+      if (columns == null || dateCol < 0 || valueCols.isEmpty()) {
+        response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+        return;
+      }
+      SortedMap<String, SortedSet<String>> graphs =
+          new TreeMap<String, SortedSet<String>>();
+      while ((line = br.readLine()) != null) {
+        String[] elements = line.split(",");
+        if (elements.length != columns.length) {
+          response.sendError(
+              HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+          return;
+        }
+        String date = elements[dateCol];
+        String variable = "";
+        if (!variableCols.isEmpty()) {
+          for (int variableCol : variableCols) {
+            String variableString = elements[variableCol];
+            if (variableString.equals("TRUE")) {
+              variable += columns[variableCol] + "_";
+            } else if (variableString.equals("FALSE")) {
+              variable += "not" + columns[variableCol] + "_";
+            } else {
+              variable += variableString + "_";
+            }
+          }
+        }
+        for (int valueCol : valueCols) {
+          if (elements[valueCol].equals("NA")) {
+            continue;
+          }
+          String graphName = variable + columns[valueCol];
+          if (!graphs.containsKey(graphName)) {
+            graphs.put(graphName, new TreeSet<String>());
+          }
+          String dateAndValue = date + "=" + elements[valueCol];
+          graphs.get(graphName).add(dateAndValue);
+        }
+      }
+      StringBuilder sb = new StringBuilder();
+      SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+      dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      for (Map.Entry<String, SortedSet<String>> e : graphs.entrySet()) {
+        String graphName = e.getKey();
+        SortedSet<String> datesAndValues = e.getValue();
+        if (datesAndValues.isEmpty()) {
+          continue;
+        }
+        String[] firstDateAndValue = datesAndValues.first().split("=");
+        String firstDate = firstDateAndValue[0];
+        String firstValue = firstDateAndValue[1];
+        String lastDate = datesAndValues.last().split("=")[0];
+        sb.append(",\n\"" + graphName + "\":{"
+            + "\"first\":\"" + firstDate + "\","
+            + "\"last\":\"" + lastDate + "\","
+            + "\"values\":[");
+        int written = 0;
+        String previousDate = firstDate;
+        long previousDateMillis = dateFormat.parse(firstDate).getTime();
+        for (String dateAndValue : datesAndValues) {
+          String parts[] = dateAndValue.split("=");
+          String date = parts[0];
+          long dateMillis = dateFormat.parse(date).getTime();
+          String value = parts[1];
+          while (dateMillis - 86400L * 1000L > previousDateMillis) {
+            sb.append((written++ > 0 ? "," : "") + "null");
+            previousDateMillis += 86400L * 1000L;
+            previousDate = dateFormat.format(previousDateMillis);
+          }
+          sb.append((written++ > 0 ? "," : "") + value);
+          previousDate = date;
+          previousDateMillis = dateMillis;
+        }
+        sb.append("]}");
+      }
+      br.close();
+      jsonString = "[" + sb.toString().substring(1) + "\n]";
+    } catch (IOException e) {
+      response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      return;
+    } catch (ParseException e) {
+      response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      return;
+    }
+
+    /* Make sure that we have a graph to return. */
+    if (jsonString == null) {
+      response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      return;
+    }
+
+    /* Write JSON file to response. */
+    response.setHeader("Access-Control-Allow-Origin", "*");
+    response.setContentType("application/json");
+    response.setCharacterEncoding("utf-8");
+    response.getWriter().print(jsonString);
+  }
+}
+



More information about the tor-commits mailing list