commit be6a9856d1782cea5f76253e60c047acc01daac2
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Tue Dec 10 12:15:47 2013 +0100
Remove graphs/* URLs returning graph data in JSON format.
The better data for third parties is contained in stats/*.csv files,
because it's less specific to our graphs. It's also documented on
stats.html which didn't apply to graphs/*.
---
etc/web.xml | 11 -
.../ernie/web/graphs/GraphDataServlet.java | 279 --------------------
2 files changed, 290 deletions(-)
diff --git a/etc/web.xml b/etc/web.xml
index c8e4935..992de93 100644
--- a/etc/web.xml
+++ b/etc/web.xml
@@ -263,17 +263,6 @@
<url-pattern>/consensus-health.html</url-pattern>
</servlet-mapping>
- <servlet>
- <servlet-name>GraphData</servlet-name>
- <servlet-class>
- org.torproject.ernie.web.graphs.GraphDataServlet
- </servlet-class>
- </servlet>
- <servlet-mapping>
- <servlet-name>GraphData</servlet-name>
- <url-pattern>/graphs/*</url-pattern>
- </servlet-mapping>
-
<welcome-file-list>
<welcome-file>index.html</welcome-file>
</welcome-file-list>
diff --git a/src/org/torproject/ernie/web/graphs/GraphDataServlet.java b/src/org/torproject/ernie/web/graphs/GraphDataServlet.java
deleted file mode 100644
index 0e35bdc..0000000
--- a/src/org/torproject/ernie/web/graphs/GraphDataServlet.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.web.graphs;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.logging.Logger;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-/**
- * Servlet that reads an HTTP request for a JSON-formatted graph data
- * document, asks the RObjectGenerator to generate the CSV file behind it,
- * converts it to a JSON object, and returns it to the client.
- */
-public class GraphDataServlet extends HttpServlet {
-
- private static final long serialVersionUID = 1799558498301134024L;
-
- private RObjectGenerator rObjectGenerator;
-
- /* Available graph data files. */
- private SortedMap<String, String> availableGraphDataFiles;
-
- /* Variable columns in CSV files that are in long form, not wide. */
- private SortedMap<String, String> variableColumns;
-
- /* Value columns in CSV files if only specific value columns shall be
- * included in results. */
- private SortedMap<String, String> valueColumns;
-
- private Logger logger;
-
- public void init() {
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(GraphDataServlet.class.toString());
-
- /* Initialize map of available graph data files and corresponding CSV
- * files. */
- this.availableGraphDataFiles = new TreeMap<String, String>();
- this.availableGraphDataFiles.put("relays", "networksize");
- this.availableGraphDataFiles.put("bridges", "networksize");
- this.availableGraphDataFiles.put("cloudbridges", "cloudbridges");
- this.availableGraphDataFiles.put("relays-by-country",
- "relaycountries");
- this.availableGraphDataFiles.put("relays-by-flags", "relayflags");
- this.availableGraphDataFiles.put("relays-by-version", "versions");
- this.availableGraphDataFiles.put("relays-by-platform", "platforms");
- this.availableGraphDataFiles.put("relay-bandwidth", "bandwidth");
- this.availableGraphDataFiles.put("relay-dir-bandwidth", "dirbytes");
- this.availableGraphDataFiles.put("relay-bandwidth-history-by-flags",
- "bwhist-flags");
- this.availableGraphDataFiles.put("relay-bandwidth-by-flags",
- "bandwidth-flags");
- this.availableGraphDataFiles.put("direct-users-by-country",
- "direct-users");
- this.availableGraphDataFiles.put("bridge-users-by-country",
- "bridge-users");
- this.availableGraphDataFiles.put("torperf", "torperf");
-
- /* Initialize map of graphs with specific variable columns. */
- this.variableColumns = new TreeMap<String, String>();
- this.variableColumns.put("relays-by-country", "country");
- this.variableColumns.put("relay-bandwidth-history-by-flags",
- "isexit,isguard");
- this.variableColumns.put("torperf", "source");
-
- /* Initialize map of graphs with specific value columns. */
- this.valueColumns = new TreeMap<String, String>();
- this.valueColumns.put("relays", "relays");
- this.valueColumns.put("bridges", "bridges");
-
- /* Get a reference to the R object generator that we need to generate
- * CSV files. */
- this.rObjectGenerator = (RObjectGenerator) getServletContext().
- getAttribute("RObjectGenerator");
- }
-
- public void doGet(HttpServletRequest request,
- HttpServletResponse response) throws IOException,
- ServletException {
-
- /* Check if the directory listing was requested. */
- String requestURI = request.getRequestURI();
- if (requestURI.equals("/ernie/graphs/")) {
- request.setAttribute("directory", "/graphs");
- request.setAttribute("extension", "");
- request.setAttribute("files",
- this.availableGraphDataFiles.keySet());
- request.getRequestDispatcher("/WEB-INF/dir.jsp").forward(request,
- response);
- return;
- }
-
- /* Find out which JSON file was requested and make sure we know this
- * JSON file type. */
- String requestedJsonFile = request.getRequestURI();
- if (requestedJsonFile.contains("/")) {
- requestedJsonFile = requestedJsonFile.substring(requestedJsonFile.
- lastIndexOf("/") + 1);
- }
- if (!availableGraphDataFiles.containsKey(requestedJsonFile)) {
- logger.info("Did not recognize requested .csv file from request "
- + "URI: '" + request.getRequestURI() + "'. Responding with 404 "
- + "Not Found.");
- response.sendError(HttpServletResponse.SC_NOT_FOUND);
- return;
- }
- String requestedCsvFile = this.availableGraphDataFiles.get(
- requestedJsonFile);
- logger.fine("CSV file '" + requestedCsvFile + ".csv' requested.");
-
- /* Request CSV file from R object generator, which may ask Rserve to
- * generate it. */
- RObject csvFile = this.rObjectGenerator.generateCsv(requestedCsvFile,
- true);
-
- /* Make sure that we have a CSV to convert into JSON. */
- if (csvFile == null) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- /* Look up if we converted this CSV to JSON format before. If not,
- * convert it now. */
- String jsonString;
- if (!this.lastConvertedCsvFile.containsKey(requestedJsonFile) ||
- this.lastConvertedCsvFile.get(requestedJsonFile) <
- csvFile.getLastModified()) {
- jsonString = this.convertCsvToJson(requestedJsonFile,
- new String(csvFile.getBytes()));
- this.lastConvertedCsvFile.put(requestedJsonFile,
- csvFile.getLastModified());
- this.convertedCsvFiles.put(requestedJsonFile, jsonString);
- } else {
- jsonString = this.convertedCsvFiles.get(requestedJsonFile);
- }
-
- /* Make sure we have a JSON string to return. */
- if (jsonString == null) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- /* Write JSON string to response. */
- response.setHeader("Access-Control-Allow-Origin", "*");
- response.setContentType("application/json");
- response.setCharacterEncoding("utf-8");
- response.getWriter().print(jsonString);
- }
-
- private Map<String, Long> lastConvertedCsvFile =
- new HashMap<String, Long>();
- private Map<String, String> convertedCsvFiles =
- new HashMap<String, String>();
- private String convertCsvToJson(String requestedJsonFile,
- String csvFileContent) {
- String jsonString = null;
- try {
- BufferedReader br = new BufferedReader(new StringReader(
- csvFileContent));
- String line;
- String[] columns = null;
- int dateCol = -1;
- SortedSet<Integer> variableCols = new TreeSet<Integer>();
- SortedSet<Integer> valueCols = new TreeSet<Integer>();
- if ((line = br.readLine()) != null) {
- columns = line.split(",");
- for (int i = 0; i < columns.length; i++) {
- if (columns[i].equals("date")) {
- dateCol = i;
- } else if (this.variableColumns.containsKey(requestedJsonFile)
- && this.variableColumns.get(requestedJsonFile).contains(
- columns[i])) {
- variableCols.add(i);
- } else if (!this.valueColumns.containsKey(requestedJsonFile) ||
- this.valueColumns.get(requestedJsonFile).contains(
- columns[i])) {
- valueCols.add(i);
- }
- }
- }
- if (columns == null || dateCol < 0 || valueCols.isEmpty()) {
- return null;
- }
- SortedMap<String, SortedSet<String>> graphs =
- new TreeMap<String, SortedSet<String>>();
- while ((line = br.readLine()) != null) {
- String[] elements = line.split(",");
- if (elements.length != columns.length) {
- return null;
- }
- String date = elements[dateCol];
- String variable = "";
- if (!variableCols.isEmpty()) {
- for (int variableCol : variableCols) {
- String variableString = elements[variableCol];
- if (variableString.equals("TRUE")) {
- variable += columns[variableCol] + "_";
- } else if (variableString.equals("FALSE")) {
- variable += "not" + columns[variableCol] + "_";
- } else {
- variable += variableString + "_";
- }
- }
- }
- for (int valueCol : valueCols) {
- if (elements[valueCol].equals("NA")) {
- continue;
- }
- String graphName = variable + columns[valueCol];
- if (!graphs.containsKey(graphName)) {
- graphs.put(graphName, new TreeSet<String>());
- }
- String dateAndValue = date + "=" + elements[valueCol];
- graphs.get(graphName).add(dateAndValue);
- }
- }
- StringBuilder sb = new StringBuilder();
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- for (Map.Entry<String, SortedSet<String>> e : graphs.entrySet()) {
- String graphName = e.getKey();
- SortedSet<String> datesAndValues = e.getValue();
- if (datesAndValues.isEmpty()) {
- continue;
- }
- String[] firstDateAndValue = datesAndValues.first().split("=");
- String firstDate = firstDateAndValue[0];
- String lastDate = datesAndValues.last().split("=")[0];
- sb.append(",\n\"" + graphName + "\":{"
- + "\"first\":\"" + firstDate + "\","
- + "\"last\":\"" + lastDate + "\","
- + "\"values\":[");
- int written = 0;
- String previousDate = firstDate;
- long previousDateMillis = dateFormat.parse(previousDate).
- getTime();
- for (String dateAndValue : datesAndValues) {
- String parts[] = dateAndValue.split("=");
- String date = parts[0];
- long dateMillis = dateFormat.parse(date).getTime();
- String value = parts[1];
- while (dateMillis - 86400L * 1000L > previousDateMillis) {
- sb.append((written++ > 0 ? "," : "") + "null");
- previousDateMillis += 86400L * 1000L;
- previousDate = dateFormat.format(previousDateMillis);
- }
- sb.append((written++ > 0 ? "," : "") + value);
- previousDate = date;
- previousDateMillis = dateMillis;
- }
- sb.append("]}");
- }
- br.close();
- jsonString = "[" + sb.toString().substring(1) + "\n]";
- } catch (IOException e) {
- return null;
- } catch (ParseException e) {
- return null;
- }
- return jsonString;
- }
-}
-
