[tor-commits] [metrics-web/master] Stop providing old .csv files.
karsten at torproject.org
karsten at torproject.org
Mon Apr 28 21:30:58 UTC 2014
commit 9b0f692823ea9851c0b184cb6408882fb0cb5e9a
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Mon Apr 28 23:28:14 2014 +0200
Stop providing old .csv files.
All contents are available in files listed on stats.html.
---
website/etc/web.xml | 11 -
website/rserve/csv.R | 213 --------------------
.../torproject/metrics/web/graphs/CsvServlet.java | 97 ---------
.../metrics/web/graphs/RObjectGenerator.java | 43 ----
4 files changed, 364 deletions(-)
diff --git a/website/etc/web.xml b/website/etc/web.xml
index 33ff390..b46ce5a 100644
--- a/website/etc/web.xml
+++ b/website/etc/web.xml
@@ -231,17 +231,6 @@
</servlet-mapping>
<servlet>
- <servlet-name>Csv</servlet-name>
- <servlet-class>
- org.torproject.metrics.web.graphs.CsvServlet
- </servlet-class>
- </servlet>
- <servlet-mapping>
- <servlet-name>Csv</servlet-name>
- <url-pattern>/csv/*</url-pattern>
- </servlet-mapping>
-
- <servlet>
<servlet-name>ExoneraTor</servlet-name>
<servlet-class>
org.torproject.metrics.web.status.ExoneraTorServlet
diff --git a/website/rserve/csv.R b/website/rserve/csv.R
deleted file mode 100644
index e2a2172..0000000
--- a/website/rserve/csv.R
+++ /dev/null
@@ -1,213 +0,0 @@
-options(scipen = 15)
-
-export_networksize <- function(path) {
- s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "servers.csv", sep = ""), stringsAsFactors = FALSE)
- s <- s[s$flag == '' & s$country == '' & s$version == '' &
- s$platform == '' & s$ec2bridge == '',
- c("date", "relays", "bridges")]
- write.csv(s, path, quote = FALSE, row.names = FALSE)
-}
-
-export_cloudbridges <- function(path) {
- s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "servers.csv", sep = ""), stringsAsFactors = FALSE)
- s <- s[s$flag == '' & s$country == '' & s$version == '' &
- s$platform == '' & s$ec2bridge == 't', ]
- cloudbridges <- data.frame(date = s$date, cloudbridges = s$bridges)
- write.csv(cloudbridges, path, quote = FALSE, row.names = FALSE)
-}
-
-export_relaycountries <- function(path) {
- s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "servers.csv", sep = ""), stringsAsFactors = FALSE)
- s <- s[s$flag == '' & s$country != '' & s$version == '' &
- s$platform == '' & s$ec2bridge == '',
- c("date", "country", "relays")]
- write.csv(s, path, quote = FALSE, row.names = FALSE)
-}
-
-export_versions <- function(path) {
- s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "servers.csv", sep = ""), stringsAsFactors = FALSE)
- s <- s[s$flag == '' & s$country == '' & s$version != '' &
- s$platform == '' & s$ec2bridge == '',
- c("date", "version", "relays")]
- versions <- cast(s, date ~ version, value = "relays")
- versions <- versions[order(versions$date), ]
- write.csv(versions, path, quote = FALSE, row.names = FALSE)
-}
-
-export_platforms <- function(path) {
- s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "servers.csv", sep = ""), stringsAsFactors = FALSE)
- s <- s[s$flag == '' & s$country == '' & s$version == '' &
- s$platform != '' & s$ec2bridge == '',
- c("date", "platform", "relays")]
- s <- data.frame(date = s$date,
- platform = ifelse(s$platform == 'FreeBSD', 'bsd',
- tolower(s$platform)), relays = s$relays)
- s <- cast(s, date ~ platform, value = "relays")
- platforms <- s[order(s$date), ]
- write.csv(platforms, path, quote = FALSE, row.names = FALSE)
-}
-
-export_bandwidth <- function(path) {
- b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "bandwidth.csv", sep = ""), stringsAsFactors = FALSE)
- b <- b[b$isexit == '' & b$isguard == '', ]
- b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
- bwadv = b$advbw,
- bwhist = floor((b$bwread + b$bwwrite) / 2))
- b <- b[order(b$date), ]
- write.csv(b, path, quote = FALSE, row.names = FALSE)
-}
-
-export_bwhist_flags <- function(path) {
- b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "bandwidth.csv", sep = ""), stringsAsFactors = FALSE)
- b <- b[b$isexit != '' & b$isguard != '' & !is.na(b$bwread) &
- !is.na(b$bwwrite), ]
- b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
- isexit = b$isexit == 't', isguard = b$isguard == 't',
- read = b$bwread, written = b$bwwrite)
- write.csv(b, path, quote = FALSE, row.names = FALSE)
-}
-
-export_dirbytes <- function(path) {
- b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "bandwidth.csv", sep = ""), stringsAsFactors = FALSE)
- b <- b[b$isexit == '' & b$isguard == '' & !is.na(b$dirread) &
- !is.na(b$dirwrite), ]
- b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
- dirread = b$dirread, dirwrite = b$dirwrite)
- b <- b[order(b$date), ]
- write.csv(b, path, quote = FALSE, row.names = FALSE)
-}
-
-export_relayflags <- function(path) {
- s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "servers.csv", sep = ""), stringsAsFactors = FALSE)
- s <- s[s$country == '' & s$version == '' & s$platform == '' &
- s$ec2bridge == '', ]
- s <- data.frame(date = as.Date(s$date, "%Y-%m-%d"),
- flag = ifelse(s$flag == '', 'running', tolower(s$flag)),
- relays = s$relays)
- s <- cast(s, date ~ flag, value = "relays")
- relayflags <- s[order(s$date), ]
- write.csv(relayflags, path, quote = FALSE, row.names = FALSE)
-}
-
-export_torperf <- function(path) {
- t <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "torperf.csv", sep = ""), stringsAsFactors = FALSE)
- t <- data.frame(
- source = paste(ifelse(t$source == '', 'all', t$source),
- ifelse(t$size == 50 * 1024, '50kb',
- ifelse(t$size == 1024 * 1024, '1mb', '5mb')),
- sep = '-'),
- date = as.Date(t$date, "%Y-%m-%d"),
- q1 = t$q1, md = t$md, q3 = t$q3)
- torperf <- t[order(t$source, t$date), ]
- write.csv(torperf, path, quote = FALSE, row.names = FALSE)
-}
-
-export_torperf_failures <- function(path) {
- t <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "torperf.csv", sep = ""), stringsAsFactors = FALSE)
- t <- data.frame(
- source = paste(ifelse(t$source == '', 'all', t$source),
- ifelse(t$size == 50 * 1024, '50kb',
- ifelse(t$size == 1024 * 1024, '1mb', '5mb')),
- sep = '-'),
- date = as.Date(t$date, "%Y-%m-%d"),
- timeouts = t$timeouts, failures = t$failures, requests = t$requests)
- torperf <- t[order(t$source, t$date), ]
- write.csv(torperf, path, quote = FALSE, row.names = FALSE)
-}
-
-export_connbidirect <- function(path) {
- c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "connbidirect.csv", sep = ""), stringsAsFactors = FALSE)
- write.csv(format(c, trim = TRUE, scientific = FALSE), path,
- quote = FALSE, row.names = FALSE)
-}
-
-export_bandwidth_flags <- function(path) {
- b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "bandwidth.csv", sep = ""), stringsAsFactors = FALSE)
- b <- b[b$isexit != '' & b$isguard != '', ]
- b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"),
- isexit = b$isexit == 't', isguard = b$isguard == 't',
- advbw = b$advbw,
- bwhist = floor((b$bwread + b$bwwrite) / 2))
- b <- rbind(
- data.frame(b[b$isguard == TRUE, ], flag = "guard"),
- data.frame(b[b$isexit == TRUE, ], flag = "exit"))
- b <- data.frame(date = b$date, advbw = b$advbw, bwhist = b$bwhist,
- flag = b$flag)
- b <- aggregate(list(advbw = b$advbw, bwhist = b$bwhist),
- by = list(date = b$date, flag = b$flag), FUN = sum,
- na.rm = TRUE, na.action = NULL)
- b <- melt(b, id.vars = c("date", "flag"))
- b <- data.frame(date = b$date, type = b$variable, flag = b$flag,
- value = b$value)
- b <- b[b$value > 0, ]
- write.csv(b, path, quote = FALSE, row.names = FALSE)
-}
-
-export_userstats <- function(path) {
- c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "clients.csv", sep = ""), stringsAsFactors = FALSE)
- c <- data.frame(date = c$date, node = c$node, country = c$country,
- transport = c$transport, version = c$version,
- frac = c$frac, users = c$clients)
- write.csv(format(c, trim = TRUE, scientific = FALSE), path,
- quote = FALSE, row.names = FALSE)
-}
-
-help_export_monthly_userstats <- function(path, aggr_fun) {
- c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "clients.csv", sep = ""), stringsAsFactors = FALSE)
- c <- c[c$country != '' & c$transport == '' & c$version == '', ]
- u <- data.frame(date = c$date, country = c$country, users = c$clients,
- stringsAsFactors = FALSE)
- u <- aggregate(list(users = u$users),
- by = list(date = u$date, country = u$country), sum)
- u <- aggregate(list(users = u$users),
- by = list(country = u$country,
- month = substr(u$date, 1, 7)), aggr_fun)
- u <- rbind(u, data.frame(country = "zy",
- aggregate(list(users = u$users),
- by = list(month = u$month), sum)))
- u <- cast(u, country ~ month, value = "users")
- u[u$country == "zy", "country"] <- "all"
- u[, 2:length(u)] <- floor(u[, 2:length(u)])
- write.csv(u, path, quote = FALSE, row.names = FALSE)
-}
-
-export_monthly_userstats_peak <- function(path) {
- help_export_monthly_userstats(path, max)
-}
-
-export_monthly_userstats_average <- function(path) {
- help_export_monthly_userstats(path, mean)
-}
-
-export_userstats_detector <- function(path) {
- c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/",
- "clients.csv", sep = ""), stringsAsFactors = FALSE)
- c <- c[c$country != '' & c$transport == '' & c$version == '' &
- c$node == 'relay', ]
- u <- data.frame(country = c$country, date = c$date, users = c$clients,
- stringsAsFactors = FALSE)
- u <- rbind(u, data.frame(country = "zy",
- aggregate(list(users = u$users),
- by = list(date = u$date), sum)))
- u <- data.frame(date = u$date, country = u$country,
- users = floor(u$users))
- u <- cast(u, date ~ country, value = "users")
- names(u)[names(u) == "zy"] <- "all"
- write.csv(u, path, quote = FALSE, row.names = FALSE)
-}
-
diff --git a/website/src/org/torproject/metrics/web/graphs/CsvServlet.java b/website/src/org/torproject/metrics/web/graphs/CsvServlet.java
deleted file mode 100644
index 96e5f6a..0000000
--- a/website/src/org/torproject/metrics/web/graphs/CsvServlet.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.metrics.web.graphs;
-
-import java.io.IOException;
-import java.util.SortedSet;
-import java.util.logging.Logger;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-/**
- * Servlet that reads an HTTP request for a comma-separated value file,
- * asks the GraphGenerator to generate this file, and returns it to the
- * client.
- */
-public class CsvServlet extends HttpServlet {
-
- private static final long serialVersionUID = 7501442926823719958L;
-
- private RObjectGenerator rObjectGenerator;
-
- /* Available CSV files. */
- private SortedSet<String> availableCsvFiles;
-
- private Logger logger;
-
- public void init() {
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(CsvServlet.class.toString());
-
- /* Get a reference to the R object generator that we need to generate
- * CSV files. */
- this.rObjectGenerator = (RObjectGenerator) getServletContext().
- getAttribute("RObjectGenerator");
- this.availableCsvFiles = rObjectGenerator.getAvailableCsvFiles();
- }
-
- public void doGet(HttpServletRequest request,
- HttpServletResponse response) throws IOException,
- ServletException {
-
- /* Check if the directory listing was requested. */
- String requestURI = request.getRequestURI();
- if (requestURI.equals("/ernie/csv/")) {
- request.setAttribute("directory", "/csv");
- request.setAttribute("extension", ".csv");
- request.setAttribute("files", this.availableCsvFiles);
- request.getRequestDispatcher("/WEB-INF/dir.jsp").forward(request,
- response);
- return;
- }
-
- /* Find out which CSV file was requested and make sure we know this
- * CSV file type. */
- String requestedCsvFile = requestURI;
- if (requestedCsvFile.endsWith(".csv")) {
- requestedCsvFile = requestedCsvFile.substring(0,
- requestedCsvFile.length() - ".csv".length());
- }
- if (requestedCsvFile.contains("/")) {
- requestedCsvFile = requestedCsvFile.substring(requestedCsvFile.
- lastIndexOf("/") + 1);
- }
- if (!availableCsvFiles.contains(requestedCsvFile)) {
- logger.info("Did not recognize requested .csv file from request "
- + "URI: '" + requestURI + "'. Responding with 404 Not Found.");
- response.sendError(HttpServletResponse.SC_NOT_FOUND);
- return;
- }
- logger.fine("CSV file '" + requestedCsvFile + ".csv' requested.");
-
- /* Request CSV file from R object generator, which asks Rserve to
- * generate it. */
- RObject csvFile = this.rObjectGenerator.generateCsv(
- requestedCsvFile, true);
-
- /* Make sure that we have a .csv file to return. */
- if (csvFile == null) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- /* Write CSV file to response. */
- String csvFileContent = new String(csvFile.getBytes());
- response.setContentType("text/csv");
- response.setHeader("Content-Length", String.valueOf(
- csvFileContent.length()));
- response.setHeader("Content-Disposition",
- "inline; filename=\"" + requestedCsvFile + ".csv\"");
- response.getWriter().print(csvFileContent);
- }
-}
-
diff --git a/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java b/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java
index 855c644..1179137 100644
--- a/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java
+++ b/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java
@@ -17,8 +17,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
@@ -37,7 +35,6 @@ public class RObjectGenerator implements ServletContextListener {
private String cachedGraphsDirectory;
private long maxCacheAge;
- private SortedSet<String> availableCsvFiles;
private Map<String, String> availableTables;
private Map<String, String> availableGraphs;
private Set<String> availableGraphFileTypes;
@@ -54,26 +51,6 @@ public class RObjectGenerator implements ServletContextListener {
this.cachedGraphsDirectory = servletContext.getInitParameter(
"cachedGraphsDir");
- /* Initialize map of available CSV files. */
- this.availableCsvFiles = new TreeSet<String>();
- this.availableCsvFiles.add("bandwidth");
- this.availableCsvFiles.add("bandwidth-flags");
- this.availableCsvFiles.add("bwhist-flags");
- this.availableCsvFiles.add("connbidirect");
- this.availableCsvFiles.add("cloudbridges");
- this.availableCsvFiles.add("dirbytes");
- this.availableCsvFiles.add("monthly-userstats-average");
- this.availableCsvFiles.add("monthly-userstats-peak");
- this.availableCsvFiles.add("networksize");
- this.availableCsvFiles.add("platforms");
- this.availableCsvFiles.add("relaycountries");
- this.availableCsvFiles.add("relayflags");
- this.availableCsvFiles.add("torperf");
- this.availableCsvFiles.add("torperf-failures");
- this.availableCsvFiles.add("userstats");
- this.availableCsvFiles.add("userstats-detector");
- this.availableCsvFiles.add("versions");
-
this.availableTables = new HashMap<String, String>();
this.availableTables.put("userstats-relay", "start,end,filename");
this.availableTables.put("userstats-bridge", "start,end,filename");
@@ -130,9 +107,6 @@ public class RObjectGenerator implements ServletContextListener {
} catch (InterruptedException e) {
}
}
- for (String csvFile : availableCsvFiles) {
- generateCsv(csvFile, false);
- }
for (String tableName : availableTables.keySet()) {
generateTable(tableName, tableName, new HashMap(), false);
}
@@ -192,23 +166,6 @@ public class RObjectGenerator implements ServletContextListener {
checkCache);
}
- public SortedSet<String> getAvailableCsvFiles() {
- return this.availableCsvFiles;
- }
-
- public RObject generateCsv(String requestedCsvFile,
- boolean checkCache) {
- /* Prepare filename and R query string. */
- String rQuery = "export_" + requestedCsvFile.replaceAll("-", "_")
- + "(path = '%s')";
- String csvFilename = requestedCsvFile + ".csv";
-
- /* See if we need to generate this .csv file. */
- File csvFile = new File(this.cachedGraphsDirectory + "/"
- + csvFilename);
- return this.generateRObject(rQuery, csvFile, csvFilename, checkCache);
- }
-
public List<Map<String, String>> generateTable(String tableName,
String requestedTable, Map parameterMap, boolean checkCache) {
More information about the tor-commits
mailing list