commit 9b0f692823ea9851c0b184cb6408882fb0cb5e9a Author: Karsten Loesing karsten.loesing@gmx.net Date: Mon Apr 28 23:28:14 2014 +0200

Stop providing old .csv files.

All contents are available in files listed on stats.html. --- website/etc/web.xml | 11 - website/rserve/csv.R | 213 -------------------- .../torproject/metrics/web/graphs/CsvServlet.java | 97 --------- .../metrics/web/graphs/RObjectGenerator.java | 43 ---- 4 files changed, 364 deletions(-)

diff --git a/website/etc/web.xml b/website/etc/web.xml index 33ff390..b46ce5a 100644 --- a/website/etc/web.xml +++ b/website/etc/web.xml @@ -231,17 +231,6 @@ </servlet-mapping>

<servlet> - <servlet-name>Csv</servlet-name> - <servlet-class> - org.torproject.metrics.web.graphs.CsvServlet - </servlet-class> - </servlet> - <servlet-mapping> - <servlet-name>Csv</servlet-name> - <url-pattern>/csv/*</url-pattern> - </servlet-mapping> - - <servlet> <servlet-name>ExoneraTor</servlet-name> <servlet-class> org.torproject.metrics.web.status.ExoneraTorServlet diff --git a/website/rserve/csv.R b/website/rserve/csv.R deleted file mode 100644 index e2a2172..0000000 --- a/website/rserve/csv.R +++ /dev/null @@ -1,213 +0,0 @@ -options(scipen = 15) - -export_networksize <- function(path) { - s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "servers.csv", sep = ""), stringsAsFactors = FALSE) - s <- s[s$flag == '' & s$country == '' & s$version == '' & - s$platform == '' & s$ec2bridge == '', - c("date", "relays", "bridges")] - write.csv(s, path, quote = FALSE, row.names = FALSE) -} - -export_cloudbridges <- function(path) { - s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "servers.csv", sep = ""), stringsAsFactors = FALSE) - s <- s[s$flag == '' & s$country == '' & s$version == '' & - s$platform == '' & s$ec2bridge == 't', ] - cloudbridges <- data.frame(date = s$date, cloudbridges = s$bridges) - write.csv(cloudbridges, path, quote = FALSE, row.names = FALSE) -} - -export_relaycountries <- function(path) { - s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "servers.csv", sep = ""), stringsAsFactors = FALSE) - s <- s[s$flag == '' & s$country != '' & s$version == '' & - s$platform == '' & s$ec2bridge == '', - c("date", "country", "relays")] - write.csv(s, path, quote = FALSE, row.names = FALSE) -} - -export_versions <- function(path) { - s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "servers.csv", sep = ""), stringsAsFactors = FALSE) - s <- s[s$flag == '' & s$country == '' & s$version != '' & - s$platform == '' & s$ec2bridge == '', - c("date", "version", "relays")] - versions <- cast(s, date ~ version, value = "relays") - versions <- versions[order(versions$date), ] - write.csv(versions, path, quote = FALSE, row.names = FALSE) -} - -export_platforms <- function(path) { - s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "servers.csv", sep = ""), stringsAsFactors = FALSE) - s <- s[s$flag == '' & s$country == '' & s$version == '' & - s$platform != '' & s$ec2bridge == '', - c("date", "platform", "relays")] - s <- data.frame(date = s$date, - platform = ifelse(s$platform == 'FreeBSD', 'bsd', - tolower(s$platform)), relays = s$relays) - s <- cast(s, date ~ platform, value = "relays") - platforms <- s[order(s$date), ] - write.csv(platforms, path, quote = FALSE, row.names = FALSE) -} - -export_bandwidth <- function(path) { - b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "bandwidth.csv", sep = ""), stringsAsFactors = FALSE) - b <- b[b$isexit == '' & b$isguard == '', ] - b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"), - bwadv = b$advbw, - bwhist = floor((b$bwread + b$bwwrite) / 2)) - b <- b[order(b$date), ] - write.csv(b, path, quote = FALSE, row.names = FALSE) -} - -export_bwhist_flags <- function(path) { - b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "bandwidth.csv", sep = ""), stringsAsFactors = FALSE) - b <- b[b$isexit != '' & b$isguard != '' & !is.na(b$bwread) & - !is.na(b$bwwrite), ] - b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"), - isexit = b$isexit == 't', isguard = b$isguard == 't', - read = b$bwread, written = b$bwwrite) - write.csv(b, path, quote = FALSE, row.names = FALSE) -} - -export_dirbytes <- function(path) { - b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "bandwidth.csv", sep = ""), stringsAsFactors = FALSE) - b <- b[b$isexit == '' & b$isguard == '' & !is.na(b$dirread) & - !is.na(b$dirwrite), ] - b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"), - dirread = b$dirread, dirwrite = b$dirwrite) - b <- b[order(b$date), ] - write.csv(b, path, quote = FALSE, row.names = FALSE) -} - -export_relayflags <- function(path) { - s <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "servers.csv", sep = ""), stringsAsFactors = FALSE) - s <- s[s$country == '' & s$version == '' & s$platform == '' & - s$ec2bridge == '', ] - s <- data.frame(date = as.Date(s$date, "%Y-%m-%d"), - flag = ifelse(s$flag == '', 'running', tolower(s$flag)), - relays = s$relays) - s <- cast(s, date ~ flag, value = "relays") - relayflags <- s[order(s$date), ] - write.csv(relayflags, path, quote = FALSE, row.names = FALSE) -} - -export_torperf <- function(path) { - t <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "torperf.csv", sep = ""), stringsAsFactors = FALSE) - t <- data.frame( - source = paste(ifelse(t$source == '', 'all', t$source), - ifelse(t$size == 50 * 1024, '50kb', - ifelse(t$size == 1024 * 1024, '1mb', '5mb')), - sep = '-'), - date = as.Date(t$date, "%Y-%m-%d"), - q1 = t$q1, md = t$md, q3 = t$q3) - torperf <- t[order(t$source, t$date), ] - write.csv(torperf, path, quote = FALSE, row.names = FALSE) -} - -export_torperf_failures <- function(path) { - t <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "torperf.csv", sep = ""), stringsAsFactors = FALSE) - t <- data.frame( - source = paste(ifelse(t$source == '', 'all', t$source), - ifelse(t$size == 50 * 1024, '50kb', - ifelse(t$size == 1024 * 1024, '1mb', '5mb')), - sep = '-'), - date = as.Date(t$date, "%Y-%m-%d"), - timeouts = t$timeouts, failures = t$failures, requests = t$requests) - torperf <- t[order(t$source, t$date), ] - write.csv(torperf, path, quote = FALSE, row.names = FALSE) -} - -export_connbidirect <- function(path) { - c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "connbidirect.csv", sep = ""), stringsAsFactors = FALSE) - write.csv(format(c, trim = TRUE, scientific = FALSE), path, - quote = FALSE, row.names = FALSE) -} - -export_bandwidth_flags <- function(path) { - b <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "bandwidth.csv", sep = ""), stringsAsFactors = FALSE) - b <- b[b$isexit != '' & b$isguard != '', ] - b <- data.frame(date = as.Date(b$date, "%Y-%m-%d"), - isexit = b$isexit == 't', isguard = b$isguard == 't', - advbw = b$advbw, - bwhist = floor((b$bwread + b$bwwrite) / 2)) - b <- rbind( - data.frame(b[b$isguard == TRUE, ], flag = "guard"), - data.frame(b[b$isexit == TRUE, ], flag = "exit")) - b <- data.frame(date = b$date, advbw = b$advbw, bwhist = b$bwhist, - flag = b$flag) - b <- aggregate(list(advbw = b$advbw, bwhist = b$bwhist), - by = list(date = b$date, flag = b$flag), FUN = sum, - na.rm = TRUE, na.action = NULL) - b <- melt(b, id.vars = c("date", "flag")) - b <- data.frame(date = b$date, type = b$variable, flag = b$flag, - value = b$value) - b <- b[b$value > 0, ] - write.csv(b, path, quote = FALSE, row.names = FALSE) -} - -export_userstats <- function(path) { - c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "clients.csv", sep = ""), stringsAsFactors = FALSE) - c <- data.frame(date = c$date, node = c$node, country = c$country, - transport = c$transport, version = c$version, - frac = c$frac, users = c$clients) - write.csv(format(c, trim = TRUE, scientific = FALSE), path, - quote = FALSE, row.names = FALSE) -} - -help_export_monthly_userstats <- function(path, aggr_fun) { - c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "clients.csv", sep = ""), stringsAsFactors = FALSE) - c <- c[c$country != '' & c$transport == '' & c$version == '', ] - u <- data.frame(date = c$date, country = c$country, users = c$clients, - stringsAsFactors = FALSE) - u <- aggregate(list(users = u$users), - by = list(date = u$date, country = u$country), sum) - u <- aggregate(list(users = u$users), - by = list(country = u$country, - month = substr(u$date, 1, 7)), aggr_fun) - u <- rbind(u, data.frame(country = "zy", - aggregate(list(users = u$users), - by = list(month = u$month), sum))) - u <- cast(u, country ~ month, value = "users") - u[u$country == "zy", "country"] <- "all" - u[, 2:length(u)] <- floor(u[, 2:length(u)]) - write.csv(u, path, quote = FALSE, row.names = FALSE) -} - -export_monthly_userstats_peak <- function(path) { - help_export_monthly_userstats(path, max) -} - -export_monthly_userstats_average <- function(path) { - help_export_monthly_userstats(path, mean) -} - -export_userstats_detector <- function(path) { - c <- read.csv(paste("/srv/metrics.torproject.org/web/shared/stats/", - "clients.csv", sep = ""), stringsAsFactors = FALSE) - c <- c[c$country != '' & c$transport == '' & c$version == '' & - c$node == 'relay', ] - u <- data.frame(country = c$country, date = c$date, users = c$clients, - stringsAsFactors = FALSE) - u <- rbind(u, data.frame(country = "zy", - aggregate(list(users = u$users), - by = list(date = u$date), sum))) - u <- data.frame(date = u$date, country = u$country, - users = floor(u$users)) - u <- cast(u, date ~ country, value = "users") - names(u)[names(u) == "zy"] <- "all" - write.csv(u, path, quote = FALSE, row.names = FALSE) -} - diff --git a/website/src/org/torproject/metrics/web/graphs/CsvServlet.java b/website/src/org/torproject/metrics/web/graphs/CsvServlet.java deleted file mode 100644 index 96e5f6a..0000000 --- a/website/src/org/torproject/metrics/web/graphs/CsvServlet.java +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.metrics.web.graphs; - -import java.io.IOException; -import java.util.SortedSet; -import java.util.logging.Logger; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * Servlet that reads an HTTP request for a comma-separated value file, - * asks the GraphGenerator to generate this file, and returns it to the - * client. - */ -public class CsvServlet extends HttpServlet { - - private static final long serialVersionUID = 7501442926823719958L; - - private RObjectGenerator rObjectGenerator; - - /* Available CSV files. */ - private SortedSet<String> availableCsvFiles; - - private Logger logger; - - public void init() { - - /* Initialize logger. */ - this.logger = Logger.getLogger(CsvServlet.class.toString()); - - /* Get a reference to the R object generator that we need to generate - * CSV files. */ - this.rObjectGenerator = (RObjectGenerator) getServletContext(). - getAttribute("RObjectGenerator"); - this.availableCsvFiles = rObjectGenerator.getAvailableCsvFiles(); - } - - public void doGet(HttpServletRequest request, - HttpServletResponse response) throws IOException, - ServletException { - - /* Check if the directory listing was requested. */ - String requestURI = request.getRequestURI(); - if (requestURI.equals("/ernie/csv/")) { - request.setAttribute("directory", "/csv"); - request.setAttribute("extension", ".csv"); - request.setAttribute("files", this.availableCsvFiles); - request.getRequestDispatcher("/WEB-INF/dir.jsp").forward(request, - response); - return; - } - - /* Find out which CSV file was requested and make sure we know this - * CSV file type. */ - String requestedCsvFile = requestURI; - if (requestedCsvFile.endsWith(".csv")) { - requestedCsvFile = requestedCsvFile.substring(0, - requestedCsvFile.length() - ".csv".length()); - } - if (requestedCsvFile.contains("/")) { - requestedCsvFile = requestedCsvFile.substring(requestedCsvFile. - lastIndexOf("/") + 1); - } - if (!availableCsvFiles.contains(requestedCsvFile)) { - logger.info("Did not recognize requested .csv file from request " - + "URI: '" + requestURI + "'. Responding with 404 Not Found."); - response.sendError(HttpServletResponse.SC_NOT_FOUND); - return; - } - logger.fine("CSV file '" + requestedCsvFile + ".csv' requested."); - - /* Request CSV file from R object generator, which asks Rserve to - * generate it. */ - RObject csvFile = this.rObjectGenerator.generateCsv( - requestedCsvFile, true); - - /* Make sure that we have a .csv file to return. */ - if (csvFile == null) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; - } - - /* Write CSV file to response. */ - String csvFileContent = new String(csvFile.getBytes()); - response.setContentType("text/csv"); - response.setHeader("Content-Length", String.valueOf( - csvFileContent.length())); - response.setHeader("Content-Disposition", - "inline; filename="" + requestedCsvFile + ".csv""); - response.getWriter().print(csvFileContent); - } -} - diff --git a/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java b/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java index 855c644..1179137 100644 --- a/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java +++ b/website/src/org/torproject/metrics/web/graphs/RObjectGenerator.java @@ -17,8 +17,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet;

import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; @@ -37,7 +35,6 @@ public class RObjectGenerator implements ServletContextListener { private String cachedGraphsDirectory; private long maxCacheAge;

- private SortedSet<String> availableCsvFiles; private Map<String, String> availableTables; private Map<String, String> availableGraphs; private Set<String> availableGraphFileTypes; @@ -54,26 +51,6 @@ public class RObjectGenerator implements ServletContextListener { this.cachedGraphsDirectory = servletContext.getInitParameter( "cachedGraphsDir");

- /* Initialize map of available CSV files. */ - this.availableCsvFiles = new TreeSet<String>(); - this.availableCsvFiles.add("bandwidth"); - this.availableCsvFiles.add("bandwidth-flags"); - this.availableCsvFiles.add("bwhist-flags"); - this.availableCsvFiles.add("connbidirect"); - this.availableCsvFiles.add("cloudbridges"); - this.availableCsvFiles.add("dirbytes"); - this.availableCsvFiles.add("monthly-userstats-average"); - this.availableCsvFiles.add("monthly-userstats-peak"); - this.availableCsvFiles.add("networksize"); - this.availableCsvFiles.add("platforms"); - this.availableCsvFiles.add("relaycountries"); - this.availableCsvFiles.add("relayflags"); - this.availableCsvFiles.add("torperf"); - this.availableCsvFiles.add("torperf-failures"); - this.availableCsvFiles.add("userstats"); - this.availableCsvFiles.add("userstats-detector"); - this.availableCsvFiles.add("versions"); - this.availableTables = new HashMap<String, String>(); this.availableTables.put("userstats-relay", "start,end,filename"); this.availableTables.put("userstats-bridge", "start,end,filename"); @@ -130,9 +107,6 @@ public class RObjectGenerator implements ServletContextListener { } catch (InterruptedException e) { } } - for (String csvFile : availableCsvFiles) { - generateCsv(csvFile, false); - } for (String tableName : availableTables.keySet()) { generateTable(tableName, tableName, new HashMap(), false); } @@ -192,23 +166,6 @@ public class RObjectGenerator implements ServletContextListener { checkCache); }

- public SortedSet<String> getAvailableCsvFiles() { - return this.availableCsvFiles; - } - - public RObject generateCsv(String requestedCsvFile, - boolean checkCache) { - /* Prepare filename and R query string. */ - String rQuery = "export_" + requestedCsvFile.replaceAll("-", "_") - + "(path = '%s')"; - String csvFilename = requestedCsvFile + ".csv"; - - /* See if we need to generate this .csv file. */ - File csvFile = new File(this.cachedGraphsDirectory + "/" - + csvFilename); - return this.generateRObject(rQuery, csvFile, csvFilename, checkCache); - } - public List<Map<String, String>> generateTable(String tableName, String requestedTable, Map parameterMap, boolean checkCache) {