[tor-commits] [metrics-web/master] Add OnionPerf's onion server measurements.

karsten at torproject.org karsten at torproject.org
Mon May 1 14:44:19 UTC 2017


commit 917cc649b2012ea409fea1b73a7b5715e5ecb78a
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon May 1 11:28:25 2017 +0200

    Add OnionPerf's onion server measurements.
    
    Adding these measurements requires two changes:
     1. Add new data-processing module onionperf that will replace the
        torperf module.
     2. Extend torperf graphs on the website.
---
 modules/onionperf/.gitignore                       |   2 +
 modules/onionperf/build.xml                        |  19 ++
 .../org/torproject/metrics/onionperf/Main.java     | 255 +++++++++++++++++++++
 .../src/main/resources/init-onionperf.sql          |  90 ++++++++
 shared/bin/40-run-onionperf-stats.sh               |   5 +
 shared/bin/99-copy-stats-files.sh                  |   1 +
 shared/build.xml                                   |   1 +
 website/rserve/graphs.R                            |  26 ++-
 .../org/torproject/metrics/web/GraphServlet.java   |   4 +
 .../metrics/web/graphs/GraphParameterChecker.java  |  21 ++
 website/src/main/resources/etc/metrics.json        |   6 +-
 website/src/main/resources/web/WEB-INF/graph.jsp   |   9 +
 website/src/main/resources/web/WEB-INF/stats.jsp   |  58 +++++
 13 files changed, 483 insertions(+), 14 deletions(-)

diff --git a/modules/onionperf/.gitignore b/modules/onionperf/.gitignore
new file mode 100644
index 0000000..4a8ab3e
--- /dev/null
+++ b/modules/onionperf/.gitignore
@@ -0,0 +1,2 @@
+/stats/*.csv
+
diff --git a/modules/onionperf/build.xml b/modules/onionperf/build.xml
new file mode 100644
index 0000000..3ddf2d3
--- /dev/null
+++ b/modules/onionperf/build.xml
@@ -0,0 +1,19 @@
+<project default="run" name="onionperf" basedir=".">
+
+  <property name="mainclass" value="org.torproject.metrics.onionperf.Main"/>
+
+  <include file="../../shared/build-base.xml" as="basetask"/>
+  <target name="clean" depends="basetask.clean"/>
+  <target name="compile" depends="basetask.compile"/>
+  <target name="run" depends="basetask.run"/>
+
+  <path id="classpath">
+    <pathelement path="${classes}"/>
+    <path refid="base.classpath" />
+    <fileset dir="${libs}">
+      <include name="postgresql-jdbc3-9.2.jar"/>
+    </fileset>
+  </path>
+
+</project>
+
diff --git a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
new file mode 100644
index 0000000..f54fa9b
--- /dev/null
+++ b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
@@ -0,0 +1,255 @@
+package org.torproject.metrics.onionperf;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.TorperfResult;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+
+public class Main {
+
+  /** Logger for this class. */
+  private static Logger log = LoggerFactory.getLogger(Main.class);
+
+  /** Executes this data-processing module. */
+  public static void main(String[] args) throws Exception {
+    log.info("Starting onionperf module.");
+    String dbUrlString = "jdbc:postgresql:onionperf";
+    Connection connection = connectToDatabase(dbUrlString);
+    importOnionPerfFiles(connection);
+    SortedSet<String> statistics = queryOnionPerf(connection);
+    writeStatistics(Paths.get("stats", "onionperf.csv"), statistics);
+    disconnectFromDatabase(connection);
+    log.info("Terminated onionperf module.");
+  }
+
+  private static Connection connectToDatabase(String jdbcString)
+      throws SQLException {
+    log.info("Connecting to database.");
+    Connection connection = DriverManager.getConnection(jdbcString);
+    connection.setAutoCommit(false);
+    log.info("Successfully connected to database.");
+    return connection;
+  }
+
+  private static void importOnionPerfFiles(Connection connection)
+      throws SQLException {
+
+    PreparedStatement psMeasurementsSelect = connection.prepareStatement(
+        "SELECT measurement_id FROM measurements WHERE source = ? "
+        + "AND filesize = ? AND start = ?");
+
+    PreparedStatement psMeasurementsInsert = connection.prepareStatement(
+        "INSERT INTO measurements (source, filesize, start, socket, connect, "
+        + "negotiate, request, response, datarequest, dataresponse, "
+        + "datacomplete, writebytes, readbytes, didtimeout, dataperc0, "
+        + "dataperc10, dataperc20, dataperc30, dataperc40, dataperc50, "
+        + "dataperc60, dataperc70, dataperc80, dataperc90, dataperc100, "
+        + "launch, used_at, timeout, quantile, circ_id, used_by, "
+        + "endpointlocal, endpointproxy, endpointremote, hostnamelocal, "
+        + "hostnameremote, sourceaddress) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, "
+        + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
+        + "?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS);
+
+    DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader();
+    dr.addDirectory(new File("../../shared/in/archive/torperf"));
+    dr.addDirectory(new File("../../shared/in/recent/torperf"));
+    Iterator<DescriptorFile> dfs = dr.readDescriptors();
+    Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+    while (dfs.hasNext()) {
+      DescriptorFile df = dfs.next();
+      for (Descriptor d : df.getDescriptors()) {
+        if (!(d instanceof TorperfResult)) {
+          continue;
+        }
+        TorperfResult tr = (TorperfResult) d;
+        int measurementId = -1;
+        String truncatedSource = truncateString(tr.getSource(), 32);
+        psMeasurementsSelect.clearParameters();
+        psMeasurementsSelect.setString(1, truncatedSource);
+        psMeasurementsSelect.setInt(2, tr.getFileSize());
+        psMeasurementsSelect.setTimestamp(3,
+            new Timestamp(tr.getStartMillis()), calendar);
+        try (ResultSet rs = psMeasurementsSelect.executeQuery()) {
+          if (rs.next()) {
+            measurementId = rs.getInt(1);
+          }
+        }
+        if (measurementId < 0) {
+          psMeasurementsInsert.clearParameters();
+          psMeasurementsInsert.setString(1, truncatedSource);
+          psMeasurementsInsert.setInt(2, tr.getFileSize());
+          psMeasurementsInsert.setTimestamp(3,
+              new Timestamp(tr.getStartMillis()), calendar);
+          long[] timestamps = new long[] { tr.getSocketMillis(),
+              tr.getConnectMillis(), tr.getNegotiateMillis(),
+              tr.getRequestMillis(), tr.getResponseMillis(),
+              tr.getDataRequestMillis(), tr.getDataResponseMillis(),
+              tr.getDataCompleteMillis() };
+          for (int i = 4, j = 0; j < timestamps.length; i++, j++) {
+            if (timestamps[j] == 0L) {
+              psMeasurementsInsert.setNull(i, Types.INTEGER);
+            } else {
+              psMeasurementsInsert.setInt(i,
+                  (int) (timestamps[j] - tr.getStartMillis()));
+            }
+          }
+          psMeasurementsInsert.setInt(12, tr.getWriteBytes());
+          psMeasurementsInsert.setInt(13, tr.getReadBytes());
+          if (null == tr.didTimeout()) {
+            psMeasurementsInsert.setNull(14, Types.BOOLEAN);
+          } else {
+            psMeasurementsInsert.setBoolean(14, tr.didTimeout());
+          }
+          for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) {
+            if (null == tr.getDataPercentiles()
+                || !tr.getDataPercentiles().containsKey(p)) {
+              psMeasurementsInsert.setNull(i, Types.INTEGER);
+            } else {
+              psMeasurementsInsert.setInt(i,
+                  (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis()));
+            }
+          }
+          if (tr.getLaunchMillis() < 0L) {
+            psMeasurementsInsert.setNull(26, Types.TIMESTAMP);
+          } else {
+            psMeasurementsInsert.setTimestamp(26,
+                new Timestamp(tr.getLaunchMillis()), calendar);
+          }
+          if (tr.getUsedAtMillis() < 0L) {
+            psMeasurementsInsert.setNull(27, Types.TIMESTAMP);
+          } else {
+            psMeasurementsInsert.setTimestamp(27,
+                new Timestamp(tr.getUsedAtMillis()), calendar);
+          }
+          if (tr.getTimeout() < 0L) {
+            psMeasurementsInsert.setNull(28, Types.INTEGER);
+          } else {
+            psMeasurementsInsert.setInt(28, (int) tr.getTimeout());
+          }
+          if (tr.getQuantile() < 0.0) {
+            psMeasurementsInsert.setNull(29, Types.REAL);
+          } else {
+            psMeasurementsInsert.setDouble(29, tr.getQuantile());
+          }
+          if (tr.getCircId() < 0L) {
+            psMeasurementsInsert.setNull(30, Types.INTEGER);
+          } else {
+            psMeasurementsInsert.setInt(30, tr.getCircId());
+          }
+          if (tr.getUsedBy() < 0L) {
+            psMeasurementsInsert.setNull(31, Types.INTEGER);
+          } else {
+            psMeasurementsInsert.setInt(31, tr.getUsedBy());
+          }
+          String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL",
+              "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL",
+              "HOSTNAMEREMOTE", "SOURCEADDRESS" };
+          for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) {
+            if (null == tr.getUnrecognizedKeys()
+                || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) {
+              psMeasurementsInsert.setNull(i, Types.VARCHAR);
+            } else {
+              psMeasurementsInsert.setString(i, truncateString(
+                  tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64));
+            }
+          }
+          psMeasurementsInsert.execute();
+          try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) {
+            if (rs.next()) {
+              measurementId = rs.getInt(1);
+            }
+          }
+        }
+        /* Could use measurementId to insert path. */
+      }
+      connection.commit();
+    }
+  }
+
+  private static String truncateString(String originalString,
+      int truncateAfter) {
+    if (originalString.length() > truncateAfter) {
+      originalString = originalString.substring(0, truncateAfter);
+    }
+    return originalString;
+  }
+
+  static SortedSet<String> queryOnionPerf(Connection connection)
+      throws SQLException {
+    log.info("Querying statistics from database.");
+    SortedSet<String> statistics = new TreeSet<>();
+    Statement st = connection.createStatement();
+    String queryString = "SELECT date, filesize, source, server, q1, md, q3, "
+        + "timeouts, failures, requests FROM onionperf";
+    DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
+    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+    try (ResultSet rs = st.executeQuery(queryString)) {
+      while (rs.next()) {
+        statistics.add(String.format("%s,%d,%s,%s,%.0f,%.0f,%.0f,%d,%d,%d",
+            dateFormat.format(rs.getDate("date", calendar)),
+            rs.getInt("filesize"),
+            emptyNull(rs.getString("source")),
+            emptyNull(rs.getString("server")),
+            rs.getDouble("q1"),
+            rs.getDouble("md"),
+            rs.getDouble("q3"),
+            rs.getInt("timeouts"),
+            rs.getInt("failures"),
+            rs.getInt("requests")));
+      }
+    }
+    return statistics;
+  }
+
+  private static String emptyNull(String text) {
+    return null == text ? "" : text;
+  }
+
+  static void writeStatistics(Path webstatsPath,
+      SortedSet<String> statistics) throws IOException {
+    webstatsPath.toFile().getParentFile().mkdirs();
+    List<String> lines = new ArrayList<>();
+    lines.add("date,filesize,source,server,q1,md,q3,timeouts,failures,requests");
+    lines.addAll(statistics);
+    log.info("Writing {} lines to {}.", lines.size(),
+        webstatsPath.toFile().getAbsolutePath());
+    Files.write(webstatsPath, lines, StandardCharsets.UTF_8);
+  }
+
+  private static void disconnectFromDatabase(Connection connection)
+      throws SQLException {
+    log.info("Disconnecting from database.");
+    connection.close();
+  }
+}
+
diff --git a/modules/onionperf/src/main/resources/init-onionperf.sql b/modules/onionperf/src/main/resources/init-onionperf.sql
new file mode 100644
index 0000000..466c1d7
--- /dev/null
+++ b/modules/onionperf/src/main/resources/init-onionperf.sql
@@ -0,0 +1,90 @@
+-- Copyright 2017 The Tor Project
+-- See LICENSE for licensing information
+
+CREATE TABLE IF NOT EXISTS measurements (
+  measurement_id SERIAL PRIMARY KEY,
+  source CHARACTER VARYING(32) NOT NULL,
+  filesize INTEGER NOT NULL,
+  start TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+  socket INTEGER,
+  connect INTEGER,
+  negotiate INTEGER,
+  request INTEGER,
+  response INTEGER,
+  datarequest INTEGER,
+  dataresponse INTEGER,
+  datacomplete INTEGER,
+  writebytes INTEGER,
+  readbytes INTEGER,
+  didtimeout BOOLEAN,
+  dataperc0 INTEGER,
+  dataperc10 INTEGER,
+  dataperc20 INTEGER,
+  dataperc30 INTEGER,
+  dataperc40 INTEGER,
+  dataperc50 INTEGER,
+  dataperc60 INTEGER,
+  dataperc70 INTEGER,
+  dataperc80 INTEGER,
+  dataperc90 INTEGER,
+  dataperc100 INTEGER,
+  launch TIMESTAMP WITHOUT TIME ZONE,
+  used_at TIMESTAMP WITHOUT TIME ZONE,
+  timeout INTEGER,
+  quantile REAL,
+  circ_id INTEGER,
+  used_by INTEGER,
+  endpointlocal CHARACTER VARYING(64),
+  endpointproxy CHARACTER VARYING(64),
+  endpointremote CHARACTER VARYING(64),
+  hostnamelocal CHARACTER VARYING(64),
+  hostnameremote CHARACTER VARYING(64),
+  sourceaddress CHARACTER VARYING(64),
+  UNIQUE (source, filesize, start)
+);
+
+CREATE TYPE server AS ENUM ('public', 'onion');
+
+CREATE OR REPLACE VIEW onionperf AS
+SELECT date,
+  filesize,
+  source,
+  server,
+  q[1] AS q1,
+  q[2] AS md,
+  q[3] AS q3,
+  timeouts,
+  failures,
+  requests
+FROM (
+SELECT DATE(start) AS date,
+  filesize,
+  source,
+  CASE WHEN endpointremote LIKE '%.onion%' THEN 'onion'
+    ELSE 'public' END AS server,
+  PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) AS q,
+  COUNT(CASE WHEN didtimeout OR datacomplete < 1 THEN 1 ELSE NULL END)
+    AS timeouts,
+  COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1
+    AND readbytes < filesize THEN 1 ELSE NULL END) AS failures,
+  COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1
+    AND readbytes >= filesize then 1 else null end) AS requests
+FROM measurements
+GROUP BY date, filesize, source, server
+UNION
+SELECT DATE(start) AS date,
+  filesize,
+  '' AS source,
+  CASE WHEN endpointremote LIKE '%.onion%' THEN 'onion'
+    ELSE 'public' END AS server,
+  PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete) AS q,
+  COUNT(CASE WHEN didtimeout OR datacomplete < 1 THEN 1 ELSE NULL END)
+    AS timeouts,
+  COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1
+    AND readbytes < filesize THEN 1 ELSE NULL END) AS failures,
+  COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1
+    AND readbytes >= filesize then 1 else null end) AS requests
+FROM measurements
+GROUP BY date, filesize, 3, server) sub
+ORDER BY date, filesize, source, server;
+
diff --git a/shared/bin/40-run-onionperf-stats.sh b/shared/bin/40-run-onionperf-stats.sh
new file mode 100755
index 0000000..6f025e7
--- /dev/null
+++ b/shared/bin/40-run-onionperf-stats.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+cd modules/onionperf/
+ant | grep "\[java\]" | grep -Ev " DEBUG | INFO "
+cd ../../
+
diff --git a/shared/bin/99-copy-stats-files.sh b/shared/bin/99-copy-stats-files.sh
index a828686..cc0c9fc 100755
--- a/shared/bin/99-copy-stats-files.sh
+++ b/shared/bin/99-copy-stats-files.sh
@@ -1,5 +1,6 @@
 #!/bin/sh
 mkdir -p shared/stats
+cp -a modules/onionperf/stats/*.csv shared/stats/
 cp -a modules/legacy/stats/*.csv shared/stats/
 cp -a modules/connbidirect/stats/connbidirect2.csv shared/stats/
 cp -a modules/advbwdist/stats/advbwdist.csv shared/stats/
diff --git a/shared/build.xml b/shared/build.xml
index cb51d5f..3b0a799 100644
--- a/shared/build.xml
+++ b/shared/build.xml
@@ -28,6 +28,7 @@
       <fileset dir="../modules/connbidirect/src" includes="**/*.java"/>
       <fileset dir="../modules/hidserv/src" includes="**/*.java"/>
       <fileset dir="../modules/legacy/src" includes="**/*.java"/>
+      <fileset dir="../modules/onionperf/src" includes="**/*.java"/>
       <fileset dir="../modules/webstats/src" includes="**/*.java"/>
       <classpath>
         <path refid="checkstyle.classpath" />
diff --git a/website/rserve/graphs.R b/website/rserve/graphs.R
index 988e3b2..fc84aa8 100644
--- a/website/rserve/graphs.R
+++ b/website/rserve/graphs.R
@@ -559,18 +559,19 @@ plot_relayflags <- function(start, end, flags, path) {
   ggsave(filename = path, width = 8, height = 5, dpi = 72)
 }
 
-plot_torperf <- function(start, end, source, filesize, path) {
+plot_torperf <- function(start, end, source, server, filesize, path) {
   end <- min(end, as.character(Sys.Date() - 2))
-  size <- ifelse(filesize == '50kb', 50 * 1024,
+  filesizeVal <- ifelse(filesize == '50kb', 50 * 1024,
           ifelse(filesize == '1mb', 1024 * 1024, 5 * 1024 * 1024))
   t <- read.csv(paste("/srv/metrics.torproject.org/metrics/shared/stats/",
-                "torperf.csv", sep = ""), stringsAsFactors = FALSE)
+                "onionperf.csv", sep = ""), stringsAsFactors = FALSE)
   known_sources <- c("all", unique(t[t$source != "", "source"]))
   colours <- data.frame(source = known_sources,
       colour = brewer.pal(length(known_sources), "Paired"),
       stringsAsFactors = FALSE)
-  t <- t[t$date >= start & t$date <= end & t$size == size &
-         t$source == ifelse(source == 'all', '', source), ]
+  t <- t[t$date >= start & t$date <= end & t$filesize == filesizeVal &
+         t$source == ifelse(source == 'all', '', source) &
+         t$server == server, ]
   torperf <- data.frame(date = as.Date(t$date, "%Y-%m-%d"),
                         q1 = t$q1, md = t$md, q3 = t$q3)
   dates <- seq(from = as.Date(start, "%Y-%m-%d"),
@@ -605,19 +606,20 @@ plot_torperf <- function(start, end, source, filesize, path) {
       labels = c("Median", "1st to 3rd quartile"),
       values = paste(colour, c("", "66"), sep = "")) +
     ggtitle(paste("Time in seconds to complete", filesizeStr,
-        "request")) +
+        "request to", server, "server")) +
     theme(legend.position = "top")
   ggsave(filename = path, width = 8, height = 5, dpi = 72)
 }
 
-plot_torperf_failures <- function(start, end, source, filesize, path) {
+plot_torperf_failures <- function(start, end, source, server, filesize, path) {
   end <- min(end, as.character(Sys.Date() - 2))
-  size <- ifelse(filesize == '50kb', 50 * 1024,
+  filesizeVal <- ifelse(filesize == '50kb', 50 * 1024,
           ifelse(filesize == '1mb', 1024 * 1024, 5 * 1024 * 1024))
   t <- read.csv(paste("/srv/metrics.torproject.org/metrics/shared/stats/",
-                "torperf.csv", sep = ""), stringsAsFactors = FALSE)
-  t <- t[t$date >= start & t$date <= end & t$size == size &
-         t$source == ifelse(source == 'all', '', source), ]
+                "onionperf.csv", sep = ""), stringsAsFactors = FALSE)
+  t <- t[t$date >= start & t$date <= end & t$filesize == filesizeVal &
+         t$source == ifelse(source == 'all', '', source) &
+         t$server == server, ]
   torperf <- data.frame(date = as.Date(t$date, "%Y-%m-%d"),
                         timeouts = t$timeouts, failures = t$failures,
                         requests = t$requests)
@@ -656,7 +658,7 @@ plot_torperf_failures <- function(start, end, source, filesize, path) {
         h.start = 45, breaks = c("timeouts", "failures"),
         labels = c("Timeouts", "Failures")) +
     ggtitle(paste("Timeouts and failures of", filesizeStr,
-        "requests")) +
+        "requests to", server, "server")) +
     theme(legend.position = "top")
   ggsave(filename = path, width = 8, height = 5, dpi = 72)
 }
diff --git a/website/src/main/java/org/torproject/metrics/web/GraphServlet.java b/website/src/main/java/org/torproject/metrics/web/GraphServlet.java
index 48b0301..12652dc 100644
--- a/website/src/main/java/org/torproject/metrics/web/GraphServlet.java
+++ b/website/src/main/java/org/torproject/metrics/web/GraphServlet.java
@@ -114,6 +114,9 @@ public class GraphServlet extends MetricServlet {
         { "op-hk", "" },
         { "op-nl", "" },
         { "op-us", "" }});
+    this.defaultParameters.put("server", new String[][] {
+        { "public", " checked" },
+        { "onion", "" }});
     this.defaultParameters.put("filesize", new String[][] {
         { "50kb", " checked", "50 KiB" },
         { "1mb", "", "1 MiB" },
@@ -184,6 +187,7 @@ public class GraphServlet extends MetricServlet {
             || parameter.equals("transport")
             || parameter.equals("version")
             || parameter.equals("source")
+            || parameter.equals("server")
             || parameter.equals("filesize")) {
           String[][] defaultParameters =
               this.defaultParameters.get(parameter);
diff --git a/website/src/main/java/org/torproject/metrics/web/graphs/GraphParameterChecker.java b/website/src/main/java/org/torproject/metrics/web/graphs/GraphParameterChecker.java
index e8872a7..c693706 100644
--- a/website/src/main/java/org/torproject/metrics/web/graphs/GraphParameterChecker.java
+++ b/website/src/main/java/org/torproject/metrics/web/graphs/GraphParameterChecker.java
@@ -66,6 +66,7 @@ public class GraphParameterChecker {
     this.knownParameterValues.put("events", "on,off,points");
     this.knownParameterValues.put("source", "all,siv,moria,torperf,op-hk,"
         + "op-nl,op-us");
+    this.knownParameterValues.put("server", "public,onion");
     this.knownParameterValues.put("filesize", "50kb,1mb,5mb");
     this.knownParameterValues.put("transport", "obfs2,obfs3,obfs4,"
         + "websocket,fte,meek,scramblesuit,snowflake,<OR>,<??>,!<OR>");
@@ -223,6 +224,26 @@ public class GraphParameterChecker {
       recognizedGraphParameters.put("source", sourceParameter);
     }
 
+    /* Parse onionperf server if supported by the graph type. Only a single
+     * server can be passed. If no server is passed, use "public" as default. */
+    if (supportedGraphParameters.contains("server")) {
+      String[] serverParameter = (String[]) requestParameters.get("server");
+      List<String> knownServers = Arrays.asList(
+          this.knownParameterValues.get("server").split(","));
+      if (serverParameter != null) {
+        if (serverParameter.length != 1) {
+          return null;
+        }
+        if (serverParameter[0].length() == 0
+            || !knownServers.contains(serverParameter[0])) {
+          return null;
+        }
+      } else {
+        serverParameter = new String[] { "public" };
+      }
+      recognizedGraphParameters.put("server", serverParameter);
+    }
+
     /* Parse torperf file size if supported by the graph type. Only a
      * single file size can be passed. If no file size is passed, use
      * "50kb" as default. */
diff --git a/website/src/main/resources/etc/metrics.json b/website/src/main/resources/etc/metrics.json
index a335640..6c3dde1 100644
--- a/website/src/main/resources/etc/metrics.json
+++ b/website/src/main/resources/etc/metrics.json
@@ -307,10 +307,11 @@
       "start",
       "end",
       "source",
+      "server",
       "filesize"
     ],
     "data": [
-      "torperf"
+      "onionperf"
     ]
   },
   {
@@ -323,10 +324,11 @@
       "start",
       "end",
       "source",
+      "server",
       "filesize"
     ],
     "data": [
-      "torperf"
+      "onionperf"
     ]
   },
   {
diff --git a/website/src/main/resources/web/WEB-INF/graph.jsp b/website/src/main/resources/web/WEB-INF/graph.jsp
index 59261a6..20372e8 100644
--- a/website/src/main/resources/web/WEB-INF/graph.jsp
+++ b/website/src/main/resources/web/WEB-INF/graph.jsp
@@ -131,6 +131,15 @@
       </c:forEach>
       </p>
     </c:if>
+    <c:if test="${fn:length(server) > 0}">
+      <p><b>Server (beta):</b>
+      <c:forEach var="row" items="${server}">
+        <label class="radio-label">
+          <input type="radio" name="server" value="${row[0]}"${row[1]}> ${row[0]}
+        </label>
+      </c:forEach>
+      </p>
+    </c:if>
     <c:if test="${fn:length(filesize) > 0}">
       <p><b>File size:</b>
       <c:forEach var="row" items="${filesize}">
diff --git a/website/src/main/resources/web/WEB-INF/stats.jsp b/website/src/main/resources/web/WEB-INF/stats.jsp
index 0fb77ea..d0e0c2d 100644
--- a/website/src/main/resources/web/WEB-INF/stats.jsp
+++ b/website/src/main/resources/web/WEB-INF/stats.jsp
@@ -355,6 +355,64 @@ over Tor.</li>
 </div>
 
 <div class="container">
+<h2>Performance of downloading static files over Tor <a href="#onionperf" name="onionperf" class="anchor">#</a></h2>
+
+<div class="bs-callout bs-callout-warning">
+<h3>Beta</h3>
+<p>As of May 1, 2017, this data file is still under development.  If it becomes stable, it will replace <a href="#torperf">this data file</a>.  But until that is the case, it may change or disappear without prior notice.</p>
+</div>
+
+<p>The following data file contains aggregate statistics on performance when
+downloading static files of different sizes over Tor.  These statistics are
+generated by <a href="https://github.com/robgjansen/onionperf">OnionPerf</a> and
+its predecessor <a href="https://gitweb.torproject.org/torperf.git">Torperf</a>,
+which both periodically fetch static files over Tor and record several
+timestamps in the process.  The data file contains daily medians and quartiles
+as well as total numbers of requests, timeouts, and failures.  Raw Onionperf and
+Torperf measurement data is available on the <a
+href="https://collector.torproject.org/#torperf">CollecTor</a> website.</p>
+
+<p><b>Download as <a href="stats/onionperf.csv">CSV file</a>.</b></p>
+
+<p>The statistics file contains the following columns:</p>
+<ul>
+
+<li><b>date:</b> UTC date (YYYY-MM-DD) when download performance was
+measured.</li>
+
+<li><b>filesize:</b> Size of the downloaded file in bytes.</li>
+
+<li><b>source:</b> Name of the OnionPerf or Torperf service performing
+measurements.  If this column contains the empty string, all measurements are
+included, regardless of which service performed them.</li>
+
+<li><b>server:</b> Either <b>"public"</b> if the request was made to a server on
+the public internet, or <b>"onion"</b> if the request was made to an onion
+server.</li>
+
+<li><b>q1:</b> First quartile of time until receiving the last byte in
+milliseconds.</li>
+
+<li><b>md:</b> Median of time until receiving the last byte in
+milliseconds.</li>
+
+<li><b>q3:</b> Third quartile of time until receiving the last byte in
+milliseconds.</li>
+
+<li><b>timeouts:</b> Number of timeouts that occurred when attempting to
+download the static file over Tor.</li>
+
+<li><b>failures:</b> Number of failures that occurred when attempting to
+download the static file over Tor.</li>
+
+<li><b>requests:</b> Total number of requests made to download the static file
+over Tor.</li>
+
+</ul>
+
+</div>
+
+<div class="container">
 <h2>Fraction of connections used uni-/bidirectionally <a href="#connbidirect2" name="connbidirect2" class="anchor">#</a></h2>
 
 <p>The following data file contains statistics on the fraction of direct



More information about the tor-commits mailing list