[tor-commits] [metrics-web/master] Remove code to produce the deprecated torperf.csv.

karsten at torproject.org karsten at torproject.org
Fri Jun 30 13:45:08 UTC 2017


commit ace813557e36a24635dfe28847dea631147487e0
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Fri Jun 30 10:11:16 2017 +0200

    Remove code to produce the deprecated torperf.csv.
    
    Implements #22780.
---
 modules/legacy/config.template                     |   6 -
 .../org/torproject/ernie/cron/Configuration.java   |  17 --
 .../main/java/org/torproject/ernie/cron/Main.java  |   7 -
 .../ernie/cron/performance/TorperfProcessor.java   | 296 ---------------------
 website/src/main/resources/web/WEB-INF/stats.jsp   |   2 +-
 5 files changed, 1 insertion(+), 327 deletions(-)

diff --git a/modules/legacy/config.template b/modules/legacy/config.template
index 743fe1e..afa8f2d 100644
--- a/modules/legacy/config.template
+++ b/modules/legacy/config.template
@@ -41,9 +41,3 @@
 ## Write bridge stats to disk
 #WriteBridgeStats 0
 #
-## Import torperf data, if available, and write stats to disk
-#ImportWriteTorperfStats 0
-#
-## Relative path to directory to import torperf results from
-#TorperfDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/torperf/
-#
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java
index 12409cd..35d52b9 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java
@@ -44,10 +44,6 @@ public class Configuration {
 
   private boolean writeBridgeStats = false;
 
-  private boolean importWriteTorperfStats = false;
-
-  private String torperfDirectory = "in/torperf/";
-
   private String exoneraTorDatabaseJdbc = "jdbc:postgresql:"
       + "//localhost/exonerator?user=metrics&password=password";
 
@@ -100,11 +96,6 @@ public class Configuration {
         } else if (line.startsWith("WriteBridgeStats")) {
           this.writeBridgeStats = Integer.parseInt(
               line.split(" ")[1]) != 0;
-        } else if (line.startsWith("ImportWriteTorperfStats")) {
-          this.importWriteTorperfStats = Integer.parseInt(
-              line.split(" ")[1]) != 0;
-        } else if (line.startsWith("TorperfDirectory")) {
-          this.torperfDirectory = line.split(" ")[1];
         } else if (line.startsWith("ExoneraTorDatabaseJdbc")) {
           this.exoneraTorDatabaseJdbc = line.split(" ")[1];
         } else if (line.startsWith("ExoneraTorImportDirectory")) {
@@ -187,14 +178,6 @@ public class Configuration {
     return this.writeBridgeStats;
   }
 
-  public boolean getImportWriteTorperfStats() {
-    return this.importWriteTorperfStats;
-  }
-
-  public String getTorperfDirectory() {
-    return this.torperfDirectory;
-  }
-
   public String getExoneraTorDatabaseJdbc() {
     return this.exoneraTorDatabaseJdbc;
   }
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java
index 0eab86f..180031a 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java
@@ -4,7 +4,6 @@
 package org.torproject.ernie.cron;
 
 import org.torproject.ernie.cron.network.ConsensusStatsFileHandler;
-import org.torproject.ernie.cron.performance.TorperfProcessor;
 
 import java.io.File;
 import java.util.logging.Logger;
@@ -75,12 +74,6 @@ public class Main {
       csfh = null;
     }
 
-    // Import and process torperf stats
-    if (config.getImportWriteTorperfStats()) {
-      new TorperfProcessor(new File(config.getTorperfDirectory()),
-          statsDirectory);
-    }
-
     // Remove lock file
     lf.releaseLock();
 
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
deleted file mode 100644
index b5dae12..0000000
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/* Copyright 2011--2017 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.ernie.cron.performance;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorReader;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.TorperfResult;
-import org.torproject.descriptor.UnparseableDescriptor;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class TorperfProcessor {
-
-  /** Processes Torperf data from the given directory and writes
-   * aggregates statistics to the given stats directory. */
-  public TorperfProcessor(File torperfDirectory, File statsDirectory) {
-
-    if (torperfDirectory == null || statsDirectory == null) {
-      throw new IllegalArgumentException();
-    }
-
-    Logger logger = Logger.getLogger(TorperfProcessor.class.getName());
-    File rawFile = new File(statsDirectory, "torperf-raw");
-    File statsFile = new File(statsDirectory, "torperf.csv");
-    SortedMap<String, String> rawObs = new TreeMap<>();
-    SortedMap<String, String> stats = new TreeMap<>();
-    int addedRawObs = 0;
-    SimpleDateFormat formatter =
-        new SimpleDateFormat("yyyy-MM-dd,HH:mm:ss");
-    formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
-    try {
-      if (rawFile.exists()) {
-        logger.fine("Reading file " + rawFile.getAbsolutePath() + "...");
-        BufferedReader br = new BufferedReader(new FileReader(rawFile));
-        String line = br.readLine(); // ignore header
-        while ((line = br.readLine()) != null) {
-          if (line.split(",").length != 4) {
-            logger.warning("Corrupt line in " + rawFile.getAbsolutePath()
-                + "!");
-            break;
-          }
-          String key = line.substring(0, line.lastIndexOf(","));
-          rawObs.put(key, line);
-        }
-        br.close();
-        logger.fine("Finished reading file " + rawFile.getAbsolutePath()
-            + ".");
-      }
-      if (statsFile.exists()) {
-        logger.fine("Reading file " + statsFile.getAbsolutePath()
-            + "...");
-        BufferedReader br = new BufferedReader(new FileReader(statsFile));
-        String line = br.readLine(); // ignore header
-        while ((line = br.readLine()) != null) {
-          String[] parts = line.split(",");
-          String key = String.format("%s,%s,%s", parts[0], parts[1],
-              parts[2]);
-          stats.put(key, line);
-        }
-        br.close();
-        logger.fine("Finished reading file " + statsFile.getAbsolutePath()
-            + ".");
-      }
-      if (torperfDirectory.exists()) {
-        logger.fine("Importing files in " + torperfDirectory + "/...");
-        DescriptorReader descriptorReader =
-            DescriptorSourceFactory.createDescriptorReader();
-        File historyFile = new File(statsDirectory, "torperf-history");
-        descriptorReader.setHistoryFile(historyFile);
-        for (Descriptor descriptor : descriptorReader.readDescriptors(
-            torperfDirectory)) {
-          if (descriptor instanceof UnparseableDescriptor) {
-            logger.log(Level.FINE, "Error parsing descriptor.",
-                ((UnparseableDescriptor) descriptor)
-                .getDescriptorParseException());
-            continue;
-          } else if (!(descriptor instanceof TorperfResult)) {
-            continue;
-          }
-          TorperfResult result = (TorperfResult) descriptor;
-          if (null != result.getUnrecognizedKeys()
-              && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE")
-              && result.getUnrecognizedKeys().get("ENDPOINTREMOTE")
-              .contains(".onion")) {
-            continue;
-          }
-          String source = result.getSource();
-          long fileSize = result.getFileSize();
-          if (fileSize == 51200) {
-            source += "-50kb";
-          } else if (fileSize == 1048576) {
-            source += "-1mb";
-          } else if (fileSize == 5242880) {
-            source += "-5mb";
-          } else {
-            logger.fine("Unexpected file size '" + fileSize
-                + "'.  Skipping.");
-            continue;
-          }
-          String dateTime = formatter.format(result.getStartMillis());
-          long completeMillis = result.getDataCompleteMillis()
-              - result.getStartMillis();
-          String key = source + "," + dateTime;
-          String value = key;
-          if ((result.didTimeout() == null
-              && result.getDataCompleteMillis() < 1)
-              || (result.didTimeout() != null && result.didTimeout())) {
-            value += ",-2"; // -2 for timeout
-          } else if (result.getReadBytes() < fileSize) {
-            value += ",-1"; // -1 for failure
-          } else {
-            value += "," + completeMillis;
-          }
-          if (!rawObs.containsKey(key)) {
-            rawObs.put(key, value);
-            addedRawObs++;
-          }
-        }
-        descriptorReader.saveHistoryFile(historyFile);
-        logger.fine("Finished importing files in " + torperfDirectory
-            + "/.");
-      }
-      if (rawObs.size() > 0) {
-        logger.fine("Writing file " + rawFile.getAbsolutePath() + "...");
-        rawFile.getParentFile().mkdirs();
-        BufferedWriter bw = new BufferedWriter(new FileWriter(rawFile));
-        bw.append("source,date,start,completemillis\n");
-        String tempSourceDate = null;
-        Iterator<Map.Entry<String, String>> it =
-            rawObs.entrySet().iterator();
-        List<Long> dlTimes = new ArrayList<>();
-        boolean haveWrittenFinalLine = false;
-        SortedMap<String, List<Long>> dlTimesAllSources = new TreeMap<>();
-        SortedMap<String, long[]> statusesAllSources = new TreeMap<>();
-        long failures = 0;
-        long timeouts = 0;
-        long requests = 0;
-        while (it.hasNext() || !haveWrittenFinalLine) {
-          Map.Entry<String, String> next =
-              it.hasNext() ? it.next() : null;
-          if (tempSourceDate != null
-              && (next == null || !(next.getValue().split(",")[0] + ","
-              + next.getValue().split(",")[1]).equals(tempSourceDate))) {
-            if (dlTimes.size() > 4) {
-              Collections.sort(dlTimes);
-              long q1 = dlTimes.get(dlTimes.size() / 4 - 1);
-              long md = dlTimes.get(dlTimes.size() / 2 - 1);
-              long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1);
-              String[] tempParts = tempSourceDate.split("[,]", 3);
-              String tempDate = tempParts[1];
-              int tempSize = Integer.parseInt(tempParts[0].substring(
-                  tempParts[0].lastIndexOf('-') + 1,
-                  tempParts[0].length() - 2))
-                  * 1024 * (tempParts[0].endsWith("mb") ? 1024 : 1);
-              String tempSource = tempParts[0].substring(0,
-                  tempParts[0].lastIndexOf('-'));
-              String tempDateSizeSource = String.format("%s,%d,%s",
-                  tempDate, tempSize, tempSource);
-              stats.put(tempDateSizeSource,
-                  String.format("%s,%s,%s,%s,%s,%s,%s",
-                  tempDateSizeSource, q1, md, q3, timeouts, failures,
-                  requests));
-              String allDateSizeSource = String.format("%s,%d,",
-                  tempDate, tempSize);
-              if (dlTimesAllSources.containsKey(allDateSizeSource)) {
-                dlTimesAllSources.get(allDateSizeSource).addAll(dlTimes);
-              } else {
-                dlTimesAllSources.put(allDateSizeSource, dlTimes);
-              }
-              if (statusesAllSources.containsKey(allDateSizeSource)) {
-                long[] status = statusesAllSources.get(allDateSizeSource);
-                status[0] += timeouts;
-                status[1] += failures;
-                status[2] += requests;
-              } else {
-                long[] status = new long[3];
-                status[0] = timeouts;
-                status[1] = failures;
-                status[2] = requests;
-                statusesAllSources.put(allDateSizeSource, status);
-              }
-            }
-            dlTimes = new ArrayList<>();
-            failures = timeouts = requests = 0;
-            if (next == null) {
-              haveWrittenFinalLine = true;
-            }
-          }
-          if (next != null) {
-            bw.append(next.getValue() + "\n");
-            String[] parts = next.getValue().split(",");
-            tempSourceDate = parts[0] + "," + parts[1];
-            long completeMillis = Long.parseLong(parts[3]);
-            if (completeMillis == -2L) {
-              timeouts++;
-            } else if (completeMillis == -1L) {
-              failures++;
-            } else {
-              dlTimes.add(Long.parseLong(parts[3]));
-            }
-            requests++;
-          }
-        }
-        bw.close();
-        for (Map.Entry<String, List<Long>> e
-            : dlTimesAllSources.entrySet()) {
-          String allDateSizeSource = e.getKey();
-          dlTimes = e.getValue();
-          Collections.sort(dlTimes);
-          long[] status = statusesAllSources.get(allDateSizeSource);
-          timeouts = status[0];
-          failures = status[1];
-          requests = status[2];
-          long q1 = dlTimes.get(dlTimes.size() / 4 - 1);
-          long md = dlTimes.get(dlTimes.size() / 2 - 1);
-          long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1);
-          stats.put(allDateSizeSource,
-              String.format("%s,%s,%s,%s,%s,%s,%s",
-              allDateSizeSource, q1, md, q3, timeouts, failures,
-              requests));
-        }
-        logger.fine("Finished writing file " + rawFile.getAbsolutePath()
-            + ".");
-      }
-      if (stats.size() > 0) {
-        logger.fine("Writing file " + statsFile.getAbsolutePath()
-            + "...");
-        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-        dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-        String yesterday = dateFormat.format(System.currentTimeMillis()
-            - 86400000L);
-        statsFile.getParentFile().mkdirs();
-        BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile));
-        bw.append("date,size,source,q1,md,q3,timeouts,failures,"
-            + "requests\n");
-        for (String s : stats.values()) {
-          if (s.compareTo(yesterday) < 0) {
-            bw.append(s + "\n");
-          }
-        }
-        bw.close();
-        logger.fine("Finished writing file " + statsFile.getAbsolutePath()
-            + ".");
-      }
-    } catch (IOException e) {
-      logger.log(Level.WARNING, "Failed writing "
-          + rawFile.getAbsolutePath() + " or "
-          + statsFile.getAbsolutePath() + "!", e);
-    }
-
-    /* Write stats. */
-    StringBuilder dumpStats = new StringBuilder("Finished writing "
-        + "statistics on torperf results.\nAdded " + addedRawObs
-        + " new observations in this execution.\n"
-        + "Last known obserations by source and file size are:");
-    String lastSource = null;
-    String lastLine = null;
-    for (String s : rawObs.keySet()) {
-      String[] parts = s.split(",");
-      if (lastSource == null) {
-        lastSource = parts[0];
-      } else if (!parts[0].equals(lastSource)) {
-        String lastKnownObservation = lastLine.split(",")[1] + " "
-            + lastLine.split(",")[2];
-        dumpStats.append("\n" + lastSource + " " + lastKnownObservation);
-        lastSource = parts[0];
-      }
-      lastLine = s;
-    }
-    if (lastSource != null) {
-      String lastKnownObservation = lastLine.split(",")[1] + " "
-          + lastLine.split(",")[2];
-      dumpStats.append("\n" + lastSource + " " + lastKnownObservation);
-    }
-    logger.info(dumpStats.toString());
-  }
-}
-
diff --git a/website/src/main/resources/web/WEB-INF/stats.jsp b/website/src/main/resources/web/WEB-INF/stats.jsp
index 90b9c3b..2d5a69d 100644
--- a/website/src/main/resources/web/WEB-INF/stats.jsp
+++ b/website/src/main/resources/web/WEB-INF/stats.jsp
@@ -310,7 +310,7 @@ there are users by either of the two numbers.</li>
 
 <div class="bs-callout bs-callout-warning">
 <h3>Deprecated</h3>
-<p>As of May 12, 2017, the linked data file has been replaced by <a href="#torperf-1.1">this data file</a>.  Starting on June 1, 2017, the linked data file will not be updated anymore.  This section and the linked data file will be removed in the future.</p>
+<p>As of May 12, 2017, the linked data file has been replaced by <a href="#torperf-1.1">this data file</a>.  As of June 30, 2017, the linked data file is not updated anymore.  This section and the linked data file will be removed in the future.</p>
 </div>
 
 <p>The following data file contains aggregate statistics on performance when





More information about the tor-commits mailing list