[tor-commits] [metrics-lib/master] Parse partial download times from Onionperf files.

karsten at torproject.org karsten at torproject.org
Thu May 14 07:33:17 UTC 2020


commit ba6c63a707ceb7acc6354f541a8efd9366842c53
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Thu May 14 09:23:12 2020 +0200

    Parse partial download times from Onionperf files.
    
    Implements #26673.
---
 CHANGELOG.md                                       |   6 +++-
 .../org/torproject/descriptor/TorperfResult.java   |   9 ++++++
 .../descriptor/impl/TorperfResultImpl.java         |  34 +++++++++++++++++++++
 .../onionperf/OnionPerfAnalysisConverter.java      |   9 ++++++
 .../onionperf/ParsedOnionPerfAnalysis.java         |   5 +++
 .../onionperf/OnionPerfAnalysisConverterTest.java  |  13 ++++++--
 .../resources/onionperf/onionperf.analysis.json.xz | Bin 17376 -> 17420 bytes
 7 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5ed2425..ac7ef89 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,8 @@
-# Changes in version 2.??.? - 2020-??-??
+# Changes in version 2.13.0 - 2020-??-??
+
+ * Medium changes
+   - Extend Torperf results to provide partial download times for 10,
+     20, 50, 100, 200, and 500 KiB as well as 1, 2, and 5 MiB.
 
  * Minor changes
    - Include previously unknown error codes in Torperf results
diff --git a/src/main/java/org/torproject/descriptor/TorperfResult.java b/src/main/java/org/torproject/descriptor/TorperfResult.java
index 961a206..695be01 100644
--- a/src/main/java/org/torproject/descriptor/TorperfResult.java
+++ b/src/main/java/org/torproject/descriptor/TorperfResult.java
@@ -136,6 +136,15 @@ public interface TorperfResult extends Descriptor {
   Boolean didTimeout();
 
   /**
+   * Return the times in milliseconds since the epoch when the given number of
+   * bytes were read, or null if the torperf line didn't contain that
+   * information.
+   *
+   * @since 2.13.0
+   */
+  SortedMap<Integer, Long> getPartials();
+
+  /**
    * Return the times in milliseconds since the epoch when {@code x%} of
    * expected bytes were read for {@code 0 <= x <= 100}, or null if the
    * torperf line didn't contain that information.
diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
index f8879d7..b2c58cc 100644
--- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
@@ -185,6 +185,8 @@ public class TorperfResultImpl extends DescriptorImpl
         default:
           if (key.startsWith("DATAPERC")) {
             this.parseDataPercentile(value, keyValue, line);
+          } else if (key.startsWith("PARTIAL")) {
+            this.parsePartial(value, keyValue, line);
           } else {
             if (this.unrecognizedKeys == null) {
               this.unrecognizedKeys = new TreeMap<>();
@@ -313,6 +315,31 @@ public class TorperfResultImpl extends DescriptorImpl
     }
   }
 
+  private void parsePartial(String value, String keyValue, String line)
+      throws DescriptorParseException {
+    String key = keyValue.substring(0, keyValue.indexOf("="));
+    String bytesString = key.substring("PARTIAL".length());
+    int bytes;
+    try {
+      bytes = Integer.parseInt(bytesString);
+    } catch (NumberFormatException e) {
+      /* Treat key as unrecognized below. */
+      bytes = -1;
+    }
+    if (bytes < 0) {
+      if (this.unrecognizedKeys == null) {
+        this.unrecognizedKeys = new TreeMap<>();
+      }
+      this.unrecognizedKeys.put(key, value);
+    } else {
+      long timestamp = this.parseTimestamp(value, keyValue, line);
+      if (this.partials == null) {
+        this.partials = new TreeMap<>();
+      }
+      this.partials.put(bytes, timestamp);
+    }
+  }
+
   private void parseDataPercentile(String value, String keyValue,
       String line) throws DescriptorParseException {
     String key = keyValue.substring(0, keyValue.indexOf("="));
@@ -564,6 +591,13 @@ public class TorperfResultImpl extends DescriptorImpl
     return this.didTimeout;
   }
 
+  private SortedMap<Integer, Long> partials;
+
+  @Override
+  public SortedMap<Integer, Long> getPartials() {
+    return this.partials == null ? null : new TreeMap<>(this.partials);
+  }
+
   private SortedMap<Integer, Long> dataPercentiles;
 
   @Override
diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
index 8ca5efd..5e7e683 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
@@ -265,6 +265,15 @@ public class OnionPerfAnalysisConverter {
           transfer.elapsedSeconds.command);
       torperfResultsBuilder.addTimestamp("DATARESPONSE", transfer.unixTsStart,
           transfer.elapsedSeconds.response);
+      if (null != transfer.elapsedSeconds.payloadBytes) {
+        for (Map.Entry<String, Double> payloadBytesEntry
+            : transfer.elapsedSeconds.payloadBytes.entrySet()) {
+          String key = String.format("PARTIAL%s", payloadBytesEntry.getKey());
+          Double elapsedSeconds = payloadBytesEntry.getValue();
+          torperfResultsBuilder.addTimestamp(key, transfer.unixTsStart,
+              elapsedSeconds);
+        }
+      }
       if (null != transfer.elapsedSeconds.payloadProgress) {
         for (Map.Entry<String, Double> payloadProgressEntry
             : transfer.elapsedSeconds.payloadProgress.entrySet()) {
diff --git a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
index 679879e..4eca0ff 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
@@ -200,6 +200,11 @@ public class ParsedOnionPerfAnalysis {
     Double lastByte;
 
     /**
+     * Time until the given number of bytes were read.
+     */
+    Map<String, Double> payloadBytes;
+
+    /**
      * Time until the given fraction of expected bytes were read.
      */
     Map<String, Double> payloadProgress;
diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
index 7a1a3a0..51e0896 100644
--- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
+++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
@@ -32,7 +32,10 @@ public class OnionPerfAnalysisConverterTest {
       + "ENDPOINTPROXY=localhost:127.0.0.1:35900 "
       + "ENDPOINTREMOTE=m3eahz7co6lzi6jn.onion:0.0.0.0:443 FILESIZE=1048576 "
       + "HOSTNAMELOCAL=op-nl2 HOSTNAMEREMOTE=op-nl2 LAUNCH=1587991281.38 "
-      + "NEGOTIATE=1587991280.37 "
+      + "NEGOTIATE=1587991280.37 PARTIAL10240=1587991283.81 "
+      + "PARTIAL102400=1587991284.15 PARTIAL1048576=1587991286.62 "
+      + "PARTIAL20480=1587991283.81 PARTIAL204800=1587991284.38 "
+      + "PARTIAL51200=1587991283.81 PARTIAL512000=1587991285.14 "
       + "PATH=$970F0966DAA7EBDEE44E3772045527A6854E997B,"
       + "$8101421BEFCCF4C271D5483C5AABCAAD245BBB9D,"
       + "$1A7A2516A961F2838F7F94786A8811BE82F9CFFE READBYTES=1048643 "
@@ -54,6 +57,10 @@ public class OnionPerfAnalysisConverterTest {
       + "ENDPOINTREMOTE=3czoq6qyehjio6lcdo4tb4vk5uv2bm4gfk5iacnawza22do6klsj7wy"
       + "d.onion:0.0.0.0:443 FILESIZE=1048576 HOSTNAMELOCAL=op-nl2 "
       + "HOSTNAMEREMOTE=op-nl2 LAUNCH=1587991881.70 NEGOTIATE=1587991880.37 "
+      + "PARTIAL10240=1587991910.74 PARTIAL102400=1587991913.71 "
+      + "PARTIAL1048576=1587991927.74 PARTIAL20480=1587991910.74 "
+      + "PARTIAL204800=1587991916.00 PARTIAL51200=1587991910.74 "
+      + "PARTIAL512000=1587991921.80 "
       + "PATH=$D5C6F62A5D1B3C711CA5E6F9D3772A432E96F6C2,"
       + "$94EC34B871936504BE70671B44760BC99242E1F3,"
       + "$E0F638ECCE918B5455CE29D2CD9ECC9DBD8F8B21 READBYTES=1048643 "
@@ -96,7 +103,9 @@ public class OnionPerfAnalysisConverterTest {
       String formattedTorperfResult
           = new String(descriptor.getRawDescriptorBytes()).trim();
       assertNotNull(formattedTorperfResult);
-      assertTrue(formattedTorperfResult.equals(torperfResultTransfer1m1)
+      assertTrue(String.format("Unrecognized formatted Torperf result: %s",
+          formattedTorperfResult),
+          formattedTorperfResult.equals(torperfResultTransfer1m1)
           || formattedTorperfResult.equals(torperfResultTransfer1m3)
           || formattedTorperfResult.equals(torperfResultTransfer50k2));
     }
diff --git a/src/test/resources/onionperf/onionperf.analysis.json.xz b/src/test/resources/onionperf/onionperf.analysis.json.xz
index 41a4fa1..08162a1 100644
Binary files a/src/test/resources/onionperf/onionperf.analysis.json.xz and b/src/test/resources/onionperf/onionperf.analysis.json.xz differ



More information about the tor-commits mailing list