commit ba6c63a707ceb7acc6354f541a8efd9366842c53 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu May 14 09:23:12 2020 +0200 Parse partial download times from Onionperf files. Implements #26673. --- CHANGELOG.md | 6 +++- .../org/torproject/descriptor/TorperfResult.java | 9 ++++++ .../descriptor/impl/TorperfResultImpl.java | 34 +++++++++++++++++++++ .../onionperf/OnionPerfAnalysisConverter.java | 9 ++++++ .../onionperf/ParsedOnionPerfAnalysis.java | 5 +++ .../onionperf/OnionPerfAnalysisConverterTest.java | 13 ++++++-- .../resources/onionperf/onionperf.analysis.json.xz | Bin 17376 -> 17420 bytes 7 files changed, 73 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed2425..ac7ef89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -# Changes in version 2.??.? - 2020-??-?? +# Changes in version 2.13.0 - 2020-??-?? + + * Medium changes + - Extend Torperf results to provide partial download times for 10, + 20, 50, 100, 200, and 500 KiB as well as 1, 2, and 5 MiB. * Minor changes - Include previously unknown error codes in Torperf results diff --git a/src/main/java/org/torproject/descriptor/TorperfResult.java b/src/main/java/org/torproject/descriptor/TorperfResult.java index 961a206..695be01 100644 --- a/src/main/java/org/torproject/descriptor/TorperfResult.java +++ b/src/main/java/org/torproject/descriptor/TorperfResult.java @@ -136,6 +136,15 @@ public interface TorperfResult extends Descriptor { Boolean didTimeout(); /** + * Return the times in milliseconds since the epoch when the given number of + * bytes were read, or null if the torperf line didn't contain that + * information. + * + * @since 2.13.0 + */ + SortedMap<Integer, Long> getPartials(); + + /** * Return the times in milliseconds since the epoch when {@code x%} of * expected bytes were read for {@code 0 <= x <= 100}, or null if the * torperf line didn't contain that information. diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java index f8879d7..b2c58cc 100644 --- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -185,6 +185,8 @@ public class TorperfResultImpl extends DescriptorImpl default: if (key.startsWith("DATAPERC")) { this.parseDataPercentile(value, keyValue, line); + } else if (key.startsWith("PARTIAL")) { + this.parsePartial(value, keyValue, line); } else { if (this.unrecognizedKeys == null) { this.unrecognizedKeys = new TreeMap<>(); @@ -313,6 +315,31 @@ public class TorperfResultImpl extends DescriptorImpl } } + private void parsePartial(String value, String keyValue, String line) + throws DescriptorParseException { + String key = keyValue.substring(0, keyValue.indexOf("=")); + String bytesString = key.substring("PARTIAL".length()); + int bytes; + try { + bytes = Integer.parseInt(bytesString); + } catch (NumberFormatException e) { + /* Treat key as unrecognized below. */ + bytes = -1; + } + if (bytes < 0) { + if (this.unrecognizedKeys == null) { + this.unrecognizedKeys = new TreeMap<>(); + } + this.unrecognizedKeys.put(key, value); + } else { + long timestamp = this.parseTimestamp(value, keyValue, line); + if (this.partials == null) { + this.partials = new TreeMap<>(); + } + this.partials.put(bytes, timestamp); + } + } + private void parseDataPercentile(String value, String keyValue, String line) throws DescriptorParseException { String key = keyValue.substring(0, keyValue.indexOf("=")); @@ -564,6 +591,13 @@ public class TorperfResultImpl extends DescriptorImpl return this.didTimeout; } + private SortedMap<Integer, Long> partials; + + @Override + public SortedMap<Integer, Long> getPartials() { + return this.partials == null ? null : new TreeMap<>(this.partials); + } + private SortedMap<Integer, Long> dataPercentiles; @Override diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java index 8ca5efd..5e7e683 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java +++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java @@ -265,6 +265,15 @@ public class OnionPerfAnalysisConverter { transfer.elapsedSeconds.command); torperfResultsBuilder.addTimestamp("DATARESPONSE", transfer.unixTsStart, transfer.elapsedSeconds.response); + if (null != transfer.elapsedSeconds.payloadBytes) { + for (Map.Entry<String, Double> payloadBytesEntry + : transfer.elapsedSeconds.payloadBytes.entrySet()) { + String key = String.format("PARTIAL%s", payloadBytesEntry.getKey()); + Double elapsedSeconds = payloadBytesEntry.getValue(); + torperfResultsBuilder.addTimestamp(key, transfer.unixTsStart, + elapsedSeconds); + } + } if (null != transfer.elapsedSeconds.payloadProgress) { for (Map.Entry<String, Double> payloadProgressEntry : transfer.elapsedSeconds.payloadProgress.entrySet()) { diff --git a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java index 679879e..4eca0ff 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java +++ b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java @@ -200,6 +200,11 @@ public class ParsedOnionPerfAnalysis { Double lastByte; /** + * Time until the given number of bytes were read. + */ + Map<String, Double> payloadBytes; + + /** * Time until the given fraction of expected bytes were read. */ Map<String, Double> payloadProgress; diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java index 7a1a3a0..51e0896 100644 --- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java +++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java @@ -32,7 +32,10 @@ public class OnionPerfAnalysisConverterTest { + "ENDPOINTPROXY=localhost:127.0.0.1:35900 " + "ENDPOINTREMOTE=m3eahz7co6lzi6jn.onion:0.0.0.0:443 FILESIZE=1048576 " + "HOSTNAMELOCAL=op-nl2 HOSTNAMEREMOTE=op-nl2 LAUNCH=1587991281.38 " - + "NEGOTIATE=1587991280.37 " + + "NEGOTIATE=1587991280.37 PARTIAL10240=1587991283.81 " + + "PARTIAL102400=1587991284.15 PARTIAL1048576=1587991286.62 " + + "PARTIAL20480=1587991283.81 PARTIAL204800=1587991284.38 " + + "PARTIAL51200=1587991283.81 PARTIAL512000=1587991285.14 " + "PATH=$970F0966DAA7EBDEE44E3772045527A6854E997B," + "$8101421BEFCCF4C271D5483C5AABCAAD245BBB9D," + "$1A7A2516A961F2838F7F94786A8811BE82F9CFFE READBYTES=1048643 " @@ -54,6 +57,10 @@ public class OnionPerfAnalysisConverterTest { + "ENDPOINTREMOTE=3czoq6qyehjio6lcdo4tb4vk5uv2bm4gfk5iacnawza22do6klsj7wy" + "d.onion:0.0.0.0:443 FILESIZE=1048576 HOSTNAMELOCAL=op-nl2 " + "HOSTNAMEREMOTE=op-nl2 LAUNCH=1587991881.70 NEGOTIATE=1587991880.37 " + + "PARTIAL10240=1587991910.74 PARTIAL102400=1587991913.71 " + + "PARTIAL1048576=1587991927.74 PARTIAL20480=1587991910.74 " + + "PARTIAL204800=1587991916.00 PARTIAL51200=1587991910.74 " + + "PARTIAL512000=1587991921.80 " + "PATH=$D5C6F62A5D1B3C711CA5E6F9D3772A432E96F6C2," + "$94EC34B871936504BE70671B44760BC99242E1F3," + "$E0F638ECCE918B5455CE29D2CD9ECC9DBD8F8B21 READBYTES=1048643 " @@ -96,7 +103,9 @@ public class OnionPerfAnalysisConverterTest { String formattedTorperfResult = new String(descriptor.getRawDescriptorBytes()).trim(); assertNotNull(formattedTorperfResult); - assertTrue(formattedTorperfResult.equals(torperfResultTransfer1m1) + assertTrue(String.format("Unrecognized formatted Torperf result: %s", + formattedTorperfResult), + formattedTorperfResult.equals(torperfResultTransfer1m1) || formattedTorperfResult.equals(torperfResultTransfer1m3) || formattedTorperfResult.equals(torperfResultTransfer50k2)); } diff --git a/src/test/resources/onionperf/onionperf.analysis.json.xz b/src/test/resources/onionperf/onionperf.analysis.json.xz index 41a4fa1..08162a1 100644 Binary files a/src/test/resources/onionperf/onionperf.analysis.json.xz and b/src/test/resources/onionperf/onionperf.analysis.json.xz differ