commit b44c6bccb46ca1eb9c861a84dfcd7d5d2047dee4 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Jul 16 21:37:19 2020 +0200

Parse OnionPerf analysis results format v3.0.

Implements tpo/metrics/library#40001. --- CHANGELOG.md | 1 + .../onionperf/OnionPerfAnalysisConverter.java | 235 ++++++++++++++++----- .../onionperf/ParsedOnionPerfAnalysis.java | 206 ++++++++++++++++++ .../onionperf/TorperfResultsBuilder.java | 38 ++++ .../onionperf/OnionPerfAnalysisConverterTest.java | 80 ++++++- ...20-07-13.op-nl-test1.onionperf.analysis.json.xz | Bin 0 -> 1736 bytes 6 files changed, 499 insertions(+), 61 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bb22c4..4679688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@

* Medium changes - Extend Torperf results to provide error codes. + - Parse OnionPerf analysis results format version 3.0.

# Changes in version 2.13.0 - 2020-05-16 diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java index 61fd173..f1b9d11 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java +++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java @@ -69,25 +69,32 @@ public class OnionPerfAnalysisConverter { * Torperf results. */ public List<Descriptor> asTorperfResults() throws DescriptorParseException { - ParsedOnionPerfAnalysis parsedOnionPerfAnalysis; + ParsedOnionPerfAnalysis parsedOnionPerfAnalysis + = this.parseOnionPerfAnalysis(); + this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis); + StringBuilder formattedTorperfResults + = this.formatTorperfResults(parsedOnionPerfAnalysis); + this.parseFormattedTorperfResults(formattedTorperfResults); + return this.convertedTorperfResults; + } + + /** + * Parse the OnionPerf analysis JSON document. + */ + private ParsedOnionPerfAnalysis parseOnionPerfAnalysis() + throws DescriptorParseException { try { InputStream compressedInputStream = new ByteArrayInputStream( this.rawDescriptorBytes); InputStream decompressedInputStream = new XZCompressorInputStream( compressedInputStream); byte[] decompressedBytes = IOUtils.toByteArray(decompressedInputStream); - parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes( - decompressedBytes); + return ParsedOnionPerfAnalysis.fromBytes(decompressedBytes); } catch (IOException ioException) { throw new DescriptorParseException("Ran into an I/O error while " + "attempting to parse an OnionPerf analysis document.", ioException); } - this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis); - StringBuilder formattedTorperfResults - = this.formatTorperfResults(parsedOnionPerfAnalysis); - this.parseFormattedTorperfResults(formattedTorperfResults); - return this.convertedTorperfResults; }

/** @@ -109,9 +116,9 @@ public class OnionPerfAnalysisConverter { throw new DescriptorParseException("Parsed OnionPerf analysis file does " + "not contain version information."); } else if ((parsedOnionPerfAnalysis.version instanceof Double - && (double) parsedOnionPerfAnalysis.version > 2.999) + && (double) parsedOnionPerfAnalysis.version > 3.999) || (parsedOnionPerfAnalysis.version instanceof String - && ((String) parsedOnionPerfAnalysis.version).compareTo("3.") >= 0)) { + && ((String) parsedOnionPerfAnalysis.version).compareTo("4.") >= 0)) { throw new DescriptorParseException("Parsed OnionPerf analysis file " + "contains unsupported version " + parsedOnionPerfAnalysis.version + "."); @@ -131,8 +138,7 @@ public class OnionPerfAnalysisConverter { : parsedOnionPerfAnalysis.data.entrySet()) { String nickname = data.getKey(); ParsedOnionPerfAnalysis.MeasurementData measurements = data.getValue(); - if (null == measurements.measurementIp || null == measurements.tgen - || null == measurements.tgen.transfers) { + if (null == measurements.tgen) { continue; } String measurementIp = measurements.measurementIp; @@ -153,57 +159,69 @@ public class OnionPerfAnalysisConverter { } } } - for (ParsedOnionPerfAnalysis.Transfer transfer - : measurements.tgen.transfers.values()) { - if (null == transfer.endpointLocal) { - continue; - } - String[] endpointLocalParts = transfer.endpointLocal.split(":"); - if (endpointLocalParts.length < 3) { - continue; - } - TorperfResultsBuilder torperfResultsBuilder - = new TorperfResultsBuilder(); - - torperfResultsBuilder.addString("SOURCE", nickname); - torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp); - this.formatTransferParts(torperfResultsBuilder, transfer); - List<String> errorCodeParts = null; - if (transfer.isError) { - errorCodeParts = new ArrayList<>(); - if ("PROXY".equals(transfer.errorCode)) { - errorCodeParts.add("TOR"); - } else { - errorCodeParts.add("TGEN"); - errorCodeParts.add(transfer.errorCode); + if (null != measurements.tgen.transfers) { + for (ParsedOnionPerfAnalysis.Transfer transfer + : measurements.tgen.transfers.values()) { + TorperfResultsBuilder torperfResultsBuilder + = new TorperfResultsBuilder(); + torperfResultsBuilder.addString("SOURCE", nickname); + torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp); + this.formatTransferParts(torperfResultsBuilder, transfer); + if (null != transfer.endpointLocal) { + String[] endpointLocalParts = transfer.endpointLocal.split(":"); + if (endpointLocalParts.length >= 3) { + String sourcePort = endpointLocalParts[2]; + if (streamsBySourcePort.containsKey(sourcePort)) { + for (ParsedOnionPerfAnalysis.Stream stream + : streamsBySourcePort.get(sourcePort)) { + if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) { + this.formatStreamParts(torperfResultsBuilder, stream); + if (null != stream.circuitId + && circuitsByCircuitId.containsKey(stream.circuitId)) { + ParsedOnionPerfAnalysis.Circuit circuit + = circuitsByCircuitId.get(stream.circuitId); + this.formatCircuitParts(torperfResultsBuilder, circuit); + } + } + } + } + } } + formattedTorperfResults.append(torperfResultsBuilder.build()); } - String sourcePort = endpointLocalParts[2]; - if (streamsBySourcePort.containsKey(sourcePort)) { - for (ParsedOnionPerfAnalysis.Stream stream - : streamsBySourcePort.get(sourcePort)) { - if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) { - if (null != errorCodeParts && null != stream.failureReasonLocal) { - errorCodeParts.add(stream.failureReasonLocal); - if (null != stream.failureReasonRemote) { - errorCodeParts.add(stream.failureReasonRemote); + } + if (null != measurements.tgen.streams) { + for (ParsedOnionPerfAnalysis.TgenStream stream + : measurements.tgen.streams.values()) { + TorperfResultsBuilder torperfResultsBuilder + = new TorperfResultsBuilder(); + torperfResultsBuilder.addString("SOURCE", nickname); + torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp); + this.formatTgenStreamParts(torperfResultsBuilder, stream); + if (null != stream.transportInfo + && null != stream.transportInfo.local) { + String[] endpointLocalParts = stream.transportInfo.local.split(":"); + if (endpointLocalParts.length >= 3) { + String sourcePort = endpointLocalParts[2]; + if (streamsBySourcePort.containsKey(sourcePort)) { + for (ParsedOnionPerfAnalysis.Stream torStream + : streamsBySourcePort.get(sourcePort)) { + if (Math.abs(stream.unixTsEnd + - torStream.unixTsEnd) < 150.0) { + this.formatStreamParts(torperfResultsBuilder, torStream); + if (null != torStream.circuitId && circuitsByCircuitId + .containsKey(torStream.circuitId)) { + ParsedOnionPerfAnalysis.Circuit circuit + = circuitsByCircuitId.get(torStream.circuitId); + this.formatCircuitParts(torperfResultsBuilder, circuit); + } + } } } - if (null != stream.circuitId - && circuitsByCircuitId.containsKey(stream.circuitId)) { - ParsedOnionPerfAnalysis.Circuit circuit - = circuitsByCircuitId.get(stream.circuitId); - this.formatStreamParts(torperfResultsBuilder, stream); - this.formatCircuitParts(torperfResultsBuilder, circuit); - } } } + formattedTorperfResults.append(torperfResultsBuilder.build()); } - if (null != errorCodeParts) { - String errorCode = String.join("/", errorCodeParts); - torperfResultsBuilder.addString("ERRORCODE", errorCode); - } - formattedTorperfResults.append(torperfResultsBuilder.build()); } } return formattedTorperfResults; @@ -288,6 +306,105 @@ public class OnionPerfAnalysisConverter { transfer.elapsedSeconds.lastByte); if (transfer.isError) { torperfResultsBuilder.addInteger("DIDTIMEOUT", 1); + if ("PROXY".equals(transfer.errorCode)) { + torperfResultsBuilder.addErrorCodePart("TOR"); + } else { + torperfResultsBuilder.addErrorCodePart("TGEN"); + torperfResultsBuilder.addErrorCodePart(transfer.errorCode); + } + } + } + } + + /** + * Format relevant tgen stream data as Torperf result key-value pairs. + * + * @param torperfResultsBuilder Torperf results builder to add key-value pairs + * to. + * @param stream Stream data obtained from the parsed OnionPerf analysis file. + */ + private void formatTgenStreamParts( + TorperfResultsBuilder torperfResultsBuilder, + ParsedOnionPerfAnalysis.TgenStream stream) { + torperfResultsBuilder.addString("READBYTES", "0"); + torperfResultsBuilder.addString("WRITEBYTES", "0"); + if (null != stream.byteInfo) { + torperfResultsBuilder.addString("READBYTES", + stream.byteInfo.totalBytesRecv); + torperfResultsBuilder.addString("WRITEBYTES", + stream.byteInfo.totalBytesSend); + } + if (null != stream.streamInfo) { + torperfResultsBuilder.addString("FILESIZE", stream.streamInfo.recvsize); + torperfResultsBuilder.addString("HOSTNAMELOCAL", stream.streamInfo.name); + torperfResultsBuilder.addString("HOSTNAMEREMOTE", + stream.streamInfo.peername); + } + if (null != stream.transportInfo) { + torperfResultsBuilder.addString("ENDPOINTLOCAL", + stream.transportInfo.local); + torperfResultsBuilder.addString("ENDPOINTPROXY", + stream.transportInfo.proxy); + torperfResultsBuilder.addString("ENDPOINTREMOTE", + stream.transportInfo.remote); + } + torperfResultsBuilder.addInteger("DIDTIMEOUT", 0); + + for (String key : new String[] { "START", "SOCKET", "CONNECT", "NEGOTIATE", + "REQUEST", "RESPONSE", "DATAREQUEST", "DATARESPONSE", + "DATACOMPLETE" }) { + torperfResultsBuilder.addString(key, "0.0"); + } + torperfResultsBuilder.addTimestamp("START", stream.unixTsStart, 0.0); + if (null != stream.unixTsStart) { + if (null != stream.timeInfo) { + torperfResultsBuilder.addTimestamp("SOCKET", stream.unixTsStart, + stream.timeInfo.usecsToSocketCreate); + torperfResultsBuilder.addTimestamp("CONNECT", stream.unixTsStart, + stream.timeInfo.usecsToSocketConnect); + torperfResultsBuilder.addTimestamp("NEGOTIATE", stream.unixTsStart, + stream.timeInfo.usecsToProxyChoice); + torperfResultsBuilder.addTimestamp("REQUEST", stream.unixTsStart, + stream.timeInfo.usecsToProxyRequest); + torperfResultsBuilder.addTimestamp("RESPONSE", stream.unixTsStart, + stream.timeInfo.usecsToProxyResponse); + torperfResultsBuilder.addTimestamp("DATAREQUEST", stream.unixTsStart, + stream.timeInfo.usecsToCommand); + torperfResultsBuilder.addTimestamp("DATARESPONSE", stream.unixTsStart, + stream.timeInfo.usecsToResponse); + torperfResultsBuilder.addTimestamp("DATACOMPLETE", stream.unixTsStart, + stream.timeInfo.usecsToLastByteRecv); + } + if (null != stream.elapsedSeconds) { + if (null != stream.elapsedSeconds.payloadBytesRecv) { + for (Map.Entry<String, Double> payloadBytesRecvEntry + : stream.elapsedSeconds.payloadBytesRecv.entrySet()) { + String key = String.format("PARTIAL%s", + payloadBytesRecvEntry.getKey()); + Double elapsedSeconds = payloadBytesRecvEntry.getValue(); + torperfResultsBuilder.addTimestamp(key, stream.unixTsStart, + elapsedSeconds); + } + } + if (null != stream.elapsedSeconds.payloadProgressRecv) { + for (Map.Entry<String, Double> payloadProgressRecvEntry + : stream.elapsedSeconds.payloadProgressRecv.entrySet()) { + String key = String.format("DATAPERC%.0f", + Double.parseDouble(payloadProgressRecvEntry.getKey()) * 100.0); + Double elapsedSeconds = payloadProgressRecvEntry.getValue(); + torperfResultsBuilder.addTimestamp(key, stream.unixTsStart, + elapsedSeconds); + } + } + } + if (null != stream.isError && stream.isError) { + torperfResultsBuilder.addInteger("DIDTIMEOUT", 1); + if ("PROXY".equals(stream.streamInfo.error)) { + torperfResultsBuilder.addErrorCodePart("TOR"); + } else { + torperfResultsBuilder.addErrorCodePart("TGEN"); + torperfResultsBuilder.addErrorCodePart(stream.streamInfo.error); + } } } } @@ -301,6 +418,12 @@ public class OnionPerfAnalysisConverter { */ private void formatStreamParts(TorperfResultsBuilder torperfResultsBuilder, ParsedOnionPerfAnalysis.Stream stream) { + if (null != stream.failureReasonLocal) { + torperfResultsBuilder.addErrorCodePart(stream.failureReasonLocal); + if (null != stream.failureReasonRemote) { + torperfResultsBuilder.addErrorCodePart(stream.failureReasonRemote); + } + } torperfResultsBuilder.addTimestamp("USED_AT", stream.unixTsEnd, 0.0); torperfResultsBuilder.addInteger("USED_BY", stream.streamId); } diff --git a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java index 4eca0ff..e94fdb4 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java +++ b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java @@ -4,6 +4,7 @@ package org.torproject.descriptor.onionperf;

import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -95,6 +96,11 @@ public class ParsedOnionPerfAnalysis { * Measurement data by transfer identifier. */ Map<String, Transfer> transfers; + + /** + * Measurement data by stream identifier. + */ + Map<String, TgenStream> streams; }

/** @@ -240,6 +246,206 @@ public class ParsedOnionPerfAnalysis { Double socketCreate; }

+ /** + * Measurement data related to a single stream obtained from client-side + * {@code tgen} logs. + */ + static class TgenStream { + + /** + * Information on sent and received bytes. + */ + ByteInfo byteInfo; + + /** + * Elapsed seconds until a given number or fraction of payload bytes have + * been received or sent, obtained from {@code [stream-status]}, + * {@code [stream-success]}, and {@code [stream-error]} log messages, only + * included if the measurement was a success. + */ + ElapsedSecondsPayload elapsedSeconds; + + /** + * Whether an error occurred. + */ + Boolean isError; + + /** + * Information about the TGen stream. + */ + StreamInfo streamInfo; + + /** + * Elapsed time until reaching given substeps in a measurement. + */ + TimeInfo timeInfo; + + /** + * Information about the TGen transport. + */ + TransportInfo transportInfo; + + /** + * Initial start time of the measurement, obtained by subtracting the + * largest number of elapsed microseconds in {@code time_info} from + * {@code unix_ts_end}, given in seconds since the epoch. + */ + Double unixTsStart; + + /** + * Final end time of the measurement, obtained from the log time of the + * {@code [stream-success]} or {@code [stream-error]} log message, given in + * seconds since the epoch. + */ + Double unixTsEnd; + } + + /** + * Information on sent and received bytes. + */ + static class ByteInfo { + + /** + * Total number of bytes received. + */ + @JsonProperty("total-bytes-recv") + String totalBytesRecv; + + /** + * Total number of bytes sent. + */ + @JsonProperty("total-bytes-send") + String totalBytesSend; + } + + /** + * Elapsed seconds until a given number or fraction of payload bytes have been + * received or sent, obtained from {@code [stream-status]}, + * {@code [stream-success]}, and {@code [stream-error]} log messages, only + * included if the measurement was a success. + */ + static class ElapsedSecondsPayload { + + /** + * Number of received payload bytes. + */ + Map<String, Double> payloadBytesRecv; + + /** + * Fraction of received payload bytes. + */ + Map<String, Double> payloadProgressRecv; + } + + /** + * Information about the TGen stream. + */ + static class StreamInfo { + + /** + * Error code, or {@code NONE} if no error occurred. + */ + String error; + + /** + * Hostname of the TGen client. + */ + String name; + + /** + * Hostname of the TGen server. + */ + String peername; + + /** + * Number of expected payload bytes in the response. + */ + String recvsize; + } + + /** + * Elapsed time until reaching given substeps in a measurement. + */ + static class TimeInfo { + + /** + * Elapsed microseconds until the TGen client has sent the command to the + * TGen server, or -1 if missing (step 7). + */ + @JsonProperty("usecs-to-command") + String usecsToCommand; + + /** + * Elapsed microseconds until the TGen client has received the last payload + * byte, or -1 if missing (step 10). + */ + @JsonProperty("usecs-to-last-byte-recv") + String usecsToLastByteRecv; + + /** + * Elapsed microseconds until the TGen client has received the SOCKS choice + * from the Tor client, or -1 if missing (step 4). + */ + @JsonProperty("usecs-to-proxy-choice") + String usecsToProxyChoice; + + /** + * Elapsed microseconds until the TGen client has sent the SOCKS request to + * the Tor client, or -1 if missing (step 5). + */ + @JsonProperty("usecs-to-proxy-request") + String usecsToProxyRequest; + + /** + * Elapsed microseconds until the TGen client has received the SOCKS + * response from the Tor client, or -1 if missing (step 6). + */ + @JsonProperty("usecs-to-proxy-response") + String usecsToProxyResponse; + + /** + * Elapsed microseconds until the TGen client has received the command from + * the TGen server, or -1 if missing (step 8). + */ + @JsonProperty("usecs-to-response") + String usecsToResponse; + + /** + * Elapsed microseconds until the TGen client has connected to the Tor + * client's SOCKS port, or -1 if missing (step 2). + */ + @JsonProperty("usecs-to-socket-connect") + String usecsToSocketConnect; + + /** + * Elapsed microseconds until the TGen client has opened a TCP connection + * to the Tor client's SOCKS port, or -1 if missing (step 1). + */ + @JsonProperty("usecs-to-socket-create") + String usecsToSocketCreate; + } + + /** + * Information about the TGen transport. + */ + static class TransportInfo { + + /** + * Local host name, IP address, and TCP port. + */ + String local; + + /** + * Proxy host name, IP address, and TCP port. + */ + String proxy; + + /** + * Remote host name, IP address, and TCP port. + */ + String remote; + } + /** * Measurement data obtained from client-side {@code tor} controller event * logs. diff --git a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java index a99257a..766f7ad 100644 --- a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java +++ b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java @@ -15,6 +15,19 @@ import java.util.TreeMap; */ public class TorperfResultsBuilder {

+ /** + * Error code parts, to be formatted as + * {@code ERRORCODE=part_1/part_2/.../part_n}. + */ + private List<String> errorCodeParts = null; + + void addErrorCodePart(String errorCodePart) { + if (null == errorCodeParts) { + this.errorCodeParts = new ArrayList<>(); + } + this.errorCodeParts.add(errorCodePart); + } + /** * Key-value pairs to be formatted as Torperf results line. */ @@ -77,6 +90,27 @@ public class TorperfResultsBuilder { } }

+ /** + * Add a timestamp value as the sum of a double value, formatted as seconds + * since the epoch with two decimal places, and a string value, formatted as + * microseconds since the first value, unless either of the two summands is + * {@code null} or negative. + * + * @param key Key. + * @param unixTsStart First summand representing seconds since the epoch. + * @param elapsedMicroseconds Second summand representing microseconds + * elapsed since the first summand. + */ + void addTimestamp(String key, Double unixTsStart, + String elapsedMicroseconds) { + if (null != unixTsStart && unixTsStart >= 0.0 && null != elapsedMicroseconds + && !elapsedMicroseconds.startsWith("-")) { + double elapsedSeconds = Double.parseDouble(elapsedMicroseconds) + / 1000000.0; + this.addTimestamp(key, unixTsStart, elapsedSeconds); + } + } + /** * Build the Torperf results line by putting together all key-value pairs as * {@code "key=value"}, separated by spaces, prefixed by an annotation line @@ -89,6 +123,10 @@ public class TorperfResultsBuilder { StringBuilder result = new StringBuilder(); result.append("@type torperf 1.1\r\n"); List<String> torperfResultsParts = new ArrayList<>(); + if (null != this.errorCodeParts) { + String errorCode = String.join("/", errorCodeParts); + this.addString("ERRORCODE", errorCode); + } for (Map.Entry<String, String> keyValuePairsEntry : this.keyValuePairs.entrySet()) { torperfResultsParts.add(String.format("%s=%s", diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java index 51e0896..f8d9f90 100644 --- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java +++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java @@ -3,6 +3,7 @@

package org.torproject.descriptor.onionperf;

+import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue;

@@ -16,6 +17,8 @@ import org.junit.Test; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.ArrayList; +import java.util.List;

public class OnionPerfAnalysisConverterTest {

@@ -89,6 +92,10 @@ public class OnionPerfAnalysisConverterTest { @Test public void testAsTorperfResults() throws IOException, DescriptorParseException { + List<String> expectedTorperfResults = new ArrayList<>(); + expectedTorperfResults.add(torperfResultTransfer1m1); + expectedTorperfResults.add(torperfResultTransfer1m3); + expectedTorperfResults.add(torperfResultTransfer50k2); URL resouce = getClass().getClassLoader().getResource( "onionperf/onionperf.analysis.json.xz"); assertNotNull(resouce); @@ -103,12 +110,75 @@ public class OnionPerfAnalysisConverterTest { String formattedTorperfResult = new String(descriptor.getRawDescriptorBytes()).trim(); assertNotNull(formattedTorperfResult); - assertTrue(String.format("Unrecognized formatted Torperf result: %s", - formattedTorperfResult), - formattedTorperfResult.equals(torperfResultTransfer1m1) - || formattedTorperfResult.equals(torperfResultTransfer1m3) - || formattedTorperfResult.equals(torperfResultTransfer50k2)); + String expectedTorperfResult = expectedTorperfResults.remove(0); + assertEquals(expectedTorperfResult, formattedTorperfResult); } + assertTrue(expectedTorperfResults.isEmpty()); + } + + private final String torperfResultStream155 + = "BUILDTIMES=0.62,0.90,1.26 CIRC_ID=1008 CONNECT=1594633118.41 " + + "DATACOMPLETE=1594633145.46 DATAPERC0=1594633123.64 " + + "DATAPERC10=1594633125.57 DATAPERC100=1594633145.46 " + + "DATAPERC20=1594633127.56 DATAPERC30=1594633129.56 " + + "DATAPERC40=1594633132.20 DATAPERC50=1594633134.29 " + + "DATAPERC60=1594633136.41 DATAPERC70=1594633138.84 " + + "DATAPERC80=1594633140.76 DATAPERC90=1594633142.88 " + + "DATAREQUEST=1594633123.01 DATARESPONSE=1594633123.64 DIDTIMEOUT=0 " + + "ENDPOINTLOCAL=localhost:127.0.0.1:46222 " + + "ENDPOINTPROXY=localhost:127.0.0.1:29849 " + + "ENDPOINTREMOTE=jjsvldkkjd3lxy6gljy6xmhrn5mnirzgj2mrftrx2bf3n4bbx53fxza" + + "d.onion:0.0.0.0:8080 FILESIZE=5242880 HOSTNAMELOCAL=op-nl-test1 " + + "HOSTNAMEREMOTE=op-nl-test1 LAUNCH=1594633119.06 " + + "NEGOTIATE=1594633118.41 PARTIAL10240=1594633123.80 " + + "PARTIAL102400=1594633124.11 PARTIAL1048576=1594633127.56 " + + "PARTIAL20480=1594633123.85 PARTIAL204800=1594633124.26 " + + "PARTIAL2097152=1594633132.20 PARTIAL51200=1594633123.96 " + + "PARTIAL512000=1594633125.50 PARTIAL5242880=1594633145.46 " + + "PATH=$65888719E2F619E6198F1045A93AF0176C05354D," + + "$3043C1A6DF23AFEDC8FD3C0671FADCEDFF6D3429," + + "$8E5F4EE45E0631A60E59CAA42E1464FD7120459D QUANTILE=0.8 " + + "READBYTES=5242990 REQUEST=1594633118.42 RESPONSE=1594633123.01 " + + "SOCKET=1594633118.41 SOURCE=op-nl-test1 SOURCEADDRESS=unknown " + + "START=1594633118.41 TIMEOUT=1500 USED_AT=1594633145.46 USED_BY=1498 " + + "WRITEBYTES=2174"; + + private final String torperfResultStream156 + = "CONNECT=1594633539.83 DATACOMPLETE=0.0 DATAREQUEST=0.0 " + + "DATARESPONSE=0.0 DIDTIMEOUT=1 ENDPOINTLOCAL=localhost:127.0.0.1:46246 " + + "ENDPOINTPROXY=localhost:127.0.0.1:29849 " + + "ENDPOINTREMOTE=jjsvldkkjd3lxy6gljy6xmhrn5mnirzgj2mrftrx2bf3n4bbx53fxza" + + "d.onion:0.0.0.0:8080 ERRORCODE=TOR/TIMEOUT FILESIZE=5242880 " + + "HOSTNAMELOCAL=op-nl-test1 HOSTNAMEREMOTE=(null) " + + "NEGOTIATE=1594633539.83 READBYTES=0 REQUEST=1594633539.83 " + + "RESPONSE=0.0 SOCKET=1594633539.83 SOURCE=op-nl-test1 " + + "SOURCEADDRESS=unknown START=1594633539.83 USED_AT=1594633539.83 " + + "USED_BY=1510 WRITEBYTES=0"; + + @Test + public void testAsTorperfResultsVersion3() throws IOException, + DescriptorParseException { + List<String> expectedTorperfResults = new ArrayList<>(); + expectedTorperfResults.add(torperfResultStream155); + expectedTorperfResults.add(torperfResultStream156); + URL resouce = getClass().getClassLoader().getResource( + "onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz"); + assertNotNull(resouce); + InputStream compressedInputStream = resouce.openStream(); + assertNotNull(compressedInputStream); + byte[] rawDescriptorBytes = IOUtils.toByteArray(compressedInputStream); + OnionPerfAnalysisConverter onionPerfAnalysisConverter + = new OnionPerfAnalysisConverter(rawDescriptorBytes, null); + for (Descriptor descriptor + : onionPerfAnalysisConverter.asTorperfResults()) { + assertTrue(descriptor instanceof TorperfResult); + String formattedTorperfResult + = new String(descriptor.getRawDescriptorBytes()).trim(); + assertNotNull(formattedTorperfResult); + String expectedTorperfResult = expectedTorperfResults.remove(0); + assertEquals(expectedTorperfResult, formattedTorperfResult); + } + assertTrue(expectedTorperfResults.isEmpty()); } }

diff --git a/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz b/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz new file mode 100644 index 0000000..78be8c5 Binary files /dev/null and b/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz differ