commit b44c6bccb46ca1eb9c861a84dfcd7d5d2047dee4
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Thu Jul 16 21:37:19 2020 +0200
Parse OnionPerf analysis results format v3.0.
Implements tpo/metrics/library#40001.
---
CHANGELOG.md | 1 +
.../onionperf/OnionPerfAnalysisConverter.java | 235 ++++++++++++++++-----
.../onionperf/ParsedOnionPerfAnalysis.java | 206 ++++++++++++++++++
.../onionperf/TorperfResultsBuilder.java | 38 ++++
.../onionperf/OnionPerfAnalysisConverterTest.java | 80 ++++++-
...20-07-13.op-nl-test1.onionperf.analysis.json.xz | Bin 0 -> 1736 bytes
6 files changed, 499 insertions(+), 61 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7bb22c4..4679688 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
* Medium changes
- Extend Torperf results to provide error codes.
+ - Parse OnionPerf analysis results format version 3.0.
# Changes in version 2.13.0 - 2020-05-16
diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
index 61fd173..f1b9d11 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
@@ -69,25 +69,32 @@ public class OnionPerfAnalysisConverter {
* Torperf results.
*/
public List<Descriptor> asTorperfResults() throws DescriptorParseException {
- ParsedOnionPerfAnalysis parsedOnionPerfAnalysis;
+ ParsedOnionPerfAnalysis parsedOnionPerfAnalysis
+ = this.parseOnionPerfAnalysis();
+ this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis);
+ StringBuilder formattedTorperfResults
+ = this.formatTorperfResults(parsedOnionPerfAnalysis);
+ this.parseFormattedTorperfResults(formattedTorperfResults);
+ return this.convertedTorperfResults;
+ }
+
+ /**
+ * Parse the OnionPerf analysis JSON document.
+ */
+ private ParsedOnionPerfAnalysis parseOnionPerfAnalysis()
+ throws DescriptorParseException {
try {
InputStream compressedInputStream = new ByteArrayInputStream(
this.rawDescriptorBytes);
InputStream decompressedInputStream = new XZCompressorInputStream(
compressedInputStream);
byte[] decompressedBytes = IOUtils.toByteArray(decompressedInputStream);
- parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes(
- decompressedBytes);
+ return ParsedOnionPerfAnalysis.fromBytes(decompressedBytes);
} catch (IOException ioException) {
throw new DescriptorParseException("Ran into an I/O error while "
+ "attempting to parse an OnionPerf analysis document.",
ioException);
}
- this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis);
- StringBuilder formattedTorperfResults
- = this.formatTorperfResults(parsedOnionPerfAnalysis);
- this.parseFormattedTorperfResults(formattedTorperfResults);
- return this.convertedTorperfResults;
}
/**
@@ -109,9 +116,9 @@ public class OnionPerfAnalysisConverter {
throw new DescriptorParseException("Parsed OnionPerf analysis file does "
+ "not contain version information.");
} else if ((parsedOnionPerfAnalysis.version instanceof Double
- && (double) parsedOnionPerfAnalysis.version > 2.999)
+ && (double) parsedOnionPerfAnalysis.version > 3.999)
|| (parsedOnionPerfAnalysis.version instanceof String
- && ((String) parsedOnionPerfAnalysis.version).compareTo("3.") >= 0)) {
+ && ((String) parsedOnionPerfAnalysis.version).compareTo("4.") >= 0)) {
throw new DescriptorParseException("Parsed OnionPerf analysis file "
+ "contains unsupported version " + parsedOnionPerfAnalysis.version
+ ".");
@@ -131,8 +138,7 @@ public class OnionPerfAnalysisConverter {
: parsedOnionPerfAnalysis.data.entrySet()) {
String nickname = data.getKey();
ParsedOnionPerfAnalysis.MeasurementData measurements = data.getValue();
- if (null == measurements.measurementIp || null == measurements.tgen
- || null == measurements.tgen.transfers) {
+ if (null == measurements.tgen) {
continue;
}
String measurementIp = measurements.measurementIp;
@@ -153,57 +159,69 @@ public class OnionPerfAnalysisConverter {
}
}
}
- for (ParsedOnionPerfAnalysis.Transfer transfer
- : measurements.tgen.transfers.values()) {
- if (null == transfer.endpointLocal) {
- continue;
- }
- String[] endpointLocalParts = transfer.endpointLocal.split(":");
- if (endpointLocalParts.length < 3) {
- continue;
- }
- TorperfResultsBuilder torperfResultsBuilder
- = new TorperfResultsBuilder();
-
- torperfResultsBuilder.addString("SOURCE", nickname);
- torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
- this.formatTransferParts(torperfResultsBuilder, transfer);
- List<String> errorCodeParts = null;
- if (transfer.isError) {
- errorCodeParts = new ArrayList<>();
- if ("PROXY".equals(transfer.errorCode)) {
- errorCodeParts.add("TOR");
- } else {
- errorCodeParts.add("TGEN");
- errorCodeParts.add(transfer.errorCode);
+ if (null != measurements.tgen.transfers) {
+ for (ParsedOnionPerfAnalysis.Transfer transfer
+ : measurements.tgen.transfers.values()) {
+ TorperfResultsBuilder torperfResultsBuilder
+ = new TorperfResultsBuilder();
+ torperfResultsBuilder.addString("SOURCE", nickname);
+ torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
+ this.formatTransferParts(torperfResultsBuilder, transfer);
+ if (null != transfer.endpointLocal) {
+ String[] endpointLocalParts = transfer.endpointLocal.split(":");
+ if (endpointLocalParts.length >= 3) {
+ String sourcePort = endpointLocalParts[2];
+ if (streamsBySourcePort.containsKey(sourcePort)) {
+ for (ParsedOnionPerfAnalysis.Stream stream
+ : streamsBySourcePort.get(sourcePort)) {
+ if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) {
+ this.formatStreamParts(torperfResultsBuilder, stream);
+ if (null != stream.circuitId
+ && circuitsByCircuitId.containsKey(stream.circuitId)) {
+ ParsedOnionPerfAnalysis.Circuit circuit
+ = circuitsByCircuitId.get(stream.circuitId);
+ this.formatCircuitParts(torperfResultsBuilder, circuit);
+ }
+ }
+ }
+ }
+ }
}
+ formattedTorperfResults.append(torperfResultsBuilder.build());
}
- String sourcePort = endpointLocalParts[2];
- if (streamsBySourcePort.containsKey(sourcePort)) {
- for (ParsedOnionPerfAnalysis.Stream stream
- : streamsBySourcePort.get(sourcePort)) {
- if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) {
- if (null != errorCodeParts && null != stream.failureReasonLocal) {
- errorCodeParts.add(stream.failureReasonLocal);
- if (null != stream.failureReasonRemote) {
- errorCodeParts.add(stream.failureReasonRemote);
+ }
+ if (null != measurements.tgen.streams) {
+ for (ParsedOnionPerfAnalysis.TgenStream stream
+ : measurements.tgen.streams.values()) {
+ TorperfResultsBuilder torperfResultsBuilder
+ = new TorperfResultsBuilder();
+ torperfResultsBuilder.addString("SOURCE", nickname);
+ torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
+ this.formatTgenStreamParts(torperfResultsBuilder, stream);
+ if (null != stream.transportInfo
+ && null != stream.transportInfo.local) {
+ String[] endpointLocalParts = stream.transportInfo.local.split(":");
+ if (endpointLocalParts.length >= 3) {
+ String sourcePort = endpointLocalParts[2];
+ if (streamsBySourcePort.containsKey(sourcePort)) {
+ for (ParsedOnionPerfAnalysis.Stream torStream
+ : streamsBySourcePort.get(sourcePort)) {
+ if (Math.abs(stream.unixTsEnd
+ - torStream.unixTsEnd) < 150.0) {
+ this.formatStreamParts(torperfResultsBuilder, torStream);
+ if (null != torStream.circuitId && circuitsByCircuitId
+ .containsKey(torStream.circuitId)) {
+ ParsedOnionPerfAnalysis.Circuit circuit
+ = circuitsByCircuitId.get(torStream.circuitId);
+ this.formatCircuitParts(torperfResultsBuilder, circuit);
+ }
+ }
}
}
- if (null != stream.circuitId
- && circuitsByCircuitId.containsKey(stream.circuitId)) {
- ParsedOnionPerfAnalysis.Circuit circuit
- = circuitsByCircuitId.get(stream.circuitId);
- this.formatStreamParts(torperfResultsBuilder, stream);
- this.formatCircuitParts(torperfResultsBuilder, circuit);
- }
}
}
+ formattedTorperfResults.append(torperfResultsBuilder.build());
}
- if (null != errorCodeParts) {
- String errorCode = String.join("/", errorCodeParts);
- torperfResultsBuilder.addString("ERRORCODE", errorCode);
- }
- formattedTorperfResults.append(torperfResultsBuilder.build());
}
}
return formattedTorperfResults;
@@ -288,6 +306,105 @@ public class OnionPerfAnalysisConverter {
transfer.elapsedSeconds.lastByte);
if (transfer.isError) {
torperfResultsBuilder.addInteger("DIDTIMEOUT", 1);
+ if ("PROXY".equals(transfer.errorCode)) {
+ torperfResultsBuilder.addErrorCodePart("TOR");
+ } else {
+ torperfResultsBuilder.addErrorCodePart("TGEN");
+ torperfResultsBuilder.addErrorCodePart(transfer.errorCode);
+ }
+ }
+ }
+ }
+
+ /**
+ * Format relevant tgen stream data as Torperf result key-value pairs.
+ *
+ * @param torperfResultsBuilder Torperf results builder to add key-value pairs
+ * to.
+ * @param stream Stream data obtained from the parsed OnionPerf analysis file.
+ */
+ private void formatTgenStreamParts(
+ TorperfResultsBuilder torperfResultsBuilder,
+ ParsedOnionPerfAnalysis.TgenStream stream) {
+ torperfResultsBuilder.addString("READBYTES", "0");
+ torperfResultsBuilder.addString("WRITEBYTES", "0");
+ if (null != stream.byteInfo) {
+ torperfResultsBuilder.addString("READBYTES",
+ stream.byteInfo.totalBytesRecv);
+ torperfResultsBuilder.addString("WRITEBYTES",
+ stream.byteInfo.totalBytesSend);
+ }
+ if (null != stream.streamInfo) {
+ torperfResultsBuilder.addString("FILESIZE", stream.streamInfo.recvsize);
+ torperfResultsBuilder.addString("HOSTNAMELOCAL", stream.streamInfo.name);
+ torperfResultsBuilder.addString("HOSTNAMEREMOTE",
+ stream.streamInfo.peername);
+ }
+ if (null != stream.transportInfo) {
+ torperfResultsBuilder.addString("ENDPOINTLOCAL",
+ stream.transportInfo.local);
+ torperfResultsBuilder.addString("ENDPOINTPROXY",
+ stream.transportInfo.proxy);
+ torperfResultsBuilder.addString("ENDPOINTREMOTE",
+ stream.transportInfo.remote);
+ }
+ torperfResultsBuilder.addInteger("DIDTIMEOUT", 0);
+
+ for (String key : new String[] { "START", "SOCKET", "CONNECT", "NEGOTIATE",
+ "REQUEST", "RESPONSE", "DATAREQUEST", "DATARESPONSE",
+ "DATACOMPLETE" }) {
+ torperfResultsBuilder.addString(key, "0.0");
+ }
+ torperfResultsBuilder.addTimestamp("START", stream.unixTsStart, 0.0);
+ if (null != stream.unixTsStart) {
+ if (null != stream.timeInfo) {
+ torperfResultsBuilder.addTimestamp("SOCKET", stream.unixTsStart,
+ stream.timeInfo.usecsToSocketCreate);
+ torperfResultsBuilder.addTimestamp("CONNECT", stream.unixTsStart,
+ stream.timeInfo.usecsToSocketConnect);
+ torperfResultsBuilder.addTimestamp("NEGOTIATE", stream.unixTsStart,
+ stream.timeInfo.usecsToProxyChoice);
+ torperfResultsBuilder.addTimestamp("REQUEST", stream.unixTsStart,
+ stream.timeInfo.usecsToProxyRequest);
+ torperfResultsBuilder.addTimestamp("RESPONSE", stream.unixTsStart,
+ stream.timeInfo.usecsToProxyResponse);
+ torperfResultsBuilder.addTimestamp("DATAREQUEST", stream.unixTsStart,
+ stream.timeInfo.usecsToCommand);
+ torperfResultsBuilder.addTimestamp("DATARESPONSE", stream.unixTsStart,
+ stream.timeInfo.usecsToResponse);
+ torperfResultsBuilder.addTimestamp("DATACOMPLETE", stream.unixTsStart,
+ stream.timeInfo.usecsToLastByteRecv);
+ }
+ if (null != stream.elapsedSeconds) {
+ if (null != stream.elapsedSeconds.payloadBytesRecv) {
+ for (Map.Entry<String, Double> payloadBytesRecvEntry
+ : stream.elapsedSeconds.payloadBytesRecv.entrySet()) {
+ String key = String.format("PARTIAL%s",
+ payloadBytesRecvEntry.getKey());
+ Double elapsedSeconds = payloadBytesRecvEntry.getValue();
+ torperfResultsBuilder.addTimestamp(key, stream.unixTsStart,
+ elapsedSeconds);
+ }
+ }
+ if (null != stream.elapsedSeconds.payloadProgressRecv) {
+ for (Map.Entry<String, Double> payloadProgressRecvEntry
+ : stream.elapsedSeconds.payloadProgressRecv.entrySet()) {
+ String key = String.format("DATAPERC%.0f",
+ Double.parseDouble(payloadProgressRecvEntry.getKey()) * 100.0);
+ Double elapsedSeconds = payloadProgressRecvEntry.getValue();
+ torperfResultsBuilder.addTimestamp(key, stream.unixTsStart,
+ elapsedSeconds);
+ }
+ }
+ }
+ if (null != stream.isError && stream.isError) {
+ torperfResultsBuilder.addInteger("DIDTIMEOUT", 1);
+ if ("PROXY".equals(stream.streamInfo.error)) {
+ torperfResultsBuilder.addErrorCodePart("TOR");
+ } else {
+ torperfResultsBuilder.addErrorCodePart("TGEN");
+ torperfResultsBuilder.addErrorCodePart(stream.streamInfo.error);
+ }
}
}
}
@@ -301,6 +418,12 @@ public class OnionPerfAnalysisConverter {
*/
private void formatStreamParts(TorperfResultsBuilder torperfResultsBuilder,
ParsedOnionPerfAnalysis.Stream stream) {
+ if (null != stream.failureReasonLocal) {
+ torperfResultsBuilder.addErrorCodePart(stream.failureReasonLocal);
+ if (null != stream.failureReasonRemote) {
+ torperfResultsBuilder.addErrorCodePart(stream.failureReasonRemote);
+ }
+ }
torperfResultsBuilder.addTimestamp("USED_AT", stream.unixTsEnd, 0.0);
torperfResultsBuilder.addInteger("USED_BY", stream.streamId);
}
diff --git a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
index 4eca0ff..e94fdb4 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
@@ -4,6 +4,7 @@
package org.torproject.descriptor.onionperf;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -95,6 +96,11 @@ public class ParsedOnionPerfAnalysis {
* Measurement data by transfer identifier.
*/
Map<String, Transfer> transfers;
+
+ /**
+ * Measurement data by stream identifier.
+ */
+ Map<String, TgenStream> streams;
}
/**
@@ -240,6 +246,206 @@ public class ParsedOnionPerfAnalysis {
Double socketCreate;
}
+ /**
+ * Measurement data related to a single stream obtained from client-side
+ * {@code tgen} logs.
+ */
+ static class TgenStream {
+
+ /**
+ * Information on sent and received bytes.
+ */
+ ByteInfo byteInfo;
+
+ /**
+ * Elapsed seconds until a given number or fraction of payload bytes have
+ * been received or sent, obtained from {@code [stream-status]},
+ * {@code [stream-success]}, and {@code [stream-error]} log messages, only
+ * included if the measurement was a success.
+ */
+ ElapsedSecondsPayload elapsedSeconds;
+
+ /**
+ * Whether an error occurred.
+ */
+ Boolean isError;
+
+ /**
+ * Information about the TGen stream.
+ */
+ StreamInfo streamInfo;
+
+ /**
+ * Elapsed time until reaching given substeps in a measurement.
+ */
+ TimeInfo timeInfo;
+
+ /**
+ * Information about the TGen transport.
+ */
+ TransportInfo transportInfo;
+
+ /**
+ * Initial start time of the measurement, obtained by subtracting the
+ * largest number of elapsed microseconds in {@code time_info} from
+ * {@code unix_ts_end}, given in seconds since the epoch.
+ */
+ Double unixTsStart;
+
+ /**
+ * Final end time of the measurement, obtained from the log time of the
+ * {@code [stream-success]} or {@code [stream-error]} log message, given in
+ * seconds since the epoch.
+ */
+ Double unixTsEnd;
+ }
+
+ /**
+ * Information on sent and received bytes.
+ */
+ static class ByteInfo {
+
+ /**
+ * Total number of bytes received.
+ */
+ @JsonProperty("total-bytes-recv")
+ String totalBytesRecv;
+
+ /**
+ * Total number of bytes sent.
+ */
+ @JsonProperty("total-bytes-send")
+ String totalBytesSend;
+ }
+
+ /**
+ * Elapsed seconds until a given number or fraction of payload bytes have been
+ * received or sent, obtained from {@code [stream-status]},
+ * {@code [stream-success]}, and {@code [stream-error]} log messages, only
+ * included if the measurement was a success.
+ */
+ static class ElapsedSecondsPayload {
+
+ /**
+ * Number of received payload bytes.
+ */
+ Map<String, Double> payloadBytesRecv;
+
+ /**
+ * Fraction of received payload bytes.
+ */
+ Map<String, Double> payloadProgressRecv;
+ }
+
+ /**
+ * Information about the TGen stream.
+ */
+ static class StreamInfo {
+
+ /**
+ * Error code, or {@code NONE} if no error occurred.
+ */
+ String error;
+
+ /**
+ * Hostname of the TGen client.
+ */
+ String name;
+
+ /**
+ * Hostname of the TGen server.
+ */
+ String peername;
+
+ /**
+ * Number of expected payload bytes in the response.
+ */
+ String recvsize;
+ }
+
+ /**
+ * Elapsed time until reaching given substeps in a measurement.
+ */
+ static class TimeInfo {
+
+ /**
+ * Elapsed microseconds until the TGen client has sent the command to the
+ * TGen server, or -1 if missing (step 7).
+ */
+ @JsonProperty("usecs-to-command")
+ String usecsToCommand;
+
+ /**
+ * Elapsed microseconds until the TGen client has received the last payload
+ * byte, or -1 if missing (step 10).
+ */
+ @JsonProperty("usecs-to-last-byte-recv")
+ String usecsToLastByteRecv;
+
+ /**
+ * Elapsed microseconds until the TGen client has received the SOCKS choice
+ * from the Tor client, or -1 if missing (step 4).
+ */
+ @JsonProperty("usecs-to-proxy-choice")
+ String usecsToProxyChoice;
+
+ /**
+ * Elapsed microseconds until the TGen client has sent the SOCKS request to
+ * the Tor client, or -1 if missing (step 5).
+ */
+ @JsonProperty("usecs-to-proxy-request")
+ String usecsToProxyRequest;
+
+ /**
+ * Elapsed microseconds until the TGen client has received the SOCKS
+ * response from the Tor client, or -1 if missing (step 6).
+ */
+ @JsonProperty("usecs-to-proxy-response")
+ String usecsToProxyResponse;
+
+ /**
+ * Elapsed microseconds until the TGen client has received the command from
+ * the TGen server, or -1 if missing (step 8).
+ */
+ @JsonProperty("usecs-to-response")
+ String usecsToResponse;
+
+ /**
+ * Elapsed microseconds until the TGen client has connected to the Tor
+ * client's SOCKS port, or -1 if missing (step 2).
+ */
+ @JsonProperty("usecs-to-socket-connect")
+ String usecsToSocketConnect;
+
+ /**
+ * Elapsed microseconds until the TGen client has opened a TCP connection
+ * to the Tor client's SOCKS port, or -1 if missing (step 1).
+ */
+ @JsonProperty("usecs-to-socket-create")
+ String usecsToSocketCreate;
+ }
+
+ /**
+ * Information about the TGen transport.
+ */
+ static class TransportInfo {
+
+ /**
+ * Local host name, IP address, and TCP port.
+ */
+ String local;
+
+ /**
+ * Proxy host name, IP address, and TCP port.
+ */
+ String proxy;
+
+ /**
+ * Remote host name, IP address, and TCP port.
+ */
+ String remote;
+ }
+
/**
* Measurement data obtained from client-side {@code tor} controller event
* logs.
diff --git a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
index a99257a..766f7ad 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
@@ -15,6 +15,19 @@ import java.util.TreeMap;
*/
public class TorperfResultsBuilder {
+ /**
+ * Error code parts, to be formatted as
+ * {@code ERRORCODE=part_1/part_2/.../part_n}.
+ */
+ private List<String> errorCodeParts = null;
+
+ void addErrorCodePart(String errorCodePart) {
+ if (null == errorCodeParts) {
+ this.errorCodeParts = new ArrayList<>();
+ }
+ this.errorCodeParts.add(errorCodePart);
+ }
+
/**
* Key-value pairs to be formatted as Torperf results line.
*/
@@ -77,6 +90,27 @@ public class TorperfResultsBuilder {
}
}
+ /**
+ * Add a timestamp value as the sum of a double value, formatted as seconds
+ * since the epoch with two decimal places, and a string value, formatted as
+ * microseconds since the first value, unless either of the two summands is
+ * {@code null} or negative.
+ *
+ * @param key Key.
+ * @param unixTsStart First summand representing seconds since the epoch.
+ * @param elapsedMicroseconds Second summand representing microseconds
+ * elapsed since the first summand.
+ */
+ void addTimestamp(String key, Double unixTsStart,
+ String elapsedMicroseconds) {
+ if (null != unixTsStart && unixTsStart >= 0.0 && null != elapsedMicroseconds
+ && !elapsedMicroseconds.startsWith("-")) {
+ double elapsedSeconds = Double.parseDouble(elapsedMicroseconds)
+ / 1000000.0;
+ this.addTimestamp(key, unixTsStart, elapsedSeconds);
+ }
+ }
+
/**
* Build the Torperf results line by putting together all key-value pairs as
* {@code "key=value"}, separated by spaces, prefixed by an annotation line
@@ -89,6 +123,10 @@ public class TorperfResultsBuilder {
StringBuilder result = new StringBuilder();
result.append("@type torperf 1.1\r\n");
List<String> torperfResultsParts = new ArrayList<>();
+ if (null != this.errorCodeParts) {
+ String errorCode = String.join("/", errorCodeParts);
+ this.addString("ERRORCODE", errorCode);
+ }
for (Map.Entry<String, String> keyValuePairsEntry
: this.keyValuePairs.entrySet()) {
torperfResultsParts.add(String.format("%s=%s",
diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
index 51e0896..f8d9f90 100644
--- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
+++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
@@ -3,6 +3,7 @@
package org.torproject.descriptor.onionperf;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -16,6 +17,8 @@ import org.junit.Test;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
public class OnionPerfAnalysisConverterTest {
@@ -89,6 +92,10 @@ public class OnionPerfAnalysisConverterTest {
@Test
public void testAsTorperfResults() throws IOException,
DescriptorParseException {
+ List<String> expectedTorperfResults = new ArrayList<>();
+ expectedTorperfResults.add(torperfResultTransfer1m1);
+ expectedTorperfResults.add(torperfResultTransfer1m3);
+ expectedTorperfResults.add(torperfResultTransfer50k2);
URL resouce = getClass().getClassLoader().getResource(
"onionperf/onionperf.analysis.json.xz");
assertNotNull(resouce);
@@ -103,12 +110,75 @@ public class OnionPerfAnalysisConverterTest {
String formattedTorperfResult
= new String(descriptor.getRawDescriptorBytes()).trim();
assertNotNull(formattedTorperfResult);
- assertTrue(String.format("Unrecognized formatted Torperf result: %s",
- formattedTorperfResult),
- formattedTorperfResult.equals(torperfResultTransfer1m1)
- || formattedTorperfResult.equals(torperfResultTransfer1m3)
- || formattedTorperfResult.equals(torperfResultTransfer50k2));
+ String expectedTorperfResult = expectedTorperfResults.remove(0);
+ assertEquals(expectedTorperfResult, formattedTorperfResult);
}
+ assertTrue(expectedTorperfResults.isEmpty());
+ }
+
+ private final String torperfResultStream155
+ = "BUILDTIMES=0.62,0.90,1.26 CIRC_ID=1008 CONNECT=1594633118.41 "
+ + "DATACOMPLETE=1594633145.46 DATAPERC0=1594633123.64 "
+ + "DATAPERC10=1594633125.57 DATAPERC100=1594633145.46 "
+ + "DATAPERC20=1594633127.56 DATAPERC30=1594633129.56 "
+ + "DATAPERC40=1594633132.20 DATAPERC50=1594633134.29 "
+ + "DATAPERC60=1594633136.41 DATAPERC70=1594633138.84 "
+ + "DATAPERC80=1594633140.76 DATAPERC90=1594633142.88 "
+ + "DATAREQUEST=1594633123.01 DATARESPONSE=1594633123.64 DIDTIMEOUT=0 "
+ + "ENDPOINTLOCAL=localhost:127.0.0.1:46222 "
+ + "ENDPOINTPROXY=localhost:127.0.0.1:29849 "
+ + "ENDPOINTREMOTE=jjsvldkkjd3lxy6gljy6xmhrn5mnirzgj2mrftrx2bf3n4bbx53fxza"
+ + "d.onion:0.0.0.0:8080 FILESIZE=5242880 HOSTNAMELOCAL=op-nl-test1 "
+ + "HOSTNAMEREMOTE=op-nl-test1 LAUNCH=1594633119.06 "
+ + "NEGOTIATE=1594633118.41 PARTIAL10240=1594633123.80 "
+ + "PARTIAL102400=1594633124.11 PARTIAL1048576=1594633127.56 "
+ + "PARTIAL20480=1594633123.85 PARTIAL204800=1594633124.26 "
+ + "PARTIAL2097152=1594633132.20 PARTIAL51200=1594633123.96 "
+ + "PARTIAL512000=1594633125.50 PARTIAL5242880=1594633145.46 "
+ + "PATH=$65888719E2F619E6198F1045A93AF0176C05354D,"
+ + "$3043C1A6DF23AFEDC8FD3C0671FADCEDFF6D3429,"
+ + "$8E5F4EE45E0631A60E59CAA42E1464FD7120459D QUANTILE=0.8 "
+ + "READBYTES=5242990 REQUEST=1594633118.42 RESPONSE=1594633123.01 "
+ + "SOCKET=1594633118.41 SOURCE=op-nl-test1 SOURCEADDRESS=unknown "
+ + "START=1594633118.41 TIMEOUT=1500 USED_AT=1594633145.46 USED_BY=1498 "
+ + "WRITEBYTES=2174";
+
+ private final String torperfResultStream156
+ = "CONNECT=1594633539.83 DATACOMPLETE=0.0 DATAREQUEST=0.0 "
+ + "DATARESPONSE=0.0 DIDTIMEOUT=1 ENDPOINTLOCAL=localhost:127.0.0.1:46246 "
+ + "ENDPOINTPROXY=localhost:127.0.0.1:29849 "
+ + "ENDPOINTREMOTE=jjsvldkkjd3lxy6gljy6xmhrn5mnirzgj2mrftrx2bf3n4bbx53fxza"
+ + "d.onion:0.0.0.0:8080 ERRORCODE=TOR/TIMEOUT FILESIZE=5242880 "
+ + "HOSTNAMELOCAL=op-nl-test1 HOSTNAMEREMOTE=(null) "
+ + "NEGOTIATE=1594633539.83 READBYTES=0 REQUEST=1594633539.83 "
+ + "RESPONSE=0.0 SOCKET=1594633539.83 SOURCE=op-nl-test1 "
+ + "SOURCEADDRESS=unknown START=1594633539.83 USED_AT=1594633539.83 "
+ + "USED_BY=1510 WRITEBYTES=0";
+
+ @Test
+ public void testAsTorperfResultsVersion3() throws IOException,
+ DescriptorParseException {
+ List<String> expectedTorperfResults = new ArrayList<>();
+ expectedTorperfResults.add(torperfResultStream155);
+ expectedTorperfResults.add(torperfResultStream156);
+ URL resouce = getClass().getClassLoader().getResource(
+ "onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz");
+ assertNotNull(resouce);
+ InputStream compressedInputStream = resouce.openStream();
+ assertNotNull(compressedInputStream);
+ byte[] rawDescriptorBytes = IOUtils.toByteArray(compressedInputStream);
+ OnionPerfAnalysisConverter onionPerfAnalysisConverter
+ = new OnionPerfAnalysisConverter(rawDescriptorBytes, null);
+ for (Descriptor descriptor
+ : onionPerfAnalysisConverter.asTorperfResults()) {
+ assertTrue(descriptor instanceof TorperfResult);
+ String formattedTorperfResult
+ = new String(descriptor.getRawDescriptorBytes()).trim();
+ assertNotNull(formattedTorperfResult);
+ String expectedTorperfResult = expectedTorperfResults.remove(0);
+ assertEquals(expectedTorperfResult, formattedTorperfResult);
+ }
+ assertTrue(expectedTorperfResults.isEmpty());
}
}
diff --git a/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz b/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz
new file mode 100644
index 0000000..78be8c5
Binary files /dev/null and b/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz differ
