[tor-commits] [metrics-lib/release] Add parsing support for OnionPerf analysis files.
karsten at torproject.org
karsten at torproject.org
Thu Apr 30 14:13:39 UTC 2020
commit 7eb783968bd6d9a5bde5d69409fe359df51f9a9f
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Apr 30 15:56:52 2020 +0200
Add parsing support for OnionPerf analysis files.
Implements #34070.
---
CHANGELOG.md | 4 +
.../org/torproject/descriptor/TorperfResult.java | 6 +-
.../descriptor/impl/DescriptorParserImpl.java | 4 +
.../descriptor/impl/DescriptorReaderImpl.java | 2 +
.../descriptor/impl/TorperfResultImpl.java | 11 +-
.../onionperf/OnionPerfAnalysisConverter.java | 326 ++++++++++++++++++++
.../onionperf/ParsedOnionPerfAnalysis.java | 329 +++++++++++++++++++++
.../onionperf/TorperfResultsBuilder.java | 101 +++++++
.../onionperf/OnionPerfAnalysisConverterTest.java | 108 +++++++
.../resources/onionperf/onionperf.analysis.json.xz | Bin 0 -> 17376 bytes
10 files changed, 887 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 86f78fe..c8d7508 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
# Changes in version 2.??.? - 2020-??-??
+ * Medium changes
+ - Add parsing support for OnionPerf analysis files by converting
+ and returning contained transfers as Torperf results.
+
# Changes in version 2.11.0 - 2020-04-13
diff --git a/src/main/java/org/torproject/descriptor/TorperfResult.java b/src/main/java/org/torproject/descriptor/TorperfResult.java
index da48e0b..961a206 100644
--- a/src/main/java/org/torproject/descriptor/TorperfResult.java
+++ b/src/main/java/org/torproject/descriptor/TorperfResult.java
@@ -170,9 +170,9 @@ public interface TorperfResult extends Descriptor {
List<String> getPath();
/**
- * Return a list of times in milliseconds since the epoch when circuit
- * hops were built, or null if the torperf line didn't contain that
- * information.
+ * Return a list of times in milliseconds between launching the circuit and
+ * extending to the next circuit hop, or null if the torperf line didn't
+ * contain that information.
*
* @since 1.0.0
*/
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
index 160baac..7dcd9d2 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -10,6 +10,7 @@ import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.log.LogDescriptorImpl;
+import org.torproject.descriptor.onionperf.OnionPerfAnalysisConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,6 +131,9 @@ public class DescriptorParserImpl implements DescriptorParser {
} else if (firstLines.startsWith("@type torperf 1.")) {
return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
sourceFile);
+ } else if (fileName.endsWith(".onionperf.analysis.json.xz")) {
+ return new OnionPerfAnalysisConverter(rawDescriptorBytes, sourceFile)
+ .asTorperfResults();
} else if (firstLines.startsWith("@type snowflake-stats 1.")
|| firstLines.startsWith(Key.SNOWFLAKE_STATS_END.keyword + SP)
|| firstLines.contains(NL + Key.SNOWFLAKE_STATS_END.keyword + SP)) {
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
index 8ec04a5..0da32ad 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
@@ -330,6 +330,8 @@ public class DescriptorReaderImpl implements DescriptorReader {
InputStream is = fis;
if (file.getName().endsWith(".gz")) {
is = new GzipCompressorInputStream(fis);
+ } else if (file.getName().endsWith(".xz")) {
+ is = new XZCompressorInputStream(fis);
}
byte[] rawDescriptorBytes = IOUtils.toByteArray(is);
if (rawDescriptorBytes.length > 0) {
diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
index feb551b..f8879d7 100644
--- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
@@ -23,7 +23,16 @@ public class TorperfResultImpl extends DescriptorImpl
private static final long serialVersionUID = 8961567618137500044L;
- protected static List<Descriptor> parseTorperfResults(
+ /**
+ * Parse the given descriptor to one or more {@link TorperfResult} instances.
+ *
+ * @param rawDescriptorBytes Bytes to parse
+ * @param descriptorFile Descriptor file containing the given bytes
+ * @return Parsed {@link TorperfResult} instances
+ * @throws DescriptorParseException Thrown if any of the lines cannot be
+ * parsed.
+ */
+ public static List<Descriptor> parseTorperfResults(
byte[] rawDescriptorBytes, File descriptorFile)
throws DescriptorParseException {
if (rawDescriptorBytes.length == 0) {
diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
new file mode 100644
index 0000000..1f9b1f8
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
@@ -0,0 +1,326 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.onionperf;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.impl.TorperfResultImpl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Converter that takes an OnionPerf analysis document as input and provides one
+ * or more {@link org.torproject.descriptor.TorperfResult} instances as output.
+ *
+ * <p>This conversion matches {@code tgen} transfers and {@code tor} streams by
+ * stream port and transfer/stream end timestamps. This is different from the
+ * approach taken in OnionPerf's analyze mode which only matches by stream
+ * port. The result is that converted Torperf results might contain different
+ * path or build time information as Torperf results written by OnionPerf.</p>
+ */
+public class OnionPerfAnalysisConverter {
+
+ /**
+ * Uncompressed OnionPerf analysis file bytes.
+ */
+ private final byte[] rawDescriptorBytes;
+
+ /**
+ * OnionPerf analysis file.
+ */
+ private final File descriptorFile;
+
+ /**
+ * Converted Torperf results.
+ */
+ private List<Descriptor> convertedTorperfResults;
+
+ /**
+ * Construct a new instance from the given bytes and file reference.
+ *
+ * @param rawDescriptorBytes Uncompressed document bytes.
+ * @param descriptorFile Document file reference.
+ */
+ public OnionPerfAnalysisConverter(byte[] rawDescriptorBytes,
+ File descriptorFile) {
+ this.rawDescriptorBytes = rawDescriptorBytes;
+ this.descriptorFile = descriptorFile;
+ }
+
+ /**
+ * Parse the OnionPerf analysis JSON document, do some basic verification, and
+ * convert its contents to {@link org.torproject.descriptor.TorperfResult}
+ * descriptors.
+ *
+ * @return Converted transfers.
+ * @throws DescriptorParseException Thrown if something goes wrong while
+ * parsing, verifying, or converting the OnionPerf analysis file to
+ * Torperf results.
+ */
+ public List<Descriptor> asTorperfResults() throws DescriptorParseException {
+ ParsedOnionPerfAnalysis parsedOnionPerfAnalysis;
+ try {
+ parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes(
+ this.rawDescriptorBytes);
+ } catch (IOException ioException) {
+ throw new DescriptorParseException("Ran into an I/O error while "
+ + "attempting to parse an OnionPerf analysis document.",
+ ioException);
+ }
+ this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis);
+ StringBuilder formattedTorperfResults
+ = this.formatTorperfResults(parsedOnionPerfAnalysis);
+ this.parseFormattedTorperfResults(formattedTorperfResults);
+ return this.convertedTorperfResults;
+ }
+
+ /**
+ * Verify document type and version and throw an exception when either of the
+ * two indicates that we cannot process the document.
+ *
+ * @param parsedOnionPerfAnalysis Parsed OnionPerf analysis document.
+ * @throws DescriptorParseException Thrown if either type or version indicate
+ * that we cannot process the document.
+ */
+ private void verifyDocumentTypeAndVersion(
+ ParsedOnionPerfAnalysis parsedOnionPerfAnalysis)
+ throws DescriptorParseException {
+ if (!"onionperf".equals(parsedOnionPerfAnalysis.type)) {
+ throw new DescriptorParseException("Parsed OnionPerf analysis file does "
+ + "not contain type information.");
+ }
+ if (null == parsedOnionPerfAnalysis.version) {
+ throw new DescriptorParseException("Parsed OnionPerf analysis file does "
+ + "not contain version information.");
+ } else if ((parsedOnionPerfAnalysis.version instanceof Double
+ && (double) parsedOnionPerfAnalysis.version > 1.999)
+ || (parsedOnionPerfAnalysis.version instanceof String
+ && !((String) parsedOnionPerfAnalysis.version).startsWith("1."))) {
+ throw new DescriptorParseException("Parsed OnionPerf analysis file "
+ + "contains unsupported version " + parsedOnionPerfAnalysis.version
+ + ".");
+ }
+ }
+
+ /**
+ * Format the parsed OnionPerf analysis file as one or more Torperf result
+ * strings.
+ *
+ * @param parsedOnionPerfAnalysis Parsed OnionPerf analysis document.
+ */
+ private StringBuilder formatTorperfResults(
+ ParsedOnionPerfAnalysis parsedOnionPerfAnalysis) {
+ StringBuilder formattedTorperfResults = new StringBuilder();
+ Map<String, String> errorCodes = new HashMap<>();
+ errorCodes.put("AUTH", "TGEN/AUTH");
+ errorCodes.put("READ", "TGEN/READ");
+ errorCodes.put("STALLOUT", "TGEN/STALLOUT");
+ errorCodes.put("TIMEOUT", "TGEN/TIMEOUT");
+ errorCodes.put("PROXY", "TOR");
+ errorCodes.put("PROXY_CANT_ATTACH", "TOR/CANT_ATTACH");
+ errorCodes.put("PROXY_DESTROY", "TOR/DESTROY");
+ errorCodes.put("PROXY_END_TIMEOUT", "TOR/END/TIMEOUT");
+ errorCodes.put("PROXY_END_CONNECTREFUSED", "TOR/END/CONNECTREFUSED");
+ errorCodes.put("PROXY_RESOLVEFAILED", "TOR/RESOLVEFAILED");
+ errorCodes.put("PROXY_TIMEOUT", "TOR/TIMEOUT");
+ for (Map.Entry<String, ParsedOnionPerfAnalysis.MeasurementData> data
+ : parsedOnionPerfAnalysis.data.entrySet()) {
+ String nickname = data.getKey();
+ ParsedOnionPerfAnalysis.MeasurementData measurements = data.getValue();
+ if (null == measurements.measurementIp || null == measurements.tgen
+ || null == measurements.tgen.transfers) {
+ continue;
+ }
+ String measurementIp = measurements.measurementIp;
+ Map<String, List<ParsedOnionPerfAnalysis.Stream>> streamsBySourcePort
+ = new HashMap<>();
+ Map<String, ParsedOnionPerfAnalysis.Circuit> circuitsByCircuitId
+ = new HashMap<>();
+ if (null != measurements.tor) {
+ circuitsByCircuitId = measurements.tor.circuits;
+ if (null != measurements.tor.streams) {
+ for (ParsedOnionPerfAnalysis.Stream stream
+ : measurements.tor.streams.values()) {
+ if (null != stream.source && stream.source.contains(":")) {
+ String sourcePort = stream.source.split(":")[1];
+ streamsBySourcePort.putIfAbsent(sourcePort, new ArrayList<>());
+ streamsBySourcePort.get(sourcePort).add(stream);
+ }
+ }
+ }
+ }
+ for (ParsedOnionPerfAnalysis.Transfer transfer
+ : measurements.tgen.transfers.values()) {
+ if (null == transfer.endpointLocal) {
+ continue;
+ }
+ String[] endpointLocalParts = transfer.endpointLocal.split(":");
+ if (endpointLocalParts.length < 3) {
+ continue;
+ }
+ TorperfResultsBuilder torperfResultsBuilder
+ = new TorperfResultsBuilder();
+
+ torperfResultsBuilder.addString("SOURCE", nickname);
+ torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
+ this.formatTransferParts(torperfResultsBuilder, transfer);
+ List<String> errorCodeParts = null;
+ if (transfer.isError) {
+ errorCodeParts = new ArrayList<>();
+ errorCodeParts.add(transfer.errorCode);
+ }
+ String sourcePort = endpointLocalParts[2];
+ if (streamsBySourcePort.containsKey(sourcePort)) {
+ for (ParsedOnionPerfAnalysis.Stream stream
+ : streamsBySourcePort.get(sourcePort)) {
+ if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) {
+ if (null != errorCodeParts && null != stream.failureReasonLocal) {
+ errorCodeParts.add(stream.failureReasonLocal);
+ if (null != stream.failureReasonRemote) {
+ errorCodeParts.add(stream.failureReasonRemote);
+ }
+ }
+ if (null != stream.circuitId
+ && circuitsByCircuitId.containsKey(stream.circuitId)) {
+ ParsedOnionPerfAnalysis.Circuit circuit
+ = circuitsByCircuitId.get(stream.circuitId);
+ this.formatStreamParts(torperfResultsBuilder, stream);
+ this.formatCircuitParts(torperfResultsBuilder, circuit);
+ }
+ }
+ }
+ }
+ if (null != errorCodeParts) {
+ String errorCode = String.join("_", errorCodeParts);
+ torperfResultsBuilder.addString("ERRORCODE",
+ errorCodes.getOrDefault(errorCode, errorCode));
+ }
+ formattedTorperfResults.append(torperfResultsBuilder.build());
+ }
+ }
+ return formattedTorperfResults;
+ }
+
+ /**
+ * Parse the previously formatted Torperf results.
+ *
+ * @param formattedTorperfResults Formatted Torperf result strings.
+ * @throws DescriptorParseException Thrown when an error occurs while parsing
+ * a previously formatted {@link org.torproject.descriptor.TorperfResult}
+ * string.
+ */
+ private void parseFormattedTorperfResults(
+ StringBuilder formattedTorperfResults) throws DescriptorParseException {
+ this.convertedTorperfResults = TorperfResultImpl.parseTorperfResults(
+ formattedTorperfResults.toString().getBytes(), this.descriptorFile);
+ }
+
+ /**
+ * Format relevant tgen transfer data as Torperf result key-value pairs.
+ *
+ * @param torperfResultsBuilder Torperf results builder to add key-value pairs
+ * to.
+ * @param transfer Transfer data obtained from the parsed OnionPerf analysis
+ * file.
+ */
+ private void formatTransferParts(TorperfResultsBuilder torperfResultsBuilder,
+ ParsedOnionPerfAnalysis.Transfer transfer) {
+ torperfResultsBuilder.addString("ENDPOINTLOCAL", transfer.endpointLocal);
+ torperfResultsBuilder.addString("ENDPOINTPROXY", transfer.endpointProxy);
+ torperfResultsBuilder.addString("ENDPOINTREMOTE", transfer.endpointRemote);
+ torperfResultsBuilder.addString("HOSTNAMELOCAL", transfer.hostnameLocal);
+ torperfResultsBuilder.addString("HOSTNAMEREMOTE", transfer.hostnameRemote);
+ torperfResultsBuilder.addInteger("FILESIZE", transfer.filesizeBytes);
+ torperfResultsBuilder.addInteger("READBYTES", transfer.totalBytesRead);
+ torperfResultsBuilder.addInteger("WRITEBYTES", transfer.totalBytesWrite);
+ torperfResultsBuilder.addInteger("DIDTIMEOUT", 0);
+ for (String key : new String[] { "START", "SOCKET", "CONNECT", "NEGOTIATE",
+ "REQUEST", "RESPONSE", "DATAREQUEST", "DATARESPONSE", "DATACOMPLETE",
+ "LAUNCH", "DATAPERC10", "DATAPERC20", "DATAPERC30", "DATAPERC40",
+ "DATAPERC50", "DATAPERC60", "DATAPERC70", "DATAPERC80", "DATAPERC90",
+ "DATAPERC100" }) {
+ torperfResultsBuilder.addString(key, "0.0");
+ }
+ torperfResultsBuilder.addTimestamp("START", transfer.unixTsStart, 0.0);
+ if (null != transfer.unixTsStart && null != transfer.elapsedSeconds) {
+ torperfResultsBuilder.addTimestamp("SOCKET", transfer.unixTsStart,
+ transfer.elapsedSeconds.socketCreate);
+ torperfResultsBuilder.addTimestamp("CONNECT", transfer.unixTsStart,
+ transfer.elapsedSeconds.socketConnect);
+ torperfResultsBuilder.addTimestamp("NEGOTIATE", transfer.unixTsStart,
+ transfer.elapsedSeconds.proxyChoice);
+ torperfResultsBuilder.addTimestamp("REQUEST", transfer.unixTsStart,
+ transfer.elapsedSeconds.proxyRequest);
+ torperfResultsBuilder.addTimestamp("RESPONSE", transfer.unixTsStart,
+ transfer.elapsedSeconds.proxyResponse);
+ torperfResultsBuilder.addTimestamp("DATAREQUEST", transfer.unixTsStart,
+ transfer.elapsedSeconds.command);
+ torperfResultsBuilder.addTimestamp("DATARESPONSE", transfer.unixTsStart,
+ transfer.elapsedSeconds.response);
+ if (null != transfer.elapsedSeconds.payloadProgress) {
+ for (Map.Entry<String, Double> payloadProgressEntry
+ : transfer.elapsedSeconds.payloadProgress.entrySet()) {
+ String key = String.format("DATAPERC%.0f",
+ Double.parseDouble(payloadProgressEntry.getKey()) * 100.0);
+ Double elapsedSeconds = payloadProgressEntry.getValue();
+ torperfResultsBuilder.addTimestamp(key, transfer.unixTsStart,
+ elapsedSeconds);
+ }
+ }
+ torperfResultsBuilder.addTimestamp("DATACOMPLETE", transfer.unixTsStart,
+ transfer.elapsedSeconds.lastByte);
+ if (transfer.isError) {
+ torperfResultsBuilder.addInteger("DIDTIMEOUT", 1);
+ }
+ }
+ }
+
+ /**
+ * Format relevant stream data as Torperf result key-value pairs.
+ *
+ * @param torperfResultsBuilder Torperf results builder to add key-value pairs
+ * to.
+ * @param stream Stream data obtained from the parsed OnionPerf analysis file.
+ */
+ private void formatStreamParts(TorperfResultsBuilder torperfResultsBuilder,
+ ParsedOnionPerfAnalysis.Stream stream) {
+ torperfResultsBuilder.addTimestamp("USED_AT", stream.unixTsEnd, 0.0);
+ torperfResultsBuilder.addInteger("USED_BY", stream.streamId);
+ }
+
+ /**
+ * Format relevant circuit data as Torperf result key-value pairs.
+ *
+ * @param torperfResultsBuilder Torperf results builder to add key-value pairs
+ * to.
+ * @param circuit Circuit data obtained from the parsed OnionPerf analysis
+ * file.
+ */
+ private void formatCircuitParts(TorperfResultsBuilder torperfResultsBuilder,
+ ParsedOnionPerfAnalysis.Circuit circuit) {
+ torperfResultsBuilder.addTimestamp("LAUNCH", circuit.unixTsStart, 0.0);
+ if (null != circuit.path) {
+ List<String> path = new ArrayList<>();
+ List<String> buildTimes = new ArrayList<>();
+ for (Object[] pathElement : circuit.path) {
+ String fingerprintAndNickname = (String) pathElement[0];
+ String fingerprint = fingerprintAndNickname.split("~")[0];
+ path.add(fingerprint);
+ buildTimes.add(String.format("%.2f", (Double) pathElement[1]));
+ }
+ torperfResultsBuilder.addString("PATH", String.join(",", path));
+ torperfResultsBuilder.addString("BUILDTIMES",
+ String.join(",", buildTimes));
+ torperfResultsBuilder.addInteger("TIMEOUT", circuit.buildTimeout);
+ torperfResultsBuilder.addDouble("QUANTILE", circuit.buildQuantile);
+ torperfResultsBuilder.addInteger("CIRC_ID", circuit.circuitId);
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
new file mode 100644
index 0000000..679879e
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
@@ -0,0 +1,329 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.onionperf;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Parsed OnionPerf analysis document with all relevant fields for
+ * {@link OnionPerfAnalysisConverter} to convert contained measurements to
+ * {@link org.torproject.descriptor.TorperfResult} instances.
+ */
+public class ParsedOnionPerfAnalysis {
+
+ /**
+ * Object mapper for deserializing OnionPerf analysis documents to instances
+ * of this class.
+ */
+ private static final ObjectMapper objectMapper = new ObjectMapper()
+ .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
+ .setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE)
+ .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY)
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ /**
+ * Deserialize an OnionPerf analysis document from the given uncompressed
+ * bytes.
+ *
+ * @param bytes Uncompressed contents of the OnionPerf analysis to
+ * deserialize.
+ * @return Parsed OnionPerf analysis document.
+ * @throws IOException Thrown if something goes wrong while deserializing the
+ * given JSON document, but before doing any verification or
+ * postprocessing.
+ */
+ static ParsedOnionPerfAnalysis fromBytes(byte[] bytes) throws IOException {
+ return objectMapper.readValue(bytes, ParsedOnionPerfAnalysis.class);
+ }
+
+ /**
+ * OnionPerf measurement data by source nickname.
+ */
+ Map<String, MeasurementData> data;
+
+ /**
+ * Descriptor type, which should always be {@code "onionperf"} for OnionPerf
+ * analysis documents.
+ */
+ String type;
+
+ /**
+ * Document version, which is either a {@link Double} in version 1.0 or a
+ * {@link String} in subsequent versions.
+ */
+ Object version;
+
+ /**
+ * Measurement data obtained from client-side {@code tgen} and {@code tor}
+ * controller event logs.
+ */
+ static class MeasurementData {
+
+ /**
+ * Public IP address of the OnionPerf host obtained by connecting to
+ * well-known servers and finding the IP address in the result, which may be
+ * {@code "unknown"} if OnionPerf was not able to find this information.
+ */
+ String measurementIp;
+
+ /**
+ * Measurement data obtained from client-side {@code tgen} logs.
+ */
+ TgenData tgen;
+
+ /**
+ * Measurement data obtained from client-side {@code tor} controller event
+ * logs.
+ */
+ TorData tor;
+ }
+
+ /**
+ * Measurement data obtained from client-side {@code tgen} logs.
+ */
+ static class TgenData {
+
+ /**
+ * Measurement data by transfer identifier.
+ */
+ Map<String, Transfer> transfers;
+ }
+
+ /**
+ * Measurement data related to a single transfer obtained from client-side
+ * {@code tgen} logs.
+ */
+ static class Transfer {
+
+ /**
+ * Elapsed seconds between starting a transfer at {@link #unixTsStart} and
+ * reaching a set of pre-defined states.
+ */
+ ElapsedSeconds elapsedSeconds;
+
+ /**
+ * Hostname, IP address, and port that the {@code tgen} client used to
+ * connect to the local {@code tor} SOCKS port, formatted as
+ * {@code "hostname:ip:port"}, which may be {@code "NULL:0.0.0.0:0"} if
+ * {@code tgen} was not able to find this information.
+ */
+ String endpointLocal;
+
+ /**
+ * Hostname, IP address, and port that the {@code tgen} client used to
+ * connect to the SOCKS proxy server that {@code tor} runs, formatted as
+ * {@code "hostname:ip:port"}, which may be {@code "NULL:0.0.0.0:0"} if
+ * {@code tgen} was not able to find this information.
+ */
+ String endpointProxy;
+
+ /**
+ * Hostname, IP address, and port that the {@code tgen} client used to
+ * connect to the remote server, formatted as {@code "hostname:ip:port"},
+ * which may be {@code "NULL:0.0.0.0:0"} if {@code tgen} was not able to
+ * find this information.
+ */
+ String endpointRemote;
+
+ /**
+ * Error code reported in the client {@code tgen} logs, which can be
+ * {@code "NONE"} if no error was encountered, {@code "PROXY"} in case of an
+ * error in {@code tor}, or something else for {@code tgen}-specific errors.
+ */
+ String errorCode;
+
+ /**
+ * File size in bytes of the requested file in this transfer.
+ */
+ Integer filesizeBytes;
+
+ /**
+ * Client machine hostname, which may be {@code "(NULL)"} if the
+ * {@code tgen} client was not able to find this information.
+ */
+ String hostnameLocal;
+
+ /**
+ * Server machine hostname, which may be {@code "(NULL)"} if the
+ * {@code tgen} server was not able to find this information.
+ */
+ String hostnameRemote;
+
+ /**
+ * Whether or not an error was encountered in this transfer.
+ */
+ Boolean isError;
+
+ /**
+ * Total number of bytes read in this transfer.
+ */
+ Integer totalBytesRead;
+
+ /**
+ * Total number of bytes written in this transfer.
+ */
+ Integer totalBytesWrite;
+
+ /**
+ * Unix timestamp when this transfer started.
+ */
+ Double unixTsStart;
+
+ /**
+ * Unix timestamp when this transfer ended.
+ */
+ Double unixTsEnd;
+ }
+
+ /**
+ * Elapsed seconds between starting a transfer and reaching a set of
+ * pre-defined states.
+ */
+ static class ElapsedSeconds {
+
+ /**
+ * Time until the HTTP request was written.
+ */
+ Double command;
+
+ /**
+ * Time until the payload was complete.
+ */
+ Double lastByte;
+
+ /**
+ * Time until the given fraction of expected bytes were read.
+ */
+ Map<String, Double> payloadProgress;
+
+ /**
+ * Time until SOCKS 5 authentication methods have been negotiated.
+ */
+ Double proxyChoice;
+
+ /**
+ * Time until the SOCKS request was sent.
+ */
+ Double proxyRequest;
+
+ /**
+ * Time until the SOCKS response was received.
+ */
+ Double proxyResponse;
+
+ /**
+ * Time until the first response was received.
+ */
+ Double response;
+
+ /**
+ * Time until the socket was connected.
+ */
+ Double socketConnect;
+
+ /**
+ * Time until the socket was created.
+ */
+ Double socketCreate;
+ }
+
+ /**
+ * Measurement data obtained from client-side {@code tor} controller event
+ * logs.
+ */
+ static class TorData {
+
+ /**
+ * Circuits by identifier.
+ */
+ Map<String, Circuit> circuits;
+
+ /**
+ * Streams by identifier.
+ */
+ Map<String, Stream> streams;
+ }
+
+ /**
+ * Measurement data related to a single circuit obtained from client-side
+ * {@code tor} controller event logs.
+ */
+ static class Circuit {
+
+ /**
+ * Circuit build time quantile that the {@code tor} client uses to determine
+ * its circuit-build timeout.
+ */
+ Double buildQuantile;
+
+ /**
+ * Circuit build timeout in milliseconds that the {@code tor} client used
+ * when building this circuit.
+ */
+ Integer buildTimeout;
+
+ /**
+ * Circuit identifier.
+ */
+ Integer circuitId;
+
+ /**
+ * Path information as two-dimensional array with a mixed-type
+ * {@link Object[]} for each hop with {@code "$fingerprint~nickname"} as
+ * first element and elapsed seconds between creating and extending the
+ * circuit as second element.
+ */
+ Object[][] path;
+
+ /**
+ * Unix timestamp at the start of this circuit's lifetime.
+ */
+ Double unixTsStart;
+ }
+
+ /**
+ * Measurement data related to a single stream obtained from client-side
+ * {@code tor} controller event logs.
+ */
+ static class Stream {
+
+ /**
+ * Circuit identifier of the circuit that this stream was attached to.
+ */
+ String circuitId;
+
+ /**
+ * Local reason why this stream failed.
+ */
+ String failureReasonLocal;
+
+ /**
+ * Remote reason why this stream failed.
+ */
+ String failureReasonRemote;
+
+ /**
+ * Source address and port that requested the connection.
+ */
+ String source;
+
+ /**
+ * Stream identifier.
+ */
+ Integer streamId;
+
+ /**
+ * Unix timestamp at the end of this stream's lifetime.
+ */
+ Double unixTsEnd;
+ }
+}
+
diff --git a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
new file mode 100644
index 0000000..a99257a
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
@@ -0,0 +1,101 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.onionperf;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * Builder that accepts key-value pairs and produces a single line in the
+ * Torperf results format.
+ */
+public class TorperfResultsBuilder {
+
+ /**
+ * Key-value pairs to be formatted as Torperf results line.
+ */
+ private final SortedMap<String, String> keyValuePairs = new TreeMap<>();
+
+ /**
+ * Add a string value, unless it is {@code null}.
+ *
+ * @param key Key
+ * @param stringValue String value
+ */
+ void addString(String key, String stringValue) {
+ if (null != stringValue) {
+ this.keyValuePairs.put(key, stringValue);
+ }
+ }
+
+ /**
+ * Add an int value, unless it is {@code null}.
+ *
+ * @param key Key.
+ * @param integerValue Int value.
+ */
+ void addInteger(String key, Integer integerValue) {
+ if (null != integerValue) {
+ keyValuePairs.put(key, String.valueOf(integerValue));
+ }
+ }
+
+ /**
+ * Add a double value, unless it is {@code null}.
+ *
+ * @param key Key.
+ * @param doubleValue Double value.
+ */
+ void addDouble(String key, Double doubleValue) {
+ if (null != doubleValue) {
+ keyValuePairs.put(key, String.valueOf(doubleValue));
+ }
+ }
+
+ /**
+ * Add a timestamp value as the sum of two double values, formatted as seconds
+ * since the epoch with two decimal places, unless the first summand is
+ * {@code null}.
+ *
+ * @param key Key.
+ * @param unixTsStart First summand representing seconds since the epoch.
+ * @param elapsedSeconds Second summand representing seconds elapsed since the
+ * first summand.
+ */
+ void addTimestamp(String key, Double unixTsStart, Double elapsedSeconds) {
+ if (null != unixTsStart) {
+ if (null != elapsedSeconds) {
+ keyValuePairs.put(key, String.format("%.2f",
+ unixTsStart + elapsedSeconds));
+ } else {
+ keyValuePairs.put(key, String.format("%.2f", unixTsStart));
+ }
+ }
+ }
+
+ /**
+ * Build the Torperf results line by putting together all key-value pairs as
+ * {@code "key=value"}, separated by spaces, prefixed by an annotation line
+ * {@code "@type torperf 1.1"}.
+ *
+ * @return Torperf results line using the same format as OnionPerf would
+ * write it.
+ */
+ String build() {
+ StringBuilder result = new StringBuilder();
+ result.append("@type torperf 1.1\r\n");
+ List<String> torperfResultsParts = new ArrayList<>();
+ for (Map.Entry<String, String> keyValuePairsEntry
+ : this.keyValuePairs.entrySet()) {
+ torperfResultsParts.add(String.format("%s=%s",
+ keyValuePairsEntry.getKey(), keyValuePairsEntry.getValue()));
+ }
+ result.append(String.join(" ", torperfResultsParts)).append("\r\n");
+ return result.toString();
+ }
+}
+
diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
new file mode 100644
index 0000000..d840dd2
--- /dev/null
+++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
@@ -0,0 +1,108 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.onionperf;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.TorperfResult;
+
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+public class OnionPerfAnalysisConverterTest {
+
+ private final String torperfResultTransfer1m1
+ = "BUILDTIMES=0.15,0.22,0.34 CIRC_ID=39 CONNECT=1587991280.37 "
+ + "DATACOMPLETE=1587991286.62 DATAPERC0=1587991283.81 "
+ + "DATAPERC10=1587991284.15 DATAPERC100=1587991286.62 "
+ + "DATAPERC20=1587991284.38 DATAPERC30=1587991284.66 "
+ + "DATAPERC40=1587991284.93 DATAPERC50=1587991285.14 "
+ + "DATAPERC60=1587991285.33 DATAPERC70=1587991285.67 "
+ + "DATAPERC80=1587991285.85 DATAPERC90=1587991286.14 "
+ + "DATAREQUEST=1587991283.36 DATARESPONSE=1587991283.81 DIDTIMEOUT=0 "
+ + "ENDPOINTLOCAL=localhost:127.0.0.1:40878 "
+ + "ENDPOINTPROXY=localhost:127.0.0.1:35900 "
+ + "ENDPOINTREMOTE=m3eahz7co6lzi6jn.onion:0.0.0.0:443 FILESIZE=1048576 "
+ + "HOSTNAMELOCAL=op-nl2 HOSTNAMEREMOTE=op-nl2 LAUNCH=1587991281.38 "
+ + "NEGOTIATE=1587991280.37 "
+ + "PATH=$970F0966DAA7EBDEE44E3772045527A6854E997B,"
+ + "$8101421BEFCCF4C271D5483C5AABCAAD245BBB9D,"
+ + "$1A7A2516A961F2838F7F94786A8811BE82F9CFFE READBYTES=1048643 "
+ + "REQUEST=1587991280.38 RESPONSE=1587991280.37 SOCKET=1587991280.37 "
+ + "SOURCE=op-nl2 SOURCEADDRESS=unknown START=1587991280.37 "
+ + "USED_AT=1587991286.62 USED_BY=71 WRITEBYTES=53";
+
+ private final String torperfResultTransfer1m3
+ = "BUILDTIMES=22.81,23.57,24.45 CIRC_ID=72 CONNECT=1587991880.37 "
+ + "DATACOMPLETE=1587991927.74 DATAPERC0=1587991910.74 "
+ + "DATAPERC10=1587991913.71 DATAPERC100=1587991927.74 "
+ + "DATAPERC20=1587991916.00 DATAPERC30=1587991917.92 "
+ + "DATAPERC40=1587991919.69 DATAPERC50=1587991921.80 "
+ + "DATAPERC60=1587991923.35 DATAPERC70=1587991924.91 "
+ + "DATAPERC80=1587991925.77 DATAPERC90=1587991927.04 "
+ + "DATAREQUEST=1587991909.80 DATARESPONSE=1587991910.74 DIDTIMEOUT=0 "
+ + "ENDPOINTLOCAL=localhost:127.0.0.1:41016 "
+ + "ENDPOINTPROXY=localhost:127.0.0.1:35900 "
+ + "ENDPOINTREMOTE=3czoq6qyehjio6lcdo4tb4vk5uv2bm4gfk5iacnawza22do6klsj7wy"
+ + "d.onion:0.0.0.0:443 FILESIZE=1048576 HOSTNAMELOCAL=op-nl2 "
+ + "HOSTNAMEREMOTE=op-nl2 LAUNCH=1587991881.70 NEGOTIATE=1587991880.37 "
+ + "PATH=$D5C6F62A5D1B3C711CA5E6F9D3772A432E96F6C2,"
+ + "$94EC34B871936504BE70671B44760BC99242E1F3,"
+ + "$E0F638ECCE918B5455CE29D2CD9ECC9DBD8F8B21 READBYTES=1048643 "
+ + "REQUEST=1587991880.37 RESPONSE=1587991880.37 SOCKET=1587991880.37 "
+ + "SOURCE=op-nl2 SOURCEADDRESS=unknown START=1587991880.37 "
+ + "USED_AT=1587991927.74 USED_BY=112 WRITEBYTES=53";
+
+ private final String torperfResultTransfer50k2
+ = "BUILDTIMES=0.09,0.15,0.27 CIRC_ID=49 CONNECT=1587991580.81 "
+ + "DATACOMPLETE=1587991580.80 DATAPERC10=0.0 DATAPERC100=0.0 "
+ + "DATAPERC20=0.0 DATAPERC30=0.0 DATAPERC40=0.0 DATAPERC50=0.0 "
+ + "DATAPERC60=0.0 DATAPERC70=0.0 DATAPERC80=0.0 DATAPERC90=0.0 "
+ + "DATAREQUEST=1587991580.80 DATARESPONSE=1587991580.80 DIDTIMEOUT=1 "
+ + "ENDPOINTLOCAL=localhost:127.0.0.1:40948 "
+ + "ENDPOINTPROXY=localhost:127.0.0.1:35900 "
+ + "ENDPOINTREMOTE=37.218.245.95:37.218.245.95:443 "
+ + "ERRORCODE=PROXY_END_MISC FILESIZE=51200 HOSTNAMELOCAL=op-nl2 "
+ + "HOSTNAMEREMOTE=(null) LAUNCH=1587991454.80 NEGOTIATE=1587991580.81 "
+ + "PATH=$12CF6DB4DAE106206D6C6B09988E865C0509843B,"
+ + "$1DC17C4A52A458B5C8B1E79157F8665696210E10,"
+ + "$39F17EC1BD41E652D1B80484D268E3933476FF42 READBYTES=0 "
+ + "REQUEST=1587991580.84 RESPONSE=1587991580.80 SOCKET=1587991580.81 "
+ + "SOURCE=op-nl2 SOURCEADDRESS=unknown START=1587991580.80 "
+ + "USED_AT=1587991580.84 USED_BY=93 WRITEBYTES=0";
+
+ @Test
+ public void testAsTorperfResults() throws IOException,
+ DescriptorParseException {
+ URL resouce = getClass().getClassLoader().getResource(
+ "onionperf/onionperf.analysis.json.xz");
+ assertNotNull(resouce);
+ InputStream compressedInputStream = resouce.openStream();
+ assertNotNull(compressedInputStream);
+ InputStream uncompressedInputStream = new XZCompressorInputStream(
+ compressedInputStream);
+ byte[] rawDescriptorBytes = IOUtils.toByteArray(uncompressedInputStream);
+ OnionPerfAnalysisConverter onionPerfAnalysisConverter
+ = new OnionPerfAnalysisConverter(rawDescriptorBytes, null);
+ for (Descriptor descriptor
+ : onionPerfAnalysisConverter.asTorperfResults()) {
+ assertTrue(descriptor instanceof TorperfResult);
+ String formattedTorperfResult
+ = new String(descriptor.getRawDescriptorBytes()).trim();
+ assertNotNull(formattedTorperfResult);
+ assertTrue(formattedTorperfResult.equals(torperfResultTransfer1m1)
+ || formattedTorperfResult.equals(torperfResultTransfer1m3)
+ || formattedTorperfResult.equals(torperfResultTransfer50k2));
+ }
+ }
+}
+
diff --git a/src/test/resources/onionperf/onionperf.analysis.json.xz b/src/test/resources/onionperf/onionperf.analysis.json.xz
new file mode 100644
index 0000000..41a4fa1
Binary files /dev/null and b/src/test/resources/onionperf/onionperf.analysis.json.xz differ
More information about the tor-commits
mailing list