commit 05a1cf7e7d79bbbad4ece6dabe601cd37d4dfd1d Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed May 30 10:59:09 2012 +0200

Parse new .tpf Torperf data format. --- src/org/torproject/descriptor/TorperfResult.java | 93 ++++ .../torproject/descriptor/impl/DescriptorImpl.java | 12 +- .../descriptor/impl/DescriptorParserImpl.java | 3 +- .../descriptor/impl/TorperfResultImpl.java | 466 ++++++++++++++++++++ 4 files changed, 570 insertions(+), 4 deletions(-)

diff --git a/src/org/torproject/descriptor/TorperfResult.java b/src/org/torproject/descriptor/TorperfResult.java new file mode 100644 index 0000000..f1b7a9f --- /dev/null +++ b/src/org/torproject/descriptor/TorperfResult.java @@ -0,0 +1,93 @@ +/* Copyright 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.descriptor; + +import java.util.List; +import java.util.SortedMap; + +public interface TorperfResult extends Descriptor { + + /* Return the configured name of the data source. */ + public String getSource(); + + /* Return the configured file size in bytes. */ + public int getFileSize(); + + /* Return the time when the connection process starts. */ + public long getStartMillis(); + + /* Return the time when the socket was created. */ + public long getSocketMillis(); + + /* Return the time when the socket was connected. */ + public long getConnectMillis(); + + /* Return the time when SOCKS 5 authentication methods have been + * negotiated. */ + public long getNegotiateMillis(); + + /* Return the time when the SOCKS request was sent. */ + public long getRequestMillis(); + + /* Return the time when the SOCKS response was received. */ + public long getResponseMillis(); + + /* Return the time when the HTTP request was written. */ + public long getDataRequestMillis(); + + /* Return the time when the first response was received. */ + public long getDataResponseMillis(); + + /* Return the time when the payload was complete. */ + public long getDataCompleteMillis(); + + /* Return the total number of bytes written. */ + public int getWriteBytes(); + + /* Return the total number of bytes read. */ + public int getReadBytes(); + + /* Return whether the request timed out (as opposed to failing), or null + * if the torperf line didn't contain that information. */ + public Boolean didTimeout(); + + /* Return the times when x% of expected bytes were read for x = { 10, + * 20, 30, 40, 50, 60, 70, 80, 90 }, or null if the torperf line didn't + * contain that information. */ + public SortedMap<Integer, Long> getDataPercentiles(); + + /* Return the time when the circuit was launched, or -1 if the torperf + * line didn't contain that information. */ + public long getLaunchMillis(); + + /* Return the time when the circuit was used, or -1 if the torperf line + * didn't contain that information. */ + public long getUsedAtMillis(); + + /* Return a list of fingerprints of the relays in the circuit, or null + * if the torperf line didn't contain that information. */ + public List<String> getPath(); + + /* Return a list of times in millis when circuit hops were built, or + * null if the torperf line didn't contain that information. */ + public List<Long> getBuildTimes(); + + /* Return the circuit build timeout that the Tor client used when + * building this circuit, or -1 if the torperf line didn't contain that + * information. */ + public long getTimeout(); + + /* Return the circuit build time quantile that the Tor client uses to + * determine its circuit-build timeout, or -1.0 if the torperf line + * didn't contain that information. */ + public double getQuantile(); + + /* Return the identifier of the circuit used for this measurement, or -1 + * if the torperf line didn't contain that information. */ + public int getCircId(); + + /* Return the identifier of the stream used for this measurement, or -1 + * if the torperf line didn't contain that information. */ + public int getUsedBy(); +} + diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index 27e5153..a4f1613 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -13,7 +13,7 @@ import org.torproject.descriptor.Descriptor;

public abstract class DescriptorImpl implements Descriptor {

- protected static List<Descriptor> parseRelayOrBridgeDescriptors( + protected static List<Descriptor> parseDescriptors( byte[] rawDescriptorBytes, String fileName, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { @@ -83,6 +83,9 @@ public abstract class DescriptorImpl implements Descriptor { firstLines.contains("\nsigned-directory\n")) { parsedDescriptors.add(new RelayDirectoryImpl(rawDescriptorBytes, failUnrecognizedDescriptorLines)); + } else if (firstLines.startsWith("@type torperf 1.0\n")) { + parsedDescriptors.addAll(TorperfResultImpl.parseTorperfResults( + rawDescriptorBytes, failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); @@ -156,12 +159,17 @@ public abstract class DescriptorImpl implements Descriptor {

/* Parse annotation lines from the descriptor bytes. */ private List<String> annotations = new ArrayList<String>(); - private void cutOffAnnotations(byte[] rawDescriptorBytes) { + private void cutOffAnnotations(byte[] rawDescriptorBytes) + throws DescriptorParseException { String ascii = new String(rawDescriptorBytes); int start = 0; while ((start == 0 && ascii.startsWith("@")) || (start > 0 && ascii.indexOf("\n@", start - 1) >= 0)) { int end = ascii.indexOf("\n", start); + if (end < 0) { + throw new DescriptorParseException("Annotation line does not " + + "contain a newline."); + } this.annotations.add(ascii.substring(start, end)); start = end + 1; } diff --git a/src/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/org/torproject/descriptor/impl/DescriptorParserImpl.java index 2ee6f40..c1d2ae7 100644 --- a/src/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -19,8 +19,7 @@ public class DescriptorParserImpl implements DescriptorParser {

public List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, String fileName) throws DescriptorParseException { - return DescriptorImpl.parseRelayOrBridgeDescriptors( - rawDescriptorBytes, fileName, + return DescriptorImpl.parseDescriptors(rawDescriptorBytes, fileName, this.failUnrecognizedDescriptorLines); } } diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java new file mode 100644 index 0000000..6a8c1c7 --- /dev/null +++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -0,0 +1,466 @@ +/* Copyright 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.descriptor.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Scanner; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.TorperfResult; + +public class TorperfResultImpl extends DescriptorImpl + implements TorperfResult { + + public static List<Descriptor> parseTorperfResults( + byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + if (rawDescriptorBytes.length == 0) { + throw new DescriptorParseException("Descriptor is empty."); + } + List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); + String descriptorString = new String(rawDescriptorBytes); + Scanner s = new Scanner(descriptorString).useDelimiter("\n"); + while (s.hasNext()) { + String line = s.next(); + if (line.startsWith("@type torperf ")) { + String[] parts = line.split(" "); + if (parts.length != 3) { + throw new DescriptorParseException("Illegal line '" + line + + "'."); + } + String version = parts[2]; + if (!version.startsWith("1.")) { + throw new DescriptorParseException("Unsupported version in " + + " line '" + line + "'."); + } + } else { + parsedDescriptors.add(new TorperfResultImpl(line.getBytes(), + failUnrecognizedDescriptorLines)); + } + } + return parsedDescriptors; + } + + protected TorperfResultImpl(byte[] rawDescriptorBytes, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + super(rawDescriptorBytes, failUnrecognizedDescriptorLines, false); + this.parseTorperfResultLine(new String(rawDescriptorBytes)); + } + + private void parseTorperfResultLine(String line) + throws DescriptorParseException { + if (line.isEmpty()) { + throw new DescriptorParseException("Blank lines are not allowed."); + } + String[] parts = line.split(" "); + for (int i = 0; i < parts.length; i++) { + String keyValue = parts[i]; + String[] keyValueParts = keyValue.split("="); + if (keyValueParts.length != 2) { + throw new DescriptorParseException("Illegal key-value pair in " + + "line '" + line + "'."); + } + String key = keyValueParts[0]; + this.markKeyAsParsed(key, line); + String value = keyValueParts[1]; + if (key.equals("SOURCE")) { + this.parseSource(value, keyValue, line); + } else if (key.equals("FILESIZE")) { + this.parseFileSize(value, keyValue, line); + } else if (key.equals("START")) { + this.parseStart(value, keyValue, line); + } else if (key.equals("SOCKET")) { + this.parseSocket(value, keyValue, line); + } else if (key.equals("CONNECT")) { + this.parseConnect(value, keyValue, line); + } else if (key.equals("NEGOTIATE")) { + this.parseNegotiate(value, keyValue, line); + } else if (key.equals("REQUEST")) { + this.parseRequest(value, keyValue, line); + } else if (key.equals("RESPONSE")) { + this.parseResponse(value, keyValue, line); + } else if (key.equals("DATAREQUEST")) { + this.parseDataRequest(value, keyValue, line); + } else if (key.equals("DATARESPONSE")) { + this.parseDataResponse(value, keyValue, line); + } else if (key.equals("DATACOMPLETE")) { + this.parseDataComplete(value, keyValue, line); + } else if (key.equals("WRITEBYTES")) { + this.parseWriteBytes(value, keyValue, line); + } else if (key.equals("READBYTES")) { + this.parseReadBytes(value, keyValue, line); + } else if (key.equals("DIDTIMEOUT")) { + this.parseDidTimeout(value, keyValue, line); + } else if (key.startsWith("DATAPERC")) { + this.parseDataPercentile(value, keyValue, line); + } else if (key.equals("LAUNCH")) { + this.parseLaunch(value, keyValue, line); + } else if (key.equals("USED_AT")) { + this.parseUsedAt(value, keyValue, line); + } else if (key.equals("PATH")) { + this.parsePath(value, keyValue, line); + } else if (key.equals("BUILDTIMES")) { + this.parseBuildTimes(value, keyValue, line); + } else if (key.equals("TIMEOUT")) { + this.parseTimeout(value, keyValue, line); + } else if (key.equals("QUANTILE")) { + this.parseQuantile(value, keyValue, line); + } else if (key.equals("CIRC_ID")) { + this.parseCircId(value, keyValue, line); + } else if (key.equals("USED_BY")) { + this.parseUsedBy(value, keyValue, line); + } else if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized key '" + key + + "' in line '" + line + "'."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } + } + this.checkAllRequiredKeysParsed(line); + } + + private Set<String> parsedKeys = new HashSet<String>(); + private Set<String> requiredKeys = new HashSet<String>(Arrays.asList( + ("SOURCE,FILESIZE,START,SOCKET,CONNECT,NEGOTIATE,REQUEST,RESPONSE," + + "DATAREQUEST,DATARESPONSE,DATACOMPLETE,WRITEBYTES,READBYTES"). + split(","))); + private void markKeyAsParsed(String key, String line) + throws DescriptorParseException { + if (this.parsedKeys.contains(key)) { + throw new DescriptorParseException("Key '" + key + "' is contained " + + "at least twice in line '" + line + "', but must be " + + "contained at most once."); + } + this.parsedKeys.add(key); + this.requiredKeys.remove(key); + } + private void checkAllRequiredKeysParsed(String line) + throws DescriptorParseException { + for (String key : this.requiredKeys) { + throw new DescriptorParseException("Key '" + key + "' is contained " + + "contained 0 times in line '" + line + "', but must be " + + "contained exactly once."); + } + } + + private void parseSource(String value, String keyValue, String line) + throws DescriptorParseException { + this.source = value; + } + + private void parseFileSize(String value, String keyValue, String line) + throws DescriptorParseException { + try { + this.fileSize = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal value in '" + keyValue + + "' in line '" + line + "'."); + } + } + + private void parseStart(String value, String keyValue, String line) + throws DescriptorParseException { + this.startMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseSocket(String value, String keyValue, String line) + throws DescriptorParseException { + this.socketMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseConnect(String value, String keyValue, String line) + throws DescriptorParseException { + this.connectMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseNegotiate(String value, String keyValue, String line) + throws DescriptorParseException { + this.negotiateMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseRequest(String value, String keyValue, String line) + throws DescriptorParseException { + this.requestMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseResponse(String value, String keyValue, String line) + throws DescriptorParseException { + this.responseMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseDataRequest(String value, String keyValue, + String line) throws DescriptorParseException { + this.dataRequestMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseDataResponse(String value, String keyValue, + String line) throws DescriptorParseException { + this.dataResponseMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseDataComplete(String value, String keyValue, + String line) throws DescriptorParseException { + this.dataCompleteMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseWriteBytes(String value, String keyValue, String line) + throws DescriptorParseException { + this.writeBytes = parseInt(value, keyValue, line); + } + + private void parseReadBytes(String value, String keyValue, String line) + throws DescriptorParseException { + this.readBytes = parseInt(value, keyValue, line); + } + + private void parseDidTimeout(String value, String keyValue, String line) + throws DescriptorParseException { + if (value.equals("1")) { + this.didTimeout = true; + } else if (value.equals("0")) { + this.didTimeout = false; + } else { + throw new DescriptorParseException("Illegal value in '" + keyValue + + "' in line '" + line + "'."); + } + } + + private Set<String> unparsedPercentiles = new HashSet<String>( + Arrays.asList("10,20,30,40,50,60,70,80,90".split(","))); + private void parseDataPercentile(String value, String keyValue, + String line) throws DescriptorParseException { + String percentileString = keyValue.substring("DATAPERC".length(), + keyValue.indexOf("=")); + if (!unparsedPercentiles.contains(percentileString)) { + throw new DescriptorParseException("Illegal value in '" + keyValue + + "' in line '" + line + "'."); + } + unparsedPercentiles.remove(percentileString); + if (this.dataPercentiles == null) { + this.dataPercentiles = new TreeMap<Integer, Long>(); + } + int percentile = Integer.parseInt(percentileString); + long timestamp = this.parseTimestamp(value, keyValue, line); + this.dataPercentiles.put(percentile, timestamp); + } + + private void parseLaunch(String value, String keyValue, String line) + throws DescriptorParseException { + this.launchMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parseUsedAt(String value, String keyValue, String line) + throws DescriptorParseException { + this.usedAtMillis = this.parseTimestamp(value, keyValue, line); + } + + private void parsePath(String value, String keyValue, String line) + throws DescriptorParseException { + this.path = new ArrayList<String>(); + for (String fingerprint : value.split(",")) { + if (fingerprint.length() != 41) { + throw new DescriptorParseException("Illegal value in '" + keyValue + + "' in line '" + line + "'."); + } + this.path.add(ParseHelper.parseTwentyByteHexString(line, + fingerprint.substring(1))); + } + } + + private void parseBuildTimes(String value, String keyValue, String line) + throws DescriptorParseException { + this.buildTimes = new ArrayList<Long>(); + for (String buildTimeString : value.split(",")) { + this.buildTimes.add(this.parseTimestamp(buildTimeString, keyValue, + line)); + } + } + + private void parseTimeout(String value, String keyValue, String line) + throws DescriptorParseException { + this.timeout = this.parseInt(value, keyValue, line); + } + + private void parseQuantile(String value, String keyValue, String line) + throws DescriptorParseException { + this.quantile = this.parseDouble(value, keyValue, line); + } + + private void parseCircId(String value, String keyValue, String line) + throws DescriptorParseException { + this.circId = this.parseInt(value, keyValue, line); + } + + private void parseUsedBy(String value, String keyValue, String line) + throws DescriptorParseException { + this.usedBy = this.parseInt(value, keyValue, line); + } + + private long parseTimestamp(String value, String keyValue, String line) + throws DescriptorParseException { + long timestamp = -1L; + if (value.contains(".") && value.split("\.").length == 2) { + String zeroPaddedValue = (value + "000"); + String threeDecimalPlaces = zeroPaddedValue.substring(0, + zeroPaddedValue.indexOf(".") + 4); + String millisString = threeDecimalPlaces.replaceAll("\.", ""); + try { + timestamp = Long.parseLong(millisString); + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (timestamp < 0L) { + throw new DescriptorParseException("Illegal timestamp '" + value + "' in '" + + keyValue + "' in line '" + line + "'."); + } + return timestamp; + } + + private int parseInt(String value, String keyValue, String line) + throws DescriptorParseException { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal value in '" + keyValue + + " in line '" + line + "'."); + } + } + + private double parseDouble(String value, String keyValue, String line) + throws DescriptorParseException { + try { + return Double.parseDouble(value); + } catch (NumberFormatException e) { + throw new DescriptorParseException("Illegal value in '" + keyValue + + "' in line '" + line + "'."); + } + } + + private String source; + public String getSource() { + return this.source; + } + + private int fileSize; + public int getFileSize() { + return this.fileSize; + } + + private long startMillis; + public long getStartMillis() { + return this.startMillis; + } + + private long socketMillis; + public long getSocketMillis() { + return this.socketMillis; + } + + private long connectMillis; + public long getConnectMillis() { + return this.connectMillis; + } + + private long negotiateMillis; + public long getNegotiateMillis() { + return this.negotiateMillis; + } + + private long requestMillis; + public long getRequestMillis() { + return this.requestMillis; + } + + private long responseMillis; + public long getResponseMillis() { + return this.responseMillis; + } + + private long dataRequestMillis; + public long getDataRequestMillis() { + return this.dataRequestMillis; + } + + private long dataResponseMillis; + public long getDataResponseMillis() { + return this.dataResponseMillis; + } + + private long dataCompleteMillis; + public long getDataCompleteMillis() { + return this.dataCompleteMillis; + } + + private int writeBytes; + public int getWriteBytes() { + return this.writeBytes; + } + + private int readBytes; + public int getReadBytes() { + return this.readBytes; + } + + private boolean didTimeout; + public Boolean didTimeout() { + return this.didTimeout; + } + + private SortedMap<Integer, Long> dataPercentiles; + public SortedMap<Integer, Long> getDataPercentiles() { + return this.dataPercentiles == null ? null : + new TreeMap<Integer, Long>(this.dataPercentiles); + } + + private long launchMillis = -1L; + public long getLaunchMillis() { + return this.launchMillis; + } + + private long usedAtMillis = -1L; + public long getUsedAtMillis() { + return this.usedAtMillis; + } + + private List<String> path; + public List<String> getPath() { + return new ArrayList<String>(this.path); + } + + private List<Long> buildTimes; + public List<Long> getBuildTimes() { + return new ArrayList<Long>(this.buildTimes); + } + + private long timeout = -1L; + public long getTimeout() { + return this.timeout; + } + + private double quantile = -1.0; + public double getQuantile() { + return this.quantile; + } + + private int circId = -1; + public int getCircId() { + return this.circId; + } + + private int usedBy = -1; + public int getUsedBy() { + return this.usedBy; + } +} +