commit 76dec0651148e99d069ab798693b22a33162b6b8 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Feb 22 20:59:31 2018 +0100

Add version_status field to details documents.

Implements #24256. --- CHANGELOG.md | 3 + .../torproject/onionoo/docs/DetailsDocument.java | 10 ++ .../org/torproject/onionoo/docs/DetailsStatus.java | 10 ++ .../org/torproject/onionoo/docs/NodeStatus.java | 17 +++ .../torproject/onionoo/server/ResponseBuilder.java | 2 + .../onionoo/updater/NodeDetailsStatusUpdater.java | 23 +++- .../org/torproject/onionoo/updater/TorVersion.java | 150 +++++++++++++++++++++ .../onionoo/updater/TorVersionStatus.java | 47 +++++++ .../onionoo/writer/DetailsDocumentWriter.java | 1 + .../torproject/onionoo/updater/TorVersionTest.java | 127 +++++++++++++++++ 10 files changed, 384 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a3c468..36d6471 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 5.1-1.12.0 - 2018-??-??

+ * Medium changes + - Add version_status field to details documents. + * Minor changes - Don't attempt to un-escape character sequences in contact lines (like "\uk") that only happen to start like escaped utf-8 characters diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index c749fba..e112efe 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -356,6 +356,16 @@ public class DetailsDocument extends Document { return this.version; }

+ private String version_status; + + public void setVersionStatus(String versionStatus) { + this.version_status = versionStatus; + } + + public String getVersionStatus() { + return this.version_status; + } + private SortedSet<String> alleged_family;

public void setAllegedFamily(SortedSet<String> allegedFamily) { diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index c3cdc28..f838ec0 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -550,5 +550,15 @@ public class DetailsStatus extends Document { public String getVersion() { return this.version; } + + private String version_status; + + public void setVersionStatus(String versionStatus) { + this.version_status = versionStatus; + } + + public String getVersionStatus() { + return this.version_status; + } }

diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 94da8c8..1ea8b98 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -3,6 +3,8 @@

package org.torproject.onionoo.docs;

+import org.torproject.onionoo.updater.TorVersionStatus; + import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -330,6 +332,16 @@ public class NodeStatus extends Document { return this.version; }

+ private TorVersionStatus versionStatus; + + public void setVersionStatus(TorVersionStatus versionStatus) { + this.versionStatus = versionStatus; + } + + public TorVersionStatus getVersionStatus() { + return this.versionStatus; + } + /* From exit lists: */

private SortedSet<String> exitAddresses; @@ -572,6 +584,9 @@ public class NodeStatus extends Document { if (parts.length >= 25 && !parts[24].isEmpty()) { nodeStatus.setHostName(parts[24]); } + if (parts.length >= 26) { + nodeStatus.setVersionStatus(TorVersionStatus.ofAbbreviation(parts[25])); + } return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " @@ -640,6 +655,8 @@ public class NodeStatus extends Document { .append((this.getVersion() != null ? this.getVersion() : "")); sb.append("\t") .append((this.getHostName() != null ? this.getHostName() : "")); + sb.append("\t").append(null != this.getVersionStatus() + ? this.getVersionStatus().getAbbreviation() : ""); return sb.toString(); } } diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index e2bdf82..fcf3c8f 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -339,6 +339,8 @@ public class ResponseBuilder { detailsDocument.getUnreachableOrAddresses()); } else if (field.equals("version")) { dd.setVersion(detailsDocument.getVersion()); + } else if (field.equals("version_status")) { + dd.setVersionStatus(detailsDocument.getVersionStatus()); } } /* Don't escape HTML characters, like < and >, contained in diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 4fdf98b..b8bd4f6 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -22,10 +22,8 @@ import org.slf4j.LoggerFactory;

import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; @@ -89,7 +87,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,

private SortedMap<String, Integer> lastBandwidthWeights = null;

- private Set<String> lastRecommendedServerVersions = null; + private SortedSet<TorVersion> lastRecommendedServerVersions = null;

private int relayConsensusesProcessed = 0;

@@ -306,8 +304,15 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.relayConsensusesProcessed++; if (this.relaysLastValidAfterMillis == validAfterMillis) { this.lastBandwidthWeights = consensus.getBandwidthWeights(); - this.lastRecommendedServerVersions - = new HashSet<>(consensus.getRecommendedServerVersions()); + this.lastRecommendedServerVersions = new TreeSet<>(); + for (String recommendedServerVersion : + consensus.getRecommendedServerVersions()) { + TorVersion recommendedTorServerVersion + = TorVersion.of(recommendedServerVersion); + if (null != recommendedTorServerVersion) { + this.lastRecommendedServerVersions.add(recommendedTorServerVersion); + } + } } }

@@ -798,8 +803,13 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, * the recommended_version field accordingly. */ if (null != this.lastRecommendedServerVersions && null != nodeStatus.getVersion()) { + TorVersion torVersion = TorVersion.of(nodeStatus.getVersion()); nodeStatus.setRecommendedVersion(this.lastRecommendedServerVersions - .contains(nodeStatus.getVersion())); + .contains(torVersion)); + nodeStatus.setVersionStatus(null != torVersion + ? torVersion.determineVersionStatus( + this.lastRecommendedServerVersions) + : TorVersionStatus.UNRECOMMENDED); }

Map<String, Long> exitAddresses = new HashMap<>(); @@ -909,6 +919,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, detailsStatus.setLastChangedOrAddressOrPort( nodeStatus.getLastChangedOrAddressOrPort()); detailsStatus.setVersion(nodeStatus.getVersion()); + detailsStatus.setVersionStatus(nodeStatus.getVersionStatus().toString());

this.documentStore.store(detailsStatus, fingerprint); this.documentStore.store(nodeStatus, fingerprint); diff --git a/src/main/java/org/torproject/onionoo/updater/TorVersion.java b/src/main/java/org/torproject/onionoo/updater/TorVersion.java new file mode 100644 index 0000000..d8e1683 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/updater/TorVersion.java @@ -0,0 +1,150 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.updater; + +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; + +/** + * Helper class to compare Tor versions. + * + * <p>Based on "How Tor Version Numbers Work", available at + * https://gitweb.torproject.org/torspec.git/tree/version-spec.txt</p> + */ +public class TorVersion implements Comparable<TorVersion> { + + private int majorVersion; + + private int minorVersion; + + private int microVersion; + + private String releaseSeries; + + private Integer patchLevel = null; + + private String statusTag = null; + + private static Map<String, TorVersion> knownVersions = new HashMap<>(); + + private TorVersion() { + } + + /** Return a TorVersion instance from the given tor version string that can be + * compared to other tor version strings, or null if the given string is not a + * valid tor version. */ + public static TorVersion of(String versionString) { + if (null == versionString) { + return null; + } + if (!knownVersions.containsKey(versionString)) { + TorVersion result = new TorVersion(); + String[] components = versionString.split("-")[0].split("\."); + try { + result.majorVersion = Integer.parseInt(components[0]); + result.minorVersion = Integer.parseInt(components[1]); + result.microVersion = Integer.parseInt(components[2]); + result.releaseSeries = String.format("%d.%d.%d", + result.majorVersion, result.minorVersion, result.microVersion); + if (components.length == 4) { + result.patchLevel = Integer.parseInt(components[3]); + if (versionString.contains("-")) { + result.statusTag = versionString.split("-", 2)[1].split(" ")[0]; + } + } + } catch (ArrayIndexOutOfBoundsException + | NumberFormatException exception) { + result = null; + } + knownVersions.put(versionString, result); + } + return knownVersions.get(versionString); + } + + @Override + public int compareTo(TorVersion other) { + if (null == other) { + throw new NullPointerException(); + } + int result; + if ((result = Integer.compare(this.majorVersion, + other.majorVersion)) != 0) { + return result; + } + if ((result = Integer.compare(this.minorVersion, + other.minorVersion)) != 0) { + return result; + } + if ((result = Integer.compare(this.microVersion, + other.microVersion)) != 0) { + return result; + } + if (null == this.patchLevel && null == other.patchLevel) { + return 0; + } else if (null == patchLevel) { + return -1; + } else if (null == other.patchLevel) { + return 1; + } else if ((result = Integer.compare(this.patchLevel, + other.patchLevel)) != 0) { + return result; + } + if (null == this.statusTag && null == other.statusTag) { + return 0; + } else if (null == this.statusTag) { + return -1; + } else if (null == other.statusTag) { + return 1; + } else { + return this.statusTag.compareTo(other.statusTag); + } + } + + @Override + public boolean equals(Object other) { + return null != other && other instanceof TorVersion + && this.compareTo((TorVersion) other) == 0; + } + + @Override + public int hashCode() { + return 2 * Integer.hashCode(this.majorVersion) + + 3 * Integer.hashCode(this.minorVersion) + + 5 * Integer.hashCode(this.microVersion) + + 7 * (null == this.patchLevel ? 0 : this.patchLevel) + + 11 * (null == this.statusTag ? 0 : this.statusTag.hashCode()); + } + + /** Determine the version status of this tor version in the context of the + * given recommended tor versions. */ + public TorVersionStatus determineVersionStatus( + SortedSet<TorVersion> recommendedVersions) { + if (recommendedVersions.contains(this)) { + return TorVersionStatus.RECOMMENDED; + } else if (this.compareTo(recommendedVersions.last()) > 0) { + return TorVersionStatus.EXPERIMENTAL; + } else if (this.compareTo(recommendedVersions.first()) < 0) { + return TorVersionStatus.OBSOLETE; + } else { + boolean seriesHasRecommendedVersions = false; + boolean notNewInSeries = false; + for (TorVersion recommendedVersion : recommendedVersions) { + if (this.releaseSeries.equals( + recommendedVersion.releaseSeries)) { + seriesHasRecommendedVersions = true; + if (this.compareTo(recommendedVersion) < 0) { + notNewInSeries = true; + } + } + } + if (seriesHasRecommendedVersions && !notNewInSeries) { + return TorVersionStatus.NEW_IN_SERIES; + } else { + return TorVersionStatus.UNRECOMMENDED; + } + } + } +} + diff --git a/src/main/java/org/torproject/onionoo/updater/TorVersionStatus.java b/src/main/java/org/torproject/onionoo/updater/TorVersionStatus.java new file mode 100644 index 0000000..5825064 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/updater/TorVersionStatus.java @@ -0,0 +1,47 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.updater; + +import java.util.HashMap; +import java.util.Map; + +public enum TorVersionStatus { + + RECOMMENDED("recommended", "r"), + EXPERIMENTAL("experimental", "e"), + OBSOLETE("obsolete", "o"), + NEW_IN_SERIES("new in series", "n"), + UNRECOMMENDED("unrecommended", "u"); + + private final String statusString; + + private final String abbreviation; + + TorVersionStatus(String statusString, String abbreviation) { + this.statusString = statusString; + this.abbreviation = abbreviation; + } + + public String getAbbreviation() { + return this.abbreviation; + } + + private static Map<String, TorVersionStatus> byAbbreviation = new HashMap<>(); + + static { + for (TorVersionStatus status : TorVersionStatus.values()) { + byAbbreviation.put(status.abbreviation, status); + } + } + + public static TorVersionStatus ofAbbreviation(String abbrevation) { + return byAbbreviation.get(abbrevation); + } + + @Override + public String toString() { + return this.statusString; + } +} + diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 8490f05..eca7874 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -162,6 +162,7 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setUnreachableOrAddresses(unreachableOrAddresses); } detailsDocument.setVersion(detailsStatus.getVersion()); + detailsDocument.setVersionStatus(detailsStatus.getVersionStatus()); this.documentStore.store(detailsDocument, fingerprint); }

diff --git a/src/test/java/org/torproject/onionoo/updater/TorVersionTest.java b/src/test/java/org/torproject/onionoo/updater/TorVersionTest.java new file mode 100644 index 0000000..b6a9764 --- /dev/null +++ b/src/test/java/org/torproject/onionoo/updater/TorVersionTest.java @@ -0,0 +1,127 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.updater; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import java.util.Arrays; +import java.util.Collection; +import java.util.SortedSet; +import java.util.TreeSet; + +@RunWith(Enclosed.class) +public class TorVersionTest { + + @RunWith(Parameterized.class) + public static class TorVersionStatusTest { + + private static String[] recommendedVersionStrings = new String[] { + "0.2.5.16", "0.2.5.17", "0.2.9.14", "0.2.9.15", "0.3.1.9", "0.3.1.10", + "0.3.2.8-rc", "0.3.2.9", "0.3.3.1-alpha", "0.3.3.2-alpha" }; + + /** Provide test data. */ + @Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + { "Recommended version", "0.2.5.16", "recommended" }, + { "Recommended version", "0.3.2.8-rc", "recommended" }, + { "Recommended version", "0.3.3.2-alpha", "recommended" }, + { "Experimental version", "0.3.3.2-alpha-dev", "experimental" }, + { "Experimental version", "0.3.3.3-alpha", "experimental" }, + { "Experimental version", "0.3.4.0-alpha-dev", "experimental" }, + { "Obsolete version", "0.2.5.15", "obsolete" }, + { "Obsolete version", "0.1.0.1-rc", "obsolete" }, + { "New-in-series version", "0.2.5.18", "new in series" }, + { "New-in-series version", "0.3.2.9-dev", "new in series" }, + { "New-in-series version", "0.3.2.10", "new in series" }, + { "Unrecommended version", "0.2.9.13", "unrecommended" }, + { "Unrecommended version", "0.3.1.8-dev", "unrecommended" }, + { "Unrecommended version", "0.3.3.0-alpha-dev", "unrecommended" }, + { "Unrecognized (experimental) version", "1.0-final", + "unrecommended" }, + { "Unrecognized (obsolete) version", "0.0.2pre13", "unrecommended" } + }); + } + + @Parameter + public String testDescription; + + @Parameter(1) + public String versionString; + + @Parameter(2) + public String expectedVersionStatus; + + @Test + public void test() { + SortedSet<TorVersion> recommendedTorVersions = new TreeSet<>(); + for (String recommendedVersionString : recommendedVersionStrings) { + recommendedTorVersions.add(TorVersion.of(recommendedVersionString)); + } + TorVersion torVersion = TorVersion.of(this.versionString); + String determinedVersionStatus = "unrecommended"; + if (null != torVersion) { + determinedVersionStatus = torVersion + .determineVersionStatus(recommendedTorVersions).toString(); + } + assertEquals(this.testDescription, this.expectedVersionStatus, + determinedVersionStatus); + } + } + + @RunWith(Parameterized.class) + public static class TorVersionEqualsHashCodeCompareToTest { + + /** Provide test data. */ + @Parameters + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][] { + { "0.2.5.16", "0.2.5.16", true, true, 0 }, + { "0.2.5.16", "0.2.5.17", false, false, -1 }, + { "0.3.3.1-alpha", "0.3.3.1-alpha", true, true, 0 }, + { "0.1.2.3", "00.01.02.03", true, true, 0 }, + { "0.1.2.3-alpha", "00.01.02.03-aallpphhaa", false, false, 1 } + }); + } + + @Parameter + public String firstVersionString; + + @Parameter(1) + public String secondVersionString; + + @Parameter(2) + public boolean expectedEqualsResult; + + @Parameter(3) + public boolean expectedSameHashCodes; + + @Parameter(4) + public int expectedCompareToResult; + + @Test + public void test() { + TorVersion firstVersion = TorVersion.of(this.firstVersionString); + TorVersion secondVersion = TorVersion.of(this.secondVersionString); + assertEquals(this.expectedEqualsResult, + firstVersion.equals(secondVersion)); + if (this.expectedSameHashCodes) { + assertEquals(firstVersion.hashCode(), secondVersion.hashCode()); + } + int actualCompareToResult = firstVersion.compareTo(secondVersion); + assertTrue(this.expectedCompareToResult < 0 && actualCompareToResult < 0 + || this.expectedCompareToResult == 0 && actualCompareToResult == 0 + || this.expectedCompareToResult > 0 && actualCompareToResult > 0); + } + } +} +