commit 249c8549c0536e7c8e71b5f7f500f81b2f203e6d
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Aug 7 23:06:36 2017 +0200
Add version parameter to filter for Tor version.
Implements #21427.
Requires minor protocol update.
---
.../org/torproject/onionoo/docs/DocumentStore.java | 3 +-
.../org/torproject/onionoo/docs/NodeStatus.java | 14 +++++
.../torproject/onionoo/docs/SummaryDocument.java | 13 +++-
.../org/torproject/onionoo/server/NodeIndex.java | 10 +++
.../org/torproject/onionoo/server/NodeIndexer.java | 8 +++
.../torproject/onionoo/server/RequestHandler.java | 26 ++++++++
.../torproject/onionoo/server/ResourceServlet.java | 24 +++++++-
.../onionoo/updater/NodeDetailsStatusUpdater.java | 2 +
.../onionoo/writer/SummaryDocumentWriter.java | 3 +-
.../onionoo/docs/SummaryDocumentTest.java | 2 +-
.../onionoo/server/ResourceServletTest.java | 72 +++++++++++++++++++---
.../server/SummaryDocumentComparatorTest.java | 2 +-
12 files changed, 166 insertions(+), 13 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
index 56ca406..7819d0f 100644
--- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
@@ -435,13 +435,14 @@ public class DocumentStore {
}
SortedSet<String> relayFlags = new TreeSet<>();
SortedSet<String> family = null;
+ String version = null;
long lastSeenMillis = -1L;
long consensusWeight = -1L;
long firstSeenMillis = -1L;
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, family, family);
+ asNumber, contact, family, family, version);
return summaryDocument;
}
diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
index 6a71fb6..759d83e 100644
--- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
@@ -321,6 +321,16 @@ public class NodeStatus extends Document {
return this.recommendedVersion;
}
+ private String version;
+
+ public void setVersion(String version) {
+ this.version = version.substring(version.lastIndexOf(" ") + 1);
+ }
+
+ public String getVersion() {
+ return this.version;
+ }
+
/* From exit lists: */
private SortedSet<String> exitAddresses;
@@ -547,6 +557,9 @@ public class NodeStatus extends Document {
extendedFamily.addAll(indirectFamily);
nodeStatus.setExtendedFamily(extendedFamily);
}
+ if (parts.length >= 24 && !parts[23].isEmpty()) {
+ nodeStatus.setVersion(parts[23]);
+ }
return nodeStatus;
} catch (NumberFormatException e) {
log.error("Number format exception while parsing node "
@@ -610,6 +623,7 @@ public class NodeStatus extends Document {
sb.append("\t" + StringUtils.join(this.getAllegedFamily(), ";") + ":"
+ StringUtils.join(this.getEffectiveFamily(), ";") + ":"
+ StringUtils.join(this.getIndirectFamily(), ";"));
+ sb.append("\t" + (this.getVersion() != null ? this.getVersion() : ""));
return sb.toString();
}
}
diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
index 3ca4960..527f91d 100644
--- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
@@ -269,6 +269,16 @@ public class SummaryDocument extends Document {
return this.stringArrayToSortedSet(this.ef);
}
+ private String v;
+
+ public void setVersion(String version) {
+ this.v = version;
+ }
+
+ public String getVersion() {
+ return this.v;
+ }
+
/* The familyFingerprints parameter can go away after September 8, 2015.
* See above. */
/** Instantiates a summary document with all given properties. */
@@ -277,7 +287,7 @@ public class SummaryDocument extends Document {
boolean running, SortedSet<String> relayFlags, long consensusWeight,
String countryCode, long firstSeenMillis, String asNumber,
String contact, SortedSet<String> familyFingerprints,
- SortedSet<String> effectiveFamily) {
+ SortedSet<String> effectiveFamily, String version) {
this.setRelay(isRelay);
this.setNickname(nickname);
this.setFingerprint(fingerprint);
@@ -292,6 +302,7 @@ public class SummaryDocument extends Document {
this.setContact(contact);
this.setFamilyFingerprints(familyFingerprints);
this.setEffectiveFamily(effectiveFamily);
+ this.setVersion(version);
}
}
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
index 439d302..dd4bd4d 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
@@ -169,5 +169,15 @@ class NodeIndex {
public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() {
return bridgesByLastSeenDays;
}
+
+ private Map<String, Set<String>> relaysByVersion;
+
+ public void setRelaysByVersion(Map<String, Set<String>> relaysByVersion) {
+ this.relaysByVersion = relaysByVersion;
+ }
+
+ public Map<String, Set<String>> getRelaysByVersion() {
+ return this.relaysByVersion;
+ }
}
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index f6b9510..7ed9ea6 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -150,6 +150,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
Map<String, Set<String>> newBridgesByFlag = new HashMap<>();
Map<String, Set<String>> newRelaysByContact = new HashMap<>();
Map<String, Set<String>> newRelaysByFamily = new HashMap<>();
+ Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>();
SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>();
SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>();
@@ -246,6 +247,12 @@ public class NodeIndexer implements ServletContextListener, Runnable {
}
newRelaysByContact.get(contact).add(fingerprint);
newRelaysByContact.get(contact).add(hashedFingerprint);
+ String version = entry.getVersion();
+ if (!newRelaysByVersion.containsKey(version)) {
+ newRelaysByVersion.put(version, new HashSet<String>());
+ }
+ newRelaysByVersion.get(version).add(fingerprint);
+ newRelaysByVersion.get(version).add(hashedFingerprint);
}
/* This loop can go away once all Onionoo services had their hourly
* updater write effective families to summary documents at least
@@ -319,6 +326,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays);
newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis);
newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis);
+ newNodeIndex.setRelaysByVersion(newRelaysByVersion);
synchronized (this) {
this.lastIndexed = updateStatusMillis;
this.latestNodeIndex = newNodeIndex;
diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
index 98ece58..1e08c24 100644
--- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java
+++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
@@ -7,6 +7,9 @@ import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.SummaryDocument;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -91,6 +94,12 @@ public class RequestHandler {
System.arraycopy(contact, 0, this.contact, 0, contact.length);
}
+ private String version;
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
private String[] order;
public void setOrder(String[] order) {
@@ -158,6 +167,7 @@ public class RequestHandler {
this.filterNodesByLastSeenDays();
this.filterByContact();
this.filterByFamily();
+ this.filterByVersion();
this.order();
this.offset();
this.limit();
@@ -514,6 +524,22 @@ public class RequestHandler {
this.filteredBridges.clear();
}
+ private void filterByVersion() {
+ if (null == this.version) {
+ /* Not filtering by version. */
+ return;
+ }
+ Set<String> keepRelays = new HashSet<>();
+ for (Map.Entry<String, Set<String>> e
+ : this.nodeIndex.getRelaysByVersion().entrySet()) {
+ if (e.getKey().startsWith(this.version)) {
+ keepRelays.addAll(e.getValue());
+ }
+ }
+ this.filteredRelays.keySet().retainAll(keepRelays);
+ this.filteredBridges.clear();
+ }
+
private void order() {
List<SummaryDocument> uniqueRelays = new ArrayList<>();
List<SummaryDocument> uniqueBridges = new ArrayList<>();
diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
index 869574c..164b770 100644
--- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
+++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
@@ -3,6 +3,8 @@
package org.torproject.onionoo.server;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
@@ -66,7 +68,7 @@ public class ResourceServlet extends HttpServlet {
private static Set<String> knownParameters = new HashSet<>(
Arrays.asList(("type,running,search,lookup,fingerprint,country,as,"
+ "flag,first_seen_days,last_seen_days,contact,order,limit,"
- + "offset,fields,family").split(",")));
+ + "offset,fields,family,version").split(",")));
private static Set<String> illegalSearchQualifiers =
new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -275,6 +277,15 @@ public class ResourceServlet extends HttpServlet {
}
rh.setContact(contactParts);
}
+ if (parameterMap.containsKey("version")) {
+ String versionParameter = this.parseVersionParameter(
+ parameterMap.get("version"));
+ if (null == versionParameter) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setVersion(versionParameter);
+ }
if (parameterMap.containsKey("order")) {
String[] order = this.parseOrderParameter(parameterMap.get("order"));
if (order == null) {
@@ -532,5 +543,16 @@ public class ResourceServlet extends HttpServlet {
}
return parameter.toLowerCase().split(",");
}
+
+ private static Pattern versionParameterPattern =
+ Pattern.compile("^[0-9a-zA-Z\\.-]+$");
+
+ private String parseVersionParameter(String parameter) {
+ if (!versionParameterPattern.matcher(parameter).matches()) {
+ /* Version contains illegal character(s). */
+ return null;
+ }
+ return parameter;
+ }
}
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
index cc50f4b..9fcff35 100644
--- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -283,6 +283,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
nodeStatus.setRecommendedVersion((recommendedVersions == null
|| entry.getVersion() == null) ? null :
recommendedVersions.contains(entry.getVersion()));
+ nodeStatus.setVersion(entry.getVersion());
}
if (entry.getUnmeasured()) {
if (!this.lastSeenUnmeasured.containsKey(fingerprint)
@@ -484,6 +485,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber());
updatedNodeStatus.setRecommendedVersion(
nodeStatus.getRecommendedVersion());
+ updatedNodeStatus.setVersion(nodeStatus.getVersion());
}
if (nodeStatus.getFirstSeenMillis()
< updatedNodeStatus.getFirstSeenMillis()
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
index b2f678d..4364aad 100644
--- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -91,10 +91,11 @@ public class SummaryDocumentWriter implements DocumentWriter {
SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily();
SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
String nickname = nodeStatus.getNickname();
+ String version = nodeStatus.getVersion();
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, declaredFamily, effectiveFamily);
+ asNumber, contact, declaredFamily, effectiveFamily, version);
if (this.documentStore.store(summaryDocument, fingerprint)) {
this.writtenDocuments++;
}
diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
index 413ca83..9d7625a 100644
--- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
@@ -26,7 +26,7 @@ public class SummaryDocumentTest {
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
- new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
+ new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
}
@Test()
diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
index 04c5d7d..4ebf0d2 100644
--- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
+++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
@@ -142,7 +142,8 @@ public class ResourceServletTest {
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
- new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
+ new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
+ "0.2.3.25");
org.torproject.onionoo.docs.SummaryDocument relayFerrari458 =
new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458",
"001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList(
@@ -154,7 +155,7 @@ public class ResourceServletTest {
new TreeSet<String>(Arrays.asList(new String[] {
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
new TreeSet<>(Arrays.asList(new String[] {
- "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })));
+ "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), "0.2.3.24-rc-dev");
this.relays = new TreeMap<>();
this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A",
relayTorkaZ);
@@ -170,7 +171,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
"1024D/51E2A1C7 steven j. murdoch "
+ "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
- new TreeSet<String>(), new TreeSet<String>());
+ new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25");
this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
relayTimMayTribute);
this.bridges = new TreeMap<>();
@@ -181,7 +182,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-21 18:07:03"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
- null, null);
+ null, null, null);
this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
bridgeec2bridgercc7f31fe);
org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -191,7 +192,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-20 17:37:04"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
- null, null);
+ null, null, null);
this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
bridgeUnnamed);
org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -202,7 +203,7 @@ public class ResourceServletTest {
new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), -1L, null,
DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
- null);
+ null, null);
this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
bridgegummy);
}
@@ -279,7 +280,8 @@ public class ResourceServletTest {
int expectedRelaysNumber, String[] expectedRelaysNicknames,
int expectedBridgesNumber, String[] expectedBridgesNicknames) {
this.runTest(request);
- assertNotNull(this.summaryDocument);
+ assertNotNull("Summary document is null, status code is "
+ + this.response.errorStatusCode, this.summaryDocument);
assertEquals(expectedRelaysNumber,
this.summaryDocument.relays.length);
if (expectedRelaysNicknames != null) {
@@ -1531,5 +1533,61 @@ public class ResourceServletTest {
this.assertErrorStatusCode(
"/summary?family=00000000000000000000000000000000000000", 400);
}
+
+ @Test
+ public void testVersion02325() {
+ this.assertSummaryDocument("/summary?version=0.2.3.25", 2,
+ new String[] { "TorkaZ", "TimMayTribute" }, 0, null);
+ }
+
+ @Test
+ public void testVersion02324() {
+ this.assertSummaryDocument("/summary?version=0.2.3.24-rc-dev", 1,
+ new String[] { "Ferrari458" }, 0, null);
+ }
+
+ @Test
+ public void testVersion12345() {
+ this.assertSummaryDocument("/summary?version=1.2.3.4.5", 0, null, 0, null);
+ }
+
+ @Test
+ public void testVersionBlaBlaBla() {
+ this.assertSummaryDocument("/summary?version=bla-bla-bla", 0, null, 0,
+ null);
+ }
+
+ @Test
+ public void testVersion0() {
+ this.assertSummaryDocument("/summary?version=0", 3, null, 0, null);
+ }
+
+ @Test
+ public void testVersion02() {
+ this.assertSummaryDocument("/summary?version=0.2", 3, null, 0, null);
+ }
+
+ @Test
+ public void testVersion023() {
+ this.assertSummaryDocument("/summary?version=0.2.3", 3, null, 0, null);
+ }
+
+ @Test
+ public void testVersion0232() {
+ /* This is correct when comparing strings. */
+ this.assertSummaryDocument("/summary?version=0.2.3.2", 3, null, 0, null);
+ }
+
+ @Test
+ public void testVersion023Dot() {
+ /* This is also correct when comparing strings. */
+ this.assertSummaryDocument("/summary?version=0.2.3.", 3, null, 0, null);
+ }
+
+ @Test
+ public void testVersionStart() {
+ /* This is also correct when comparing strings. */
+ this.assertErrorStatusCode("/summary?version=*", 400);
+ }
}
diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
index c1d909f..fb3f5b3 100644
--- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
+++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
@@ -40,7 +40,7 @@ public class SummaryDocumentComparatorTest {
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
- new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
+ new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
}
/** Some values for running all comparison types. */
