commit 249c8549c0536e7c8e71b5f7f500f81b2f203e6d Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Aug 7 23:06:36 2017 +0200 Add version parameter to filter for Tor version. Implements #21427. Requires minor protocol update. --- .../org/torproject/onionoo/docs/DocumentStore.java | 3 +- .../org/torproject/onionoo/docs/NodeStatus.java | 14 +++++ .../torproject/onionoo/docs/SummaryDocument.java | 13 +++- .../org/torproject/onionoo/server/NodeIndex.java | 10 +++ .../org/torproject/onionoo/server/NodeIndexer.java | 8 +++ .../torproject/onionoo/server/RequestHandler.java | 26 ++++++++ .../torproject/onionoo/server/ResourceServlet.java | 24 +++++++- .../onionoo/updater/NodeDetailsStatusUpdater.java | 2 + .../onionoo/writer/SummaryDocumentWriter.java | 3 +- .../onionoo/docs/SummaryDocumentTest.java | 2 +- .../onionoo/server/ResourceServletTest.java | 72 +++++++++++++++++++--- .../server/SummaryDocumentComparatorTest.java | 2 +- 12 files changed, 166 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 56ca406..7819d0f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -435,13 +435,14 @@ public class DocumentStore { } SortedSet<String> relayFlags = new TreeSet<>(); SortedSet<String> family = null; + String version = null; long lastSeenMillis = -1L; long consensusWeight = -1L; long firstSeenMillis = -1L; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family); + asNumber, contact, family, family, version); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 6a71fb6..759d83e 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -321,6 +321,16 @@ public class NodeStatus extends Document { return this.recommendedVersion; } + private String version; + + public void setVersion(String version) { + this.version = version.substring(version.lastIndexOf(" ") + 1); + } + + public String getVersion() { + return this.version; + } + /* From exit lists: */ private SortedSet<String> exitAddresses; @@ -547,6 +557,9 @@ public class NodeStatus extends Document { extendedFamily.addAll(indirectFamily); nodeStatus.setExtendedFamily(extendedFamily); } + if (parts.length >= 24 && !parts[23].isEmpty()) { + nodeStatus.setVersion(parts[23]); + } return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " @@ -610,6 +623,7 @@ public class NodeStatus extends Document { sb.append("\t" + StringUtils.join(this.getAllegedFamily(), ";") + ":" + StringUtils.join(this.getEffectiveFamily(), ";") + ":" + StringUtils.join(this.getIndirectFamily(), ";")); + sb.append("\t" + (this.getVersion() != null ? this.getVersion() : "")); return sb.toString(); } } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 3ca4960..527f91d 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -269,6 +269,16 @@ public class SummaryDocument extends Document { return this.stringArrayToSortedSet(this.ef); } + private String v; + + public void setVersion(String version) { + this.v = version; + } + + public String getVersion() { + return this.v; + } + /* The familyFingerprints parameter can go away after September 8, 2015. * See above. */ /** Instantiates a summary document with all given properties. */ @@ -277,7 +287,7 @@ public class SummaryDocument extends Document { boolean running, SortedSet<String> relayFlags, long consensusWeight, String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, - SortedSet<String> effectiveFamily) { + SortedSet<String> effectiveFamily, String version) { this.setRelay(isRelay); this.setNickname(nickname); this.setFingerprint(fingerprint); @@ -292,6 +302,7 @@ public class SummaryDocument extends Document { this.setContact(contact); this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); + this.setVersion(version); } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index 439d302..dd4bd4d 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -169,5 +169,15 @@ class NodeIndex { public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() { return bridgesByLastSeenDays; } + + private Map<String, Set<String>> relaysByVersion; + + public void setRelaysByVersion(Map<String, Set<String>> relaysByVersion) { + this.relaysByVersion = relaysByVersion; + } + + public Map<String, Set<String>> getRelaysByVersion() { + return this.relaysByVersion; + } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index f6b9510..7ed9ea6 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -150,6 +150,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { Map<String, Set<String>> newBridgesByFlag = new HashMap<>(); Map<String, Set<String>> newRelaysByContact = new HashMap<>(); Map<String, Set<String>> newRelaysByFamily = new HashMap<>(); + Map<String, Set<String>> newRelaysByVersion = new HashMap<>(); SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>(); @@ -246,6 +247,12 @@ public class NodeIndexer implements ServletContextListener, Runnable { } newRelaysByContact.get(contact).add(fingerprint); newRelaysByContact.get(contact).add(hashedFingerprint); + String version = entry.getVersion(); + if (!newRelaysByVersion.containsKey(version)) { + newRelaysByVersion.put(version, new HashSet<String>()); + } + newRelaysByVersion.get(version).add(fingerprint); + newRelaysByVersion.get(version).add(hashedFingerprint); } /* This loop can go away once all Onionoo services had their hourly * updater write effective families to summary documents at least @@ -319,6 +326,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays); newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis); newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis); + newNodeIndex.setRelaysByVersion(newRelaysByVersion); synchronized (this) { this.lastIndexed = updateStatusMillis; this.latestNodeIndex = newNodeIndex; diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 98ece58..1e08c24 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -7,6 +7,9 @@ import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.SummaryDocument; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -91,6 +94,12 @@ public class RequestHandler { System.arraycopy(contact, 0, this.contact, 0, contact.length); } + private String version; + + public void setVersion(String version) { + this.version = version; + } + private String[] order; public void setOrder(String[] order) { @@ -158,6 +167,7 @@ public class RequestHandler { this.filterNodesByLastSeenDays(); this.filterByContact(); this.filterByFamily(); + this.filterByVersion(); this.order(); this.offset(); this.limit(); @@ -514,6 +524,22 @@ public class RequestHandler { this.filteredBridges.clear(); } + private void filterByVersion() { + if (null == this.version) { + /* Not filtering by version. */ + return; + } + Set<String> keepRelays = new HashSet<>(); + for (Map.Entry<String, Set<String>> e + : this.nodeIndex.getRelaysByVersion().entrySet()) { + if (e.getKey().startsWith(this.version)) { + keepRelays.addAll(e.getValue()); + } + } + this.filteredRelays.keySet().retainAll(keepRelays); + this.filteredBridges.clear(); + } + private void order() { List<SummaryDocument> uniqueRelays = new ArrayList<>(); List<SummaryDocument> uniqueBridges = new ArrayList<>(); diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 869574c..164b770 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -3,6 +3,8 @@ package org.torproject.onionoo.server; +import java.io.BufferedWriter; +import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; @@ -66,7 +68,7 @@ public class ResourceServlet extends HttpServlet { private static Set<String> knownParameters = new HashSet<>( Arrays.asList(("type,running,search,lookup,fingerprint,country,as," + "flag,first_seen_days,last_seen_days,contact,order,limit," - + "offset,fields,family").split(","))); + + "offset,fields,family,version").split(","))); private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -275,6 +277,15 @@ public class ResourceServlet extends HttpServlet { } rh.setContact(contactParts); } + if (parameterMap.containsKey("version")) { + String versionParameter = this.parseVersionParameter( + parameterMap.get("version")); + if (null == versionParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setVersion(versionParameter); + } if (parameterMap.containsKey("order")) { String[] order = this.parseOrderParameter(parameterMap.get("order")); if (order == null) { @@ -532,5 +543,16 @@ public class ResourceServlet extends HttpServlet { } return parameter.toLowerCase().split(","); } + + private static Pattern versionParameterPattern = + Pattern.compile("^[0-9a-zA-Z\\.-]+$"); + + private String parseVersionParameter(String parameter) { + if (!versionParameterPattern.matcher(parameter).matches()) { + /* Version contains illegal character(s). */ + return null; + } + return parameter; + } } diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index cc50f4b..9fcff35 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -283,6 +283,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.setRecommendedVersion((recommendedVersions == null || entry.getVersion() == null) ? null : recommendedVersions.contains(entry.getVersion())); + nodeStatus.setVersion(entry.getVersion()); } if (entry.getUnmeasured()) { if (!this.lastSeenUnmeasured.containsKey(fingerprint) @@ -484,6 +485,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber()); updatedNodeStatus.setRecommendedVersion( nodeStatus.getRecommendedVersion()); + updatedNodeStatus.setVersion(nodeStatus.getVersion()); } if (nodeStatus.getFirstSeenMillis() < updatedNodeStatus.getFirstSeenMillis() diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index b2f678d..4364aad 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -91,10 +91,11 @@ public class SummaryDocumentWriter implements DocumentWriter { SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily(); SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); String nickname = nodeStatus.getNickname(); + String version = nodeStatus.getVersion(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, declaredFamily, effectiveFamily); + asNumber, contact, declaredFamily, effectiveFamily, version); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 413ca83..9d7625a 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -26,7 +26,7 @@ public class SummaryDocumentTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" }))); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); } @Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index 04c5d7d..4ebf0d2 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -142,7 +142,8 @@ public class ResourceServletTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" }))); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), + "0.2.3.25"); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458", "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( @@ -154,7 +155,7 @@ public class ResourceServletTest { new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" }))); + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), "0.2.3.24-rc-dev"); this.relays = new TreeMap<>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); @@ -170,7 +171,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024D/51E2A1C7 steven j. murdoch " + "<tor+steven.murdoch@cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", - new TreeSet<String>(), new TreeSet<String>()); + new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25"); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -181,7 +182,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null); + null, null, null); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -191,7 +192,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null); + null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -202,7 +203,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null); + null, null); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } @@ -279,7 +280,8 @@ public class ResourceServletTest { int expectedRelaysNumber, String[] expectedRelaysNicknames, int expectedBridgesNumber, String[] expectedBridgesNicknames) { this.runTest(request); - assertNotNull(this.summaryDocument); + assertNotNull("Summary document is null, status code is " + + this.response.errorStatusCode, this.summaryDocument); assertEquals(expectedRelaysNumber, this.summaryDocument.relays.length); if (expectedRelaysNicknames != null) { @@ -1531,5 +1533,61 @@ public class ResourceServletTest { this.assertErrorStatusCode( "/summary?family=00000000000000000000000000000000000000", 400); } + + @Test + public void testVersion02325() { + this.assertSummaryDocument("/summary?version=0.2.3.25", 2, + new String[] { "TorkaZ", "TimMayTribute" }, 0, null); + } + + @Test + public void testVersion02324() { + this.assertSummaryDocument("/summary?version=0.2.3.24-rc-dev", 1, + new String[] { "Ferrari458" }, 0, null); + } + + @Test + public void testVersion12345() { + this.assertSummaryDocument("/summary?version=1.2.3.4.5", 0, null, 0, null); + } + + @Test + public void testVersionBlaBlaBla() { + this.assertSummaryDocument("/summary?version=bla-bla-bla", 0, null, 0, + null); + } + + @Test + public void testVersion0() { + this.assertSummaryDocument("/summary?version=0", 3, null, 0, null); + } + + @Test + public void testVersion02() { + this.assertSummaryDocument("/summary?version=0.2", 3, null, 0, null); + } + + @Test + public void testVersion023() { + this.assertSummaryDocument("/summary?version=0.2.3", 3, null, 0, null); + } + + @Test + public void testVersion0232() { + /* This is correct when comparing strings. */ + this.assertSummaryDocument("/summary?version=0.2.3.2", 3, null, 0, null); + } + + @Test + public void testVersion023Dot() { + /* This is also correct when comparing strings. */ + this.assertSummaryDocument("/summary?version=0.2.3.", 3, null, 0, null); + } + + @Test + public void testVersionStart() { + /* This is also correct when comparing strings. */ + this.assertErrorStatusCode("/summary?version=*", 400); + } } diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index c1d909f..fb3f5b3 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -40,7 +40,7 @@ public class SummaryDocumentComparatorTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" }))); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); } /** Some values for running all comparison types. */