[tor-commits] [onionoo/master] Allows filtering by operating system

commit e95b88f0c344546df2e23262e13e3714040c36ad Author: Iain R. Learmonth irl@fsfe.org Date: Wed Jul 11 13:08:25 2018 +0100

Allows filtering by operating system

The filter is applicable to both bridges and relays. It uses basic string operations to extract the operating system from the platform line of descriptors.

Fixes: #6946 --- CHANGELOG.md | 1 + .../org/torproject/onionoo/docs/DocumentStore.java | 5 ++- .../org/torproject/onionoo/docs/NodeStatus.java | 10 +++++ .../torproject/onionoo/docs/SummaryDocument.java | 18 ++++++-- .../org/torproject/onionoo/server/NodeIndex.java | 22 +++++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 22 +++++++++ .../torproject/onionoo/server/RequestHandler.java | 30 +++++++++++++ .../torproject/onionoo/server/ResourceServlet.java | 21 ++++++++- .../onionoo/updater/NodeDetailsStatusUpdater.java | 6 +++ .../onionoo/writer/SummaryDocumentWriter.java | 5 ++- .../onionoo/docs/SummaryDocumentTest.java | 2 +- .../onionoo/server/ResourceServletTest.java | 52 +++++++++++++++++++--- .../server/SummaryDocumentComparatorTest.java | 2 +- 13 files changed, 179 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index d4cfb82..a9f21b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Medium changes - Provide more accurate DNS results in "verified_host_names" and "unverified_host_names". + - Allow filtering by operating system using the new "os" parameter.

* Minor changes - Index relays with no known country code or autonomous system diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 0d75bf9..e474684 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -448,6 +448,7 @@ public class DocumentStore { SortedSet<String> relayFlags = new TreeSet<>(); SortedSet<String> family = null; String version = null; + String operatingSystem = null; long lastSeenMillis = -1L; long consensusWeight = -1L; long firstSeenMillis = -1L; @@ -458,8 +459,8 @@ public class DocumentStore { SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family, version, hostName, - verifiedHostNames, unverifiedHostNames, + asNumber, contact, family, family, version, operatingSystem, + hostName, verifiedHostNames, unverifiedHostNames, recommendedVersion); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 64332da..46da822 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -343,6 +343,16 @@ public class NodeStatus extends Document { return this.versionStatus; }

+ private String operatingSystem; + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + + public String getOperatingSystem() { + return this.operatingSystem; + } + /* From exit lists: */

private SortedSet<String> exitAddresses; diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 8b5a5e1..92ebe69 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -298,6 +298,17 @@ public class SummaryDocument extends Document { return this.version; }

+ @JsonProperty("o") + private String operatingSystem; + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + + public String getOperatingSystem() { + return this.operatingSystem; + } + @JsonProperty("h") private String hostName;

@@ -355,9 +366,9 @@ public class SummaryDocument extends Document { boolean running, SortedSet<String> relayFlags, long consensusWeight, String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, - SortedSet<String> effectiveFamily, String version, String hostName, - List<String> verifiedHostNames, List<String> unverifiedHostNames, - Boolean recommendedVersion) { + SortedSet<String> effectiveFamily, String version, String operatingSystem, + String hostName, List<String> verifiedHostNames, + List<String> unverifiedHostNames, Boolean recommendedVersion) { this.setRelay(isRelay); this.setNickname(nickname); this.setFingerprint(fingerprint); @@ -373,6 +384,7 @@ public class SummaryDocument extends Document { this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); this.setVersion(version); + this.setOperatingSystem(operatingSystem); this.setHostName(hostName); this.setVerifiedHostNames(verifiedHostNames); this.setUnverifiedHostNames(unverifiedHostNames); diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index 77993a3..ed9ec44 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -190,6 +190,28 @@ class NodeIndex { return this.bridgesByVersion; }

+ private Map<String, Set<String>> relaysByOperatingSystem; + + public void setRelaysByOperatingSystem( + Map<String, Set<String>> relaysByOperatingSystem) { + this.relaysByOperatingSystem = relaysByOperatingSystem; + } + + public Map<String, Set<String>> getRelaysByOperatingSystem() { + return this.relaysByOperatingSystem; + } + + private Map<String, Set<String>> bridgesByOperatingSystem; + + public void setBridgesByOperatingSystem( + Map<String, Set<String>> bridgesByOperatingSystem) { + this.bridgesByOperatingSystem = bridgesByOperatingSystem; + } + + public Map<String, Set<String>> getBridgesByOperatingSystem() { + return this.bridgesByOperatingSystem; + } + private Map<String, Set<String>> relaysByHostName;

public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) { diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index f6b84b8..1f1d279 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -157,6 +157,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { Map<String, Set<String>> newRelaysByFamily = new HashMap<>(); Map<String, Set<String>> newRelaysByVersion = new HashMap<>(); Map<String, Set<String>> newBridgesByVersion = new HashMap<>(); + Map<String, Set<String>> newRelaysByOperatingSystem = new HashMap<>(); + Map<String, Set<String>> newBridgesByOperatingSystem = new HashMap<>(); Map<String, Set<String>> newRelaysByHostName = new HashMap<>(); Map<Boolean, Set<String>> newRelaysByRecommendedVersion = new HashMap<>(); newRelaysByRecommendedVersion.put(true, new HashSet<>()); @@ -281,6 +283,14 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } + String operatingSystem = entry.getOperatingSystem(); + if (null != operatingSystem) { + if (!newRelaysByOperatingSystem.containsKey(operatingSystem)) { + newRelaysByOperatingSystem.put(operatingSystem, new HashSet<>()); + } + newRelaysByOperatingSystem.get(operatingSystem).add(fingerprint); + newRelaysByOperatingSystem.get(operatingSystem).add(hashedFingerprint); + } List<String> allHostNames = new ArrayList<>(); List<String> verifiedHostNames = entry.getVerifiedHostNames(); if (null != verifiedHostNames) { @@ -367,6 +377,16 @@ public class NodeIndexer implements ServletContextListener, Runnable { newBridgesByVersion.get(version).add(hashedFingerprint); newBridgesByVersion.get(version).add(hashedHashedFingerprint); } + String operatingSystem = entry.getOperatingSystem(); + if (null != operatingSystem) { + if (!newBridgesByOperatingSystem.containsKey(operatingSystem)) { + newBridgesByOperatingSystem.put(operatingSystem, new HashSet<>()); + } + newBridgesByOperatingSystem.get(operatingSystem) + .add(hashedFingerprint); + newBridgesByOperatingSystem.get(operatingSystem) + .add(hashedHashedFingerprint); + } Boolean recommendedVersion = entry.getRecommendedVersion(); if (null != recommendedVersion) { newBridgesByRecommendedVersion.get(recommendedVersion).add( @@ -394,6 +414,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis); newNodeIndex.setRelaysByVersion(newRelaysByVersion); newNodeIndex.setBridgesByVersion(newBridgesByVersion); + newNodeIndex.setRelaysByOperatingSystem(newRelaysByOperatingSystem); + newNodeIndex.setBridgesByOperatingSystem(newBridgesByOperatingSystem); newNodeIndex.setRelaysByHostName(newRelaysByHostName); newNodeIndex.setRelaysByRecommendedVersion(newRelaysByRecommendedVersion); newNodeIndex.setBridgesByRecommendedVersion(newBridgesByRecommendedVersion); diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 63b8fe9..b9d9b23 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -97,6 +97,12 @@ public class RequestHandler { this.version = version; }

+ private String operatingSystem; + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + private String hostName;

public void setHostName(String hostName) { @@ -177,6 +183,7 @@ public class RequestHandler { this.filterByContact(); this.filterByFamily(); this.filterByVersion(); + this.filterByOperatingSystem(); this.filterByHostName(); this.filterByRecommendedVersion(); this.order(); @@ -558,6 +565,29 @@ public class RequestHandler { this.filteredBridges.keySet().retainAll(keepBridges); }

+ private void filterByOperatingSystem() { + if (null == this.operatingSystem) { + /* Not filtering by operating system. */ + return; + } + Set<String> keepRelays = new HashSet<>(); + for (Map.Entry<String, Set<String>> e + : this.nodeIndex.getRelaysByOperatingSystem().entrySet()) { + if (e.getKey().startsWith(this.operatingSystem)) { + keepRelays.addAll(e.getValue()); + } + } + this.filteredRelays.keySet().retainAll(keepRelays); + Set<String> keepBridges = new HashSet<>(); + for (Map.Entry<String, Set<String>> e + : this.nodeIndex.getBridgesByOperatingSystem().entrySet()) { + if (e.getKey().startsWith(this.operatingSystem)) { + keepBridges.addAll(e.getValue()); + } + } + this.filteredBridges.keySet().retainAll(keepBridges); + } + private void filterByHostName() { if (this.hostName == null) { /* Not filtering by host name. */ diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index a64b89c..e119518 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -69,7 +69,7 @@ public class ResourceServlet extends HttpServlet { Arrays.asList("type", "running", "search", "lookup", "fingerprint", "country", "as", "flag", "first_seen_days", "last_seen_days", "contact", "order", "limit", "offset", "fields", "family", "version", - "host_name", "recommended_version")); + "os", "host_name", "recommended_version"));

private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -293,6 +293,15 @@ public class ResourceServlet extends HttpServlet { } rh.setVersion(versionParameter); } + if (parameterMap.containsKey("os")) { + String osParameter = this.parseOperatingSystemParameter( + parameterMap.get("os")); + if (null == osParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setOperatingSystem(osParameter); + } if (parameterMap.containsKey("host_name")) { String hostNameParameter = this.parseHostNameParameter( parameterMap.get("host_name")); @@ -607,6 +616,16 @@ public class ResourceServlet extends HttpServlet { return parameter; }

+ private String parseOperatingSystemParameter(String parameter) { + for (char c : parameter.toCharArray()) { + if (c < 32 || c >= 127) { + /* Only accept printable ASCII. */ + return null; + } + } + return parameter.toLowerCase(); + } + private static Pattern hostNameParameterPattern = Pattern.compile("^[0-9A-Za-z_\.\-]+$");

diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index ad2cac7..6d22aa2 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -807,6 +807,12 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, }

nodeStatus.setContact(detailsStatus.getContact()); + if (null != detailsStatus.getPlatform()) { + String[] platformParts = detailsStatus.getPlatform().split(" on "); + if (platformParts.length > 1) { + nodeStatus.setOperatingSystem(platformParts[1].toLowerCase()); + } + }

/* Extract tor software version for bridges from their "platform" line. * (We already know this for relays from "v" lines in the consensus.) */ diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index ed36d56..dc6eba8 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -92,6 +92,7 @@ public class SummaryDocumentWriter implements DocumentWriter { SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); + String operatingSystem = nodeStatus.getOperatingSystem(); String hostName = nodeStatus.getHostName(); List<String> verifiedHostNames = nodeStatus.getVerifiedHostNames(); List<String> unverifiedHostNames = nodeStatus.getUnverifiedHostNames(); @@ -100,8 +101,8 @@ public class SummaryDocumentWriter implements DocumentWriter { nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, declaredFamily, effectiveFamily, version, - hostName, verifiedHostNames, unverifiedHostNames, - recommendedVersion); + operatingSystem, hostName, verifiedHostNames, + unverifiedHostNames, recommendedVersion); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 459d514..405fff6 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -27,7 +27,7 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null, null, true); + null, null, null, null, true); }

@Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index 4e9a5f0..145619d 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -153,7 +153,7 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", + "0.2.3.25", "linux", "ppp-62-216-201-221.dynamic.mnet-online.de", Arrays.asList( new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }), null, true); @@ -170,7 +170,7 @@ public class ResourceServletTest { new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, null, "c-68-38-171-200.hsd1.in.comcast.net", Arrays.asList( new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}), @@ -187,8 +187,8 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024d/51e2a1c7 "steven j. murdoch" " + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm=", - new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null, - null, null, false); + new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", + "windows xp", null, null, null, false); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -199,7 +199,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, "0.2.2.39", null, null, null, true); + null, null, "0.2.2.39", null, null, null, null, true); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -209,7 +209,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -220,7 +220,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, "0.2.4.4-alpha-dev", null, null, null, false); + null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } @@ -1648,6 +1648,44 @@ public class ResourceServletTest { this.assertErrorStatusCode("/summary?version=*", 400); }

+ @Test(timeout = 100) + public void testOperatingSystemLinux() { + this.assertSummaryDocument( + "/summary?os=linux", 1, new String[] {"TorkaZ"}, 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemLinuxMixedCaps() { + this.assertSummaryDocument( + "/summary?os=LiNUx", 1, new String[] {"TorkaZ"}, 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemLin() { + this.assertSummaryDocument( + "/summary?os=lin", 1, new String[] {"TorkaZ"}, 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemWindows() { + this.assertSummaryDocument( + "/summary?os=windows", 1, new String[] {"TimMayTribute"}, + 1, new String[] {"gummy"}); + } + + @Test(timeout = 100) + public void testOperatingSystemWindowsExperience() { + this.assertSummaryDocument( + "/summary?os=windows xp", 1, new String[] {"TimMayTribute"}, + 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemWindows7() { + this.assertSummaryDocument( + "/summary?os=windows 7", 0, null, 1, new String[] {"gummy"}); + } + @Test public void testHostNameDe() { this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null); diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index 569300d..2b67206 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null, null, null); + null, null, null, null, null); }

/** Some values for running all comparison types. */