[tor-commits] [onionoo/release] Allows filtering by operating system
karsten at torproject.org
karsten at torproject.org
Mon Jul 16 14:15:23 UTC 2018
commit e95b88f0c344546df2e23262e13e3714040c36ad
Author: Iain R. Learmonth <irl at fsfe.org>
Date: Wed Jul 11 13:08:25 2018 +0100
Allows filtering by operating system
The filter is applicable to both bridges and relays. It uses basic
string operations to extract the operating system from the platform line
of descriptors.
Fixes: #6946
---
CHANGELOG.md | 1 +
.../org/torproject/onionoo/docs/DocumentStore.java | 5 ++-
.../org/torproject/onionoo/docs/NodeStatus.java | 10 +++++
.../torproject/onionoo/docs/SummaryDocument.java | 18 ++++++--
.../org/torproject/onionoo/server/NodeIndex.java | 22 +++++++++
.../org/torproject/onionoo/server/NodeIndexer.java | 22 +++++++++
.../torproject/onionoo/server/RequestHandler.java | 30 +++++++++++++
.../torproject/onionoo/server/ResourceServlet.java | 21 ++++++++-
.../onionoo/updater/NodeDetailsStatusUpdater.java | 6 +++
.../onionoo/writer/SummaryDocumentWriter.java | 5 ++-
.../onionoo/docs/SummaryDocumentTest.java | 2 +-
.../onionoo/server/ResourceServletTest.java | 52 +++++++++++++++++++---
.../server/SummaryDocumentComparatorTest.java | 2 +-
13 files changed, 179 insertions(+), 17 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d4cfb82..a9f21b6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
* Medium changes
- Provide more accurate DNS results in "verified_host_names" and
"unverified_host_names".
+ - Allow filtering by operating system using the new "os" parameter.
* Minor changes
- Index relays with no known country code or autonomous system
diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
index 0d75bf9..e474684 100644
--- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
@@ -448,6 +448,7 @@ public class DocumentStore {
SortedSet<String> relayFlags = new TreeSet<>();
SortedSet<String> family = null;
String version = null;
+ String operatingSystem = null;
long lastSeenMillis = -1L;
long consensusWeight = -1L;
long firstSeenMillis = -1L;
@@ -458,8 +459,8 @@ public class DocumentStore {
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, family, family, version, hostName,
- verifiedHostNames, unverifiedHostNames,
+ asNumber, contact, family, family, version, operatingSystem,
+ hostName, verifiedHostNames, unverifiedHostNames,
recommendedVersion);
return summaryDocument;
}
diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
index 64332da..46da822 100644
--- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
@@ -343,6 +343,16 @@ public class NodeStatus extends Document {
return this.versionStatus;
}
+ private String operatingSystem;
+
+ public void setOperatingSystem(String operatingSystem) {
+ this.operatingSystem = operatingSystem;
+ }
+
+ public String getOperatingSystem() {
+ return this.operatingSystem;
+ }
+
/* From exit lists: */
private SortedSet<String> exitAddresses;
diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
index 8b5a5e1..92ebe69 100644
--- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
@@ -298,6 +298,17 @@ public class SummaryDocument extends Document {
return this.version;
}
+ @JsonProperty("o")
+ private String operatingSystem;
+
+ public void setOperatingSystem(String operatingSystem) {
+ this.operatingSystem = operatingSystem;
+ }
+
+ public String getOperatingSystem() {
+ return this.operatingSystem;
+ }
+
@JsonProperty("h")
private String hostName;
@@ -355,9 +366,9 @@ public class SummaryDocument extends Document {
boolean running, SortedSet<String> relayFlags, long consensusWeight,
String countryCode, long firstSeenMillis, String asNumber,
String contact, SortedSet<String> familyFingerprints,
- SortedSet<String> effectiveFamily, String version, String hostName,
- List<String> verifiedHostNames, List<String> unverifiedHostNames,
- Boolean recommendedVersion) {
+ SortedSet<String> effectiveFamily, String version, String operatingSystem,
+ String hostName, List<String> verifiedHostNames,
+ List<String> unverifiedHostNames, Boolean recommendedVersion) {
this.setRelay(isRelay);
this.setNickname(nickname);
this.setFingerprint(fingerprint);
@@ -373,6 +384,7 @@ public class SummaryDocument extends Document {
this.setFamilyFingerprints(familyFingerprints);
this.setEffectiveFamily(effectiveFamily);
this.setVersion(version);
+ this.setOperatingSystem(operatingSystem);
this.setHostName(hostName);
this.setVerifiedHostNames(verifiedHostNames);
this.setUnverifiedHostNames(unverifiedHostNames);
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
index 77993a3..ed9ec44 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
@@ -190,6 +190,28 @@ class NodeIndex {
return this.bridgesByVersion;
}
+ private Map<String, Set<String>> relaysByOperatingSystem;
+
+ public void setRelaysByOperatingSystem(
+ Map<String, Set<String>> relaysByOperatingSystem) {
+ this.relaysByOperatingSystem = relaysByOperatingSystem;
+ }
+
+ public Map<String, Set<String>> getRelaysByOperatingSystem() {
+ return this.relaysByOperatingSystem;
+ }
+
+ private Map<String, Set<String>> bridgesByOperatingSystem;
+
+ public void setBridgesByOperatingSystem(
+ Map<String, Set<String>> bridgesByOperatingSystem) {
+ this.bridgesByOperatingSystem = bridgesByOperatingSystem;
+ }
+
+ public Map<String, Set<String>> getBridgesByOperatingSystem() {
+ return this.bridgesByOperatingSystem;
+ }
+
private Map<String, Set<String>> relaysByHostName;
public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) {
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index f6b84b8..1f1d279 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -157,6 +157,8 @@ public class NodeIndexer implements ServletContextListener, Runnable {
Map<String, Set<String>> newRelaysByFamily = new HashMap<>();
Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
Map<String, Set<String>> newBridgesByVersion = new HashMap<>();
+ Map<String, Set<String>> newRelaysByOperatingSystem = new HashMap<>();
+ Map<String, Set<String>> newBridgesByOperatingSystem = new HashMap<>();
Map<String, Set<String>> newRelaysByHostName = new HashMap<>();
Map<Boolean, Set<String>> newRelaysByRecommendedVersion = new HashMap<>();
newRelaysByRecommendedVersion.put(true, new HashSet<>());
@@ -281,6 +283,14 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newRelaysByVersion.get(version).add(fingerprint);
newRelaysByVersion.get(version).add(hashedFingerprint);
}
+ String operatingSystem = entry.getOperatingSystem();
+ if (null != operatingSystem) {
+ if (!newRelaysByOperatingSystem.containsKey(operatingSystem)) {
+ newRelaysByOperatingSystem.put(operatingSystem, new HashSet<>());
+ }
+ newRelaysByOperatingSystem.get(operatingSystem).add(fingerprint);
+ newRelaysByOperatingSystem.get(operatingSystem).add(hashedFingerprint);
+ }
List<String> allHostNames = new ArrayList<>();
List<String> verifiedHostNames = entry.getVerifiedHostNames();
if (null != verifiedHostNames) {
@@ -367,6 +377,16 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newBridgesByVersion.get(version).add(hashedFingerprint);
newBridgesByVersion.get(version).add(hashedHashedFingerprint);
}
+ String operatingSystem = entry.getOperatingSystem();
+ if (null != operatingSystem) {
+ if (!newBridgesByOperatingSystem.containsKey(operatingSystem)) {
+ newBridgesByOperatingSystem.put(operatingSystem, new HashSet<>());
+ }
+ newBridgesByOperatingSystem.get(operatingSystem)
+ .add(hashedFingerprint);
+ newBridgesByOperatingSystem.get(operatingSystem)
+ .add(hashedHashedFingerprint);
+ }
Boolean recommendedVersion = entry.getRecommendedVersion();
if (null != recommendedVersion) {
newBridgesByRecommendedVersion.get(recommendedVersion).add(
@@ -394,6 +414,8 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis);
newNodeIndex.setRelaysByVersion(newRelaysByVersion);
newNodeIndex.setBridgesByVersion(newBridgesByVersion);
+ newNodeIndex.setRelaysByOperatingSystem(newRelaysByOperatingSystem);
+ newNodeIndex.setBridgesByOperatingSystem(newBridgesByOperatingSystem);
newNodeIndex.setRelaysByHostName(newRelaysByHostName);
newNodeIndex.setRelaysByRecommendedVersion(newRelaysByRecommendedVersion);
newNodeIndex.setBridgesByRecommendedVersion(newBridgesByRecommendedVersion);
diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
index 63b8fe9..b9d9b23 100644
--- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java
+++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
@@ -97,6 +97,12 @@ public class RequestHandler {
this.version = version;
}
+ private String operatingSystem;
+
+ public void setOperatingSystem(String operatingSystem) {
+ this.operatingSystem = operatingSystem;
+ }
+
private String hostName;
public void setHostName(String hostName) {
@@ -177,6 +183,7 @@ public class RequestHandler {
this.filterByContact();
this.filterByFamily();
this.filterByVersion();
+ this.filterByOperatingSystem();
this.filterByHostName();
this.filterByRecommendedVersion();
this.order();
@@ -558,6 +565,29 @@ public class RequestHandler {
this.filteredBridges.keySet().retainAll(keepBridges);
}
+ private void filterByOperatingSystem() {
+ if (null == this.operatingSystem) {
+ /* Not filtering by operating system. */
+ return;
+ }
+ Set<String> keepRelays = new HashSet<>();
+ for (Map.Entry<String, Set<String>> e
+ : this.nodeIndex.getRelaysByOperatingSystem().entrySet()) {
+ if (e.getKey().startsWith(this.operatingSystem)) {
+ keepRelays.addAll(e.getValue());
+ }
+ }
+ this.filteredRelays.keySet().retainAll(keepRelays);
+ Set<String> keepBridges = new HashSet<>();
+ for (Map.Entry<String, Set<String>> e
+ : this.nodeIndex.getBridgesByOperatingSystem().entrySet()) {
+ if (e.getKey().startsWith(this.operatingSystem)) {
+ keepBridges.addAll(e.getValue());
+ }
+ }
+ this.filteredBridges.keySet().retainAll(keepBridges);
+ }
+
private void filterByHostName() {
if (this.hostName == null) {
/* Not filtering by host name. */
diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
index a64b89c..e119518 100644
--- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
+++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
@@ -69,7 +69,7 @@ public class ResourceServlet extends HttpServlet {
Arrays.asList("type", "running", "search", "lookup", "fingerprint",
"country", "as", "flag", "first_seen_days", "last_seen_days",
"contact", "order", "limit", "offset", "fields", "family", "version",
- "host_name", "recommended_version"));
+ "os", "host_name", "recommended_version"));
private static Set<String> illegalSearchQualifiers =
new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -293,6 +293,15 @@ public class ResourceServlet extends HttpServlet {
}
rh.setVersion(versionParameter);
}
+ if (parameterMap.containsKey("os")) {
+ String osParameter = this.parseOperatingSystemParameter(
+ parameterMap.get("os"));
+ if (null == osParameter) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setOperatingSystem(osParameter);
+ }
if (parameterMap.containsKey("host_name")) {
String hostNameParameter = this.parseHostNameParameter(
parameterMap.get("host_name"));
@@ -607,6 +616,16 @@ public class ResourceServlet extends HttpServlet {
return parameter;
}
+ private String parseOperatingSystemParameter(String parameter) {
+ for (char c : parameter.toCharArray()) {
+ if (c < 32 || c >= 127) {
+ /* Only accept printable ASCII. */
+ return null;
+ }
+ }
+ return parameter.toLowerCase();
+ }
+
private static Pattern hostNameParameterPattern =
Pattern.compile("^[0-9A-Za-z_\\.\\-]+$");
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
index ad2cac7..6d22aa2 100644
--- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -807,6 +807,12 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
}
nodeStatus.setContact(detailsStatus.getContact());
+ if (null != detailsStatus.getPlatform()) {
+ String[] platformParts = detailsStatus.getPlatform().split(" on ");
+ if (platformParts.length > 1) {
+ nodeStatus.setOperatingSystem(platformParts[1].toLowerCase());
+ }
+ }
/* Extract tor software version for bridges from their "platform" line.
* (We already know this for relays from "v" lines in the consensus.) */
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
index ed36d56..dc6eba8 100644
--- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -92,6 +92,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
String nickname = nodeStatus.getNickname();
String version = nodeStatus.getVersion();
+ String operatingSystem = nodeStatus.getOperatingSystem();
String hostName = nodeStatus.getHostName();
List<String> verifiedHostNames = nodeStatus.getVerifiedHostNames();
List<String> unverifiedHostNames = nodeStatus.getUnverifiedHostNames();
@@ -100,8 +101,8 @@ public class SummaryDocumentWriter implements DocumentWriter {
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
asNumber, contact, declaredFamily, effectiveFamily, version,
- hostName, verifiedHostNames, unverifiedHostNames,
- recommendedVersion);
+ operatingSystem, hostName, verifiedHostNames,
+ unverifiedHostNames, recommendedVersion);
if (this.documentStore.store(summaryDocument, fingerprint)) {
this.writtenDocuments++;
}
diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
index 459d514..405fff6 100644
--- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
@@ -27,7 +27,7 @@ public class SummaryDocumentTest {
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
- null, null, null, true);
+ null, null, null, null, true);
}
@Test()
diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
index 4e9a5f0..145619d 100644
--- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
+++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
@@ -153,7 +153,7 @@ public class ResourceServletTest {
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
- "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de",
+ "0.2.3.25", "linux", "ppp-62-216-201-221.dynamic.mnet-online.de",
Arrays.asList(
new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }),
null, true);
@@ -170,7 +170,7 @@ public class ResourceServletTest {
new TreeSet<String>(Arrays.asList(new String[] {
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
new TreeSet<>(Arrays.asList(new String[] {
- "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null,
+ "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, null,
"c-68-38-171-200.hsd1.in.comcast.net",
Arrays.asList(
new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}),
@@ -187,8 +187,8 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
"1024d/51e2a1c7 \"steven j. murdoch\" "
+ "<tor+steven.murdoch at cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
- new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null,
- null, null, false);
+ new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev",
+ "windows xp", null, null, null, false);
this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
relayTimMayTribute);
this.bridges = new TreeMap<>();
@@ -199,7 +199,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-21 18:07:03"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
- null, null, "0.2.2.39", null, null, null, true);
+ null, null, "0.2.2.39", null, null, null, null, true);
this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
bridgeec2bridgercc7f31fe);
org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -209,7 +209,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-20 17:37:04"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
- null, null, null, null, null, null, null);
+ null, null, null, null, null, null, null, null);
this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
bridgeUnnamed);
org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -220,7 +220,7 @@ public class ResourceServletTest {
new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), -1L, null,
DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
- null, "0.2.4.4-alpha-dev", null, null, null, false);
+ null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false);
this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
bridgegummy);
}
@@ -1648,6 +1648,44 @@ public class ResourceServletTest {
this.assertErrorStatusCode("/summary?version=*", 400);
}
+ @Test(timeout = 100)
+ public void testOperatingSystemLinux() {
+ this.assertSummaryDocument(
+ "/summary?os=linux", 1, new String[] {"TorkaZ"}, 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testOperatingSystemLinuxMixedCaps() {
+ this.assertSummaryDocument(
+ "/summary?os=LiNUx", 1, new String[] {"TorkaZ"}, 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testOperatingSystemLin() {
+ this.assertSummaryDocument(
+ "/summary?os=lin", 1, new String[] {"TorkaZ"}, 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testOperatingSystemWindows() {
+ this.assertSummaryDocument(
+ "/summary?os=windows", 1, new String[] {"TimMayTribute"},
+ 1, new String[] {"gummy"});
+ }
+
+ @Test(timeout = 100)
+ public void testOperatingSystemWindowsExperience() {
+ this.assertSummaryDocument(
+ "/summary?os=windows xp", 1, new String[] {"TimMayTribute"},
+ 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testOperatingSystemWindows7() {
+ this.assertSummaryDocument(
+ "/summary?os=windows 7", 0, null, 1, new String[] {"gummy"});
+ }
+
@Test
public void testHostNameDe() {
this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null);
diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
index 569300d..2b67206 100644
--- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
+++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
@@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest {
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
- null, null, null, null);
+ null, null, null, null, null);
}
/** Some values for running all comparison types. */
More information about the tor-commits
mailing list