commit d5d0edc9aab3d301b228c2b177daf172d93827b5 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Oct 17 14:48:26 2017 +0200
Add new "host_name" parameter to filter by host name.
Implements #16553. --- CHANGELOG.md | 1 + .../org/torproject/onionoo/docs/DocumentStore.java | 3 +- .../org/torproject/onionoo/docs/NodeStatus.java | 15 ++++ .../torproject/onionoo/docs/SummaryDocument.java | 13 +++- .../org/torproject/onionoo/server/NodeIndex.java | 10 +++ .../org/torproject/onionoo/server/NodeIndexer.java | 11 +++ .../torproject/onionoo/server/RequestHandler.java | 26 +++++++ .../torproject/onionoo/server/ResourceServlet.java | 22 +++++- .../onionoo/updater/NodeDetailsStatusUpdater.java | 1 + .../onionoo/writer/SummaryDocumentWriter.java | 4 +- .../onionoo/docs/SummaryDocumentTest.java | 3 +- .../onionoo/server/ResourceServletTest.java | 79 ++++++++++++++++++++-- .../server/SummaryDocumentComparatorTest.java | 3 +- 13 files changed, 179 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b0741..76243ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Support quoted qualified search terms. - Skip unrecognized descriptors when importing archives rather than aborting the entire import. + - Add new "host_name" parameter to filter by host name.
# Changes in version 4.2-1.6.1 - 2017-10-26 diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 7819d0f..57d4165 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -439,10 +439,11 @@ public class DocumentStore { long lastSeenMillis = -1L; long consensusWeight = -1L; long firstSeenMillis = -1L; + String hostName = null; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family, version); + asNumber, contact, family, family, version, hostName); return summaryDocument; }
diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 24546dd..3271ccc 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -368,6 +368,16 @@ public class NodeStatus extends Document {
/* Reverse DNS lookup result */
+ private String hostName; + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getHostName() { + return this.hostName; + } + private long lastRdnsLookup = -1L;
public void setLastRdnsLookup(long lastRdnsLookup) { @@ -561,6 +571,9 @@ public class NodeStatus extends Document { if (parts.length >= 24 && !parts[23].isEmpty()) { nodeStatus.setVersion(parts[23]); } + if (parts.length >= 25 && !parts[24].isEmpty()) { + nodeStatus.setHostName(parts[24]); + } return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " @@ -627,6 +640,8 @@ public class NodeStatus extends Document { .append(StringUtils.join(this.getIndirectFamily(), ";")); sb.append("\t") .append((this.getVersion() != null ? this.getVersion() : "")); + sb.append("\t") + .append((this.getHostName() != null ? this.getHostName() : "")); return sb.toString(); } } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 527f91d..f33100d 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -279,6 +279,16 @@ public class SummaryDocument extends Document { return this.v; }
+ private String h; + + public void setHostName(String hostName) { + this.h = hostName; + } + + public String getHostName() { + return this.h; + } + /* The familyFingerprints parameter can go away after September 8, 2015. * See above. */ /** Instantiates a summary document with all given properties. */ @@ -287,7 +297,7 @@ public class SummaryDocument extends Document { boolean running, SortedSet<String> relayFlags, long consensusWeight, String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, - SortedSet<String> effectiveFamily, String version) { + SortedSet<String> effectiveFamily, String version, String hostName) { this.setRelay(isRelay); this.setNickname(nickname); this.setFingerprint(fingerprint); @@ -303,6 +313,7 @@ public class SummaryDocument extends Document { this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); this.setVersion(version); + this.setHostName(hostName); } }
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index dd4bd4d..126cd5c 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -179,5 +179,15 @@ class NodeIndex { public Map<String, Set<String>> getRelaysByVersion() { return this.relaysByVersion; } + + private Map<String, Set<String>> relaysByHostName; + + public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) { + this.relaysByHostName = relaysByHostName; + } + + public Map<String, Set<String>> getRelaysByHostName() { + return this.relaysByHostName; + } }
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 5411d0e..d609f63 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -154,6 +154,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { Map<String, Set<String>> newRelaysByContact = new HashMap<>(); Map<String, Set<String>> newRelaysByFamily = new HashMap<>(); Map<String, Set<String>> newRelaysByVersion = new HashMap<>(); + Map<String, Set<String>> newRelaysByHostName = new HashMap<>(); SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>(); @@ -258,6 +259,15 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } + String hostName = entry.getHostName(); + if (null != hostName) { + String hostNameLowerCase = hostName.toLowerCase(); + if (!newRelaysByHostName.containsKey(hostNameLowerCase)) { + newRelaysByHostName.put(hostNameLowerCase, new HashSet<>()); + } + newRelaysByHostName.get(hostNameLowerCase).add(fingerprint); + newRelaysByHostName.get(hostNameLowerCase).add(hashedFingerprint); + } } /* This loop can go away once all Onionoo services had their hourly * updater write effective families to summary documents at least @@ -332,6 +342,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis); newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis); newNodeIndex.setRelaysByVersion(newRelaysByVersion); + newNodeIndex.setRelaysByHostName(newRelaysByHostName); synchronized (this) { this.lastIndexed = updateStatusMillis; this.latestNodeIndex = newNodeIndex; diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 46c2f54..23af60b 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -97,6 +97,12 @@ public class RequestHandler { this.version = version; }
+ private String hostName; + + public void setHostName(String hostName) { + this.hostName = hostName; + } + private String[] order;
public void setOrder(String[] order) { @@ -165,6 +171,7 @@ public class RequestHandler { this.filterByContact(); this.filterByFamily(); this.filterByVersion(); + this.filterByHostName(); this.order(); this.offset(); this.limit(); @@ -537,6 +544,25 @@ public class RequestHandler { this.filteredBridges.clear(); }
+ private void filterByHostName() { + if (this.hostName == null) { + /* Not filtering by host name. */ + return; + } + String hostName = this.hostName.toLowerCase(); + Set<String> removeRelays = new HashSet<>(this.filteredRelays.keySet()); + for (Map.Entry<String, Set<String>> e : + this.nodeIndex.getRelaysByHostName().entrySet()) { + if (e.getKey().endsWith(hostName)) { + removeRelays.removeAll(e.getValue()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + private void order() { List<SummaryDocument> uniqueRelays = new ArrayList<>(); List<SummaryDocument> uniqueBridges = new ArrayList<>(); diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 2fff913..f1521e2 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -68,7 +68,7 @@ public class ResourceServlet extends HttpServlet { private static Set<String> knownParameters = new HashSet<>( Arrays.asList(("type,running,search,lookup,fingerprint,country,as," + "flag,first_seen_days,last_seen_days,contact,order,limit," - + "offset,fields,family,version").split(","))); + + "offset,fields,family,version,host_name").split(",")));
private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -292,6 +292,15 @@ public class ResourceServlet extends HttpServlet { } rh.setVersion(versionParameter); } + if (parameterMap.containsKey("host_name")) { + String hostNameParameter = this.parseHostNameParameter( + parameterMap.get("host_name")); + if (null == hostNameParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setHostName(hostNameParameter); + } if (parameterMap.containsKey("order")) { String[] order = this.parseOrderParameter(parameterMap.get("order")); if (order == null) { @@ -584,5 +593,16 @@ public class ResourceServlet extends HttpServlet { } return parameter; } + + private static Pattern hostNameParameterPattern = + Pattern.compile("^[0-9A-Za-z_\.\-]+$"); + + private String parseHostNameParameter(String parameter) { + if (!hostNameParameterPattern.matcher(parameter).matches()) { + /* Host name contains illegal character(s). */ + return null; + } + return parameter; + } }
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 04a04fa..73e826a 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -849,6 +849,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (this.rdnsLookupResults.containsKey(fingerprint)) { String hostName = this.rdnsLookupResults.get(fingerprint); detailsStatus.setHostName(hostName); + nodeStatus.setHostName(hostName); nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); }
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 4364aad..faf54fa 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -92,10 +92,12 @@ public class SummaryDocumentWriter implements DocumentWriter { SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); + String hostName = nodeStatus.getHostName(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, declaredFamily, effectiveFamily, version); + asNumber, contact, declaredFamily, effectiveFamily, version, + hostName); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 9d7625a..8c45ed2 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -26,7 +26,8 @@ public class SummaryDocumentTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, + null); }
@Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index e41eecc..2720f7a 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -143,7 +143,7 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25"); + "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de"); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458", "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( @@ -155,7 +155,8 @@ public class ResourceServletTest { new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null); + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, + "c-68-38-171-200.hsd1.in.comcast.net"); this.relays = new TreeMap<>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); @@ -171,7 +172,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024d/51e2a1c7 "steven j. murdoch" " + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm=", - new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25"); + new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25", null); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -182,7 +183,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, null); + null, null, null, null); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -192,7 +193,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null); + null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -203,7 +204,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, null); + null, null, null); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } @@ -1623,5 +1624,71 @@ public class ResourceServletTest { /* This is also correct when comparing strings. */ this.assertErrorStatusCode("/summary?version=*", 400); } + + @Test + public void testHostNameDe() { + this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null); + } + + @Test + public void testHostNameE() { + this.assertSummaryDocument("/summary?host_name=e", 1, null, 0, null); + } + + @Test + public void testHostNameDotDe() { + this.assertSummaryDocument("/summary?host_name=.de", 1, null, 0, null); + } + + @Test + public void testHostNameOnlineDe() { + this.assertSummaryDocument("/summary?host_name=online.de", 1, null, 0, + null); + } + + @Test + public void testHostNameOnlineDeSomeCapitalized() { + this.assertSummaryDocument("/summary?host_name=onLiNe.dE", 1, null, 0, + null); + } + + @Test + public void testHostNameOnline() { + this.assertSummaryDocument("/summary?host_name=online", 0, null, 0, null); + } + + @Test + public void testHostNameTorkaZFull() { + this.assertSummaryDocument( + "/summary?host_name=ppp-62-216-201-221.dynamic.mnet-online.de", + 1, null, 0, null); + } + + @Test + public void testHostNameTorkaZSub() { + this.assertSummaryDocument( + "/summary?host_name=sub.ppp-62-216-201-221.dynamic.mnet-online.de", + 0, null, 0, null); + } + + @Test + public void testHostNameNet() { + this.assertSummaryDocument("/summary?host_name=net", 1, null, 0, null); + } + + @Test + public void testHostNameCom() { + this.assertSummaryDocument("/summary?host_name=com", 0, null, 0, null); + } + + @Test + public void testHostNameDollar() { + this.assertErrorStatusCode("/summary?host_name=$", 400); + } + + @Test + public void testHostNameUmlaut() { + this.assertErrorStatusCode("/summary?host_name=äöü", 400); + } }
diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index fb3f5b3..38d94fa 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -40,7 +40,8 @@ public class SummaryDocumentComparatorTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, + null); }
/** Some values for running all comparison types. */