[tor-commits] [onionoo/release] Add new "host_name" parameter to filter by host name.
karsten at torproject.org
karsten at torproject.org
Fri Nov 17 13:48:58 UTC 2017
commit d5d0edc9aab3d301b228c2b177daf172d93827b5
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Oct 17 14:48:26 2017 +0200
Add new "host_name" parameter to filter by host name.
Implements #16553.
---
CHANGELOG.md | 1 +
.../org/torproject/onionoo/docs/DocumentStore.java | 3 +-
.../org/torproject/onionoo/docs/NodeStatus.java | 15 ++++
.../torproject/onionoo/docs/SummaryDocument.java | 13 +++-
.../org/torproject/onionoo/server/NodeIndex.java | 10 +++
.../org/torproject/onionoo/server/NodeIndexer.java | 11 +++
.../torproject/onionoo/server/RequestHandler.java | 26 +++++++
.../torproject/onionoo/server/ResourceServlet.java | 22 +++++-
.../onionoo/updater/NodeDetailsStatusUpdater.java | 1 +
.../onionoo/writer/SummaryDocumentWriter.java | 4 +-
.../onionoo/docs/SummaryDocumentTest.java | 3 +-
.../onionoo/server/ResourceServletTest.java | 79 ++++++++++++++++++++--
.../server/SummaryDocumentComparatorTest.java | 3 +-
13 files changed, 179 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12b0741..76243ef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
- Support quoted qualified search terms.
- Skip unrecognized descriptors when importing archives rather than
aborting the entire import.
+ - Add new "host_name" parameter to filter by host name.
# Changes in version 4.2-1.6.1 - 2017-10-26
diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
index 7819d0f..57d4165 100644
--- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
@@ -439,10 +439,11 @@ public class DocumentStore {
long lastSeenMillis = -1L;
long consensusWeight = -1L;
long firstSeenMillis = -1L;
+ String hostName = null;
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, family, family, version);
+ asNumber, contact, family, family, version, hostName);
return summaryDocument;
}
diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
index 24546dd..3271ccc 100644
--- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
@@ -368,6 +368,16 @@ public class NodeStatus extends Document {
/* Reverse DNS lookup result */
+ private String hostName;
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ public String getHostName() {
+ return this.hostName;
+ }
+
private long lastRdnsLookup = -1L;
public void setLastRdnsLookup(long lastRdnsLookup) {
@@ -561,6 +571,9 @@ public class NodeStatus extends Document {
if (parts.length >= 24 && !parts[23].isEmpty()) {
nodeStatus.setVersion(parts[23]);
}
+ if (parts.length >= 25 && !parts[24].isEmpty()) {
+ nodeStatus.setHostName(parts[24]);
+ }
return nodeStatus;
} catch (NumberFormatException e) {
log.error("Number format exception while parsing node "
@@ -627,6 +640,8 @@ public class NodeStatus extends Document {
.append(StringUtils.join(this.getIndirectFamily(), ";"));
sb.append("\t")
.append((this.getVersion() != null ? this.getVersion() : ""));
+ sb.append("\t")
+ .append((this.getHostName() != null ? this.getHostName() : ""));
return sb.toString();
}
}
diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
index 527f91d..f33100d 100644
--- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
@@ -279,6 +279,16 @@ public class SummaryDocument extends Document {
return this.v;
}
+ private String h;
+
+ public void setHostName(String hostName) {
+ this.h = hostName;
+ }
+
+ public String getHostName() {
+ return this.h;
+ }
+
/* The familyFingerprints parameter can go away after September 8, 2015.
* See above. */
/** Instantiates a summary document with all given properties. */
@@ -287,7 +297,7 @@ public class SummaryDocument extends Document {
boolean running, SortedSet<String> relayFlags, long consensusWeight,
String countryCode, long firstSeenMillis, String asNumber,
String contact, SortedSet<String> familyFingerprints,
- SortedSet<String> effectiveFamily, String version) {
+ SortedSet<String> effectiveFamily, String version, String hostName) {
this.setRelay(isRelay);
this.setNickname(nickname);
this.setFingerprint(fingerprint);
@@ -303,6 +313,7 @@ public class SummaryDocument extends Document {
this.setFamilyFingerprints(familyFingerprints);
this.setEffectiveFamily(effectiveFamily);
this.setVersion(version);
+ this.setHostName(hostName);
}
}
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
index dd4bd4d..126cd5c 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
@@ -179,5 +179,15 @@ class NodeIndex {
public Map<String, Set<String>> getRelaysByVersion() {
return this.relaysByVersion;
}
+
+ private Map<String, Set<String>> relaysByHostName;
+
+ public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) {
+ this.relaysByHostName = relaysByHostName;
+ }
+
+ public Map<String, Set<String>> getRelaysByHostName() {
+ return this.relaysByHostName;
+ }
}
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index 5411d0e..d609f63 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -154,6 +154,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
Map<String, Set<String>> newRelaysByContact = new HashMap<>();
Map<String, Set<String>> newRelaysByFamily = new HashMap<>();
Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
+ Map<String, Set<String>> newRelaysByHostName = new HashMap<>();
SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>();
SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>();
SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>();
@@ -258,6 +259,15 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newRelaysByVersion.get(version).add(fingerprint);
newRelaysByVersion.get(version).add(hashedFingerprint);
}
+ String hostName = entry.getHostName();
+ if (null != hostName) {
+ String hostNameLowerCase = hostName.toLowerCase();
+ if (!newRelaysByHostName.containsKey(hostNameLowerCase)) {
+ newRelaysByHostName.put(hostNameLowerCase, new HashSet<>());
+ }
+ newRelaysByHostName.get(hostNameLowerCase).add(fingerprint);
+ newRelaysByHostName.get(hostNameLowerCase).add(hashedFingerprint);
+ }
}
/* This loop can go away once all Onionoo services had their hourly
* updater write effective families to summary documents at least
@@ -332,6 +342,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis);
newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis);
newNodeIndex.setRelaysByVersion(newRelaysByVersion);
+ newNodeIndex.setRelaysByHostName(newRelaysByHostName);
synchronized (this) {
this.lastIndexed = updateStatusMillis;
this.latestNodeIndex = newNodeIndex;
diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
index 46c2f54..23af60b 100644
--- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java
+++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
@@ -97,6 +97,12 @@ public class RequestHandler {
this.version = version;
}
+ private String hostName;
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
private String[] order;
public void setOrder(String[] order) {
@@ -165,6 +171,7 @@ public class RequestHandler {
this.filterByContact();
this.filterByFamily();
this.filterByVersion();
+ this.filterByHostName();
this.order();
this.offset();
this.limit();
@@ -537,6 +544,25 @@ public class RequestHandler {
this.filteredBridges.clear();
}
+ private void filterByHostName() {
+ if (this.hostName == null) {
+ /* Not filtering by host name. */
+ return;
+ }
+ String hostName = this.hostName.toLowerCase();
+ Set<String> removeRelays = new HashSet<>(this.filteredRelays.keySet());
+ for (Map.Entry<String, Set<String>> e :
+ this.nodeIndex.getRelaysByHostName().entrySet()) {
+ if (e.getKey().endsWith(hostName)) {
+ removeRelays.removeAll(e.getValue());
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ this.filteredBridges.clear();
+ }
+
private void order() {
List<SummaryDocument> uniqueRelays = new ArrayList<>();
List<SummaryDocument> uniqueBridges = new ArrayList<>();
diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
index 2fff913..f1521e2 100644
--- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
+++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
@@ -68,7 +68,7 @@ public class ResourceServlet extends HttpServlet {
private static Set<String> knownParameters = new HashSet<>(
Arrays.asList(("type,running,search,lookup,fingerprint,country,as,"
+ "flag,first_seen_days,last_seen_days,contact,order,limit,"
- + "offset,fields,family,version").split(",")));
+ + "offset,fields,family,version,host_name").split(",")));
private static Set<String> illegalSearchQualifiers =
new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -292,6 +292,15 @@ public class ResourceServlet extends HttpServlet {
}
rh.setVersion(versionParameter);
}
+ if (parameterMap.containsKey("host_name")) {
+ String hostNameParameter = this.parseHostNameParameter(
+ parameterMap.get("host_name"));
+ if (null == hostNameParameter) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setHostName(hostNameParameter);
+ }
if (parameterMap.containsKey("order")) {
String[] order = this.parseOrderParameter(parameterMap.get("order"));
if (order == null) {
@@ -584,5 +593,16 @@ public class ResourceServlet extends HttpServlet {
}
return parameter;
}
+
+ private static Pattern hostNameParameterPattern =
+ Pattern.compile("^[0-9A-Za-z_\\.\\-]+$");
+
+ private String parseHostNameParameter(String parameter) {
+ if (!hostNameParameterPattern.matcher(parameter).matches()) {
+ /* Host name contains illegal character(s). */
+ return null;
+ }
+ return parameter;
+ }
}
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
index 04a04fa..73e826a 100644
--- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -849,6 +849,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
if (this.rdnsLookupResults.containsKey(fingerprint)) {
String hostName = this.rdnsLookupResults.get(fingerprint);
detailsStatus.setHostName(hostName);
+ nodeStatus.setHostName(hostName);
nodeStatus.setLastRdnsLookup(this.startedRdnsLookups);
}
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
index 4364aad..faf54fa 100644
--- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -92,10 +92,12 @@ public class SummaryDocumentWriter implements DocumentWriter {
SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
String nickname = nodeStatus.getNickname();
String version = nodeStatus.getVersion();
+ String hostName = nodeStatus.getHostName();
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, declaredFamily, effectiveFamily, version);
+ asNumber, contact, declaredFamily, effectiveFamily, version,
+ hostName);
if (this.documentStore.store(summaryDocument, fingerprint)) {
this.writtenDocuments++;
}
diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
index 9d7625a..8c45ed2 100644
--- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
@@ -26,7 +26,8 @@ public class SummaryDocumentTest {
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
- new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
+ new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
+ null);
}
@Test()
diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
index e41eecc..2720f7a 100644
--- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
+++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
@@ -143,7 +143,7 @@ public class ResourceServletTest {
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
- "0.2.3.25");
+ "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de");
org.torproject.onionoo.docs.SummaryDocument relayFerrari458 =
new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458",
"001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList(
@@ -155,7 +155,8 @@ public class ResourceServletTest {
new TreeSet<String>(Arrays.asList(new String[] {
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
new TreeSet<>(Arrays.asList(new String[] {
- "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null);
+ "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null,
+ "c-68-38-171-200.hsd1.in.comcast.net");
this.relays = new TreeMap<>();
this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A",
relayTorkaZ);
@@ -171,7 +172,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
"1024d/51e2a1c7 \"steven j. murdoch\" "
+ "<tor+steven.murdoch at cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
- new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25");
+ new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25", null);
this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
relayTimMayTribute);
this.bridges = new TreeMap<>();
@@ -182,7 +183,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-21 18:07:03"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
- null, null, null);
+ null, null, null, null);
this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
bridgeec2bridgercc7f31fe);
org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -192,7 +193,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-20 17:37:04"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
- null, null, null);
+ null, null, null, null);
this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
bridgeUnnamed);
org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -203,7 +204,7 @@ public class ResourceServletTest {
new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), -1L, null,
DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
- null, null);
+ null, null, null);
this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
bridgegummy);
}
@@ -1623,5 +1624,71 @@ public class ResourceServletTest {
/* This is also correct when comparing strings. */
this.assertErrorStatusCode("/summary?version=*", 400);
}
+
+ @Test
+ public void testHostNameDe() {
+ this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameE() {
+ this.assertSummaryDocument("/summary?host_name=e", 1, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameDotDe() {
+ this.assertSummaryDocument("/summary?host_name=.de", 1, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameOnlineDe() {
+ this.assertSummaryDocument("/summary?host_name=online.de", 1, null, 0,
+ null);
+ }
+
+ @Test
+ public void testHostNameOnlineDeSomeCapitalized() {
+ this.assertSummaryDocument("/summary?host_name=onLiNe.dE", 1, null, 0,
+ null);
+ }
+
+ @Test
+ public void testHostNameOnline() {
+ this.assertSummaryDocument("/summary?host_name=online", 0, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameTorkaZFull() {
+ this.assertSummaryDocument(
+ "/summary?host_name=ppp-62-216-201-221.dynamic.mnet-online.de",
+ 1, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameTorkaZSub() {
+ this.assertSummaryDocument(
+ "/summary?host_name=sub.ppp-62-216-201-221.dynamic.mnet-online.de",
+ 0, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameNet() {
+ this.assertSummaryDocument("/summary?host_name=net", 1, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameCom() {
+ this.assertSummaryDocument("/summary?host_name=com", 0, null, 0, null);
+ }
+
+ @Test
+ public void testHostNameDollar() {
+ this.assertErrorStatusCode("/summary?host_name=$", 400);
+ }
+
+ @Test
+ public void testHostNameUmlaut() {
+ this.assertErrorStatusCode("/summary?host_name=äöü", 400);
+ }
}
diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
index fb3f5b3..38d94fa 100644
--- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
+++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
@@ -40,7 +40,8 @@ public class SummaryDocumentComparatorTest {
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
- new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
+ new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
+ null);
}
/** Some values for running all comparison types. */
More information about the tor-commits
mailing list