commit a7d03e231499e37d82a47230886693a5d3127d20
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Tue Jul 24 14:46:50 2018 +0200
Add new "as_name" parameter.
Implements another part of #23713.
---
CHANGELOG.md | 1 +
.../org/torproject/onionoo/docs/DocumentStore.java | 3 +-
.../org/torproject/onionoo/docs/NodeStatus.java | 14 +++++++++
.../torproject/onionoo/docs/SummaryDocument.java | 14 ++++++++-
.../org/torproject/onionoo/server/NodeIndex.java | 10 +++++++
.../org/torproject/onionoo/server/NodeIndexer.java | 8 +++++
.../torproject/onionoo/server/RequestHandler.java | 29 ++++++++++++++++++
.../torproject/onionoo/server/ResourceServlet.java | 25 ++++++++++++++--
.../onionoo/updater/NodeDetailsStatusUpdater.java | 4 +++
.../onionoo/writer/SummaryDocumentWriter.java | 3 +-
.../onionoo/docs/SummaryDocumentTest.java | 1 +
.../onionoo/server/ResourceServletTest.java | 35 +++++++++++++++++++---
.../server/SummaryDocumentComparatorTest.java | 1 +
13 files changed, 138 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d0d091..89b36eb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@
bridges.
- Add "as" field as a copy of the "as_number" field in preparation
of removing the "as_number" field in the future.
+ - Add new "as_name" parameter to search relays by AS name.
# Changes in version 6.1-1.15.0 - 2018-07-16
diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
index b044788..8cf76c8 100644
--- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
@@ -415,6 +415,7 @@ public class DocumentStore {
List<String> addresses = new ArrayList<>();
String countryCode = null;
String asNumber = null;
+ String asName = null;
String contact = null;
for (String orAddressAndPort : detailsDocument.getOrAddresses()) {
if (!orAddressAndPort.contains(":")) {
@@ -451,7 +452,7 @@ public class DocumentStore {
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, family, family, version, operatingSystem,
+ asNumber, asName, contact, family, family, version, operatingSystem,
hostName, verifiedHostNames, unverifiedHostNames,
recommendedVersion);
return summaryDocument;
diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
index 46da822..63720d9 100644
--- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
@@ -387,6 +387,16 @@ public class NodeStatus extends Document {
return this.asNumber;
}
+ private String asName;
+
+ public void setAsName(String asName) {
+ this.asName = asName;
+ }
+
+ public String getAsName() {
+ return this.asName;
+ }
+
/* Reverse DNS lookup result */
private String hostName;
@@ -618,6 +628,9 @@ public class NodeStatus extends Document {
if (parts.length >= 26) {
nodeStatus.setVersionStatus(TorVersionStatus.ofAbbreviation(parts[25]));
}
+ if (parts.length >= 27) {
+ nodeStatus.setAsName(parts[26]);
+ }
return nodeStatus;
} catch (NumberFormatException e) {
log.error("Number format exception while parsing node "
@@ -688,6 +701,7 @@ public class NodeStatus extends Document {
.append((this.getHostName() != null ? this.getHostName() : ""));
sb.append("\t").append(null != this.getVersionStatus()
? this.getVersionStatus().getAbbreviation() : "");
+ sb.append("\t").append((this.asName != null ? this.asName : ""));
return sb.toString();
}
}
diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
index 92ebe69..fc84ba6 100644
--- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
@@ -190,6 +190,17 @@ public class SummaryDocument extends Document {
return this.asNumber;
}
+ @JsonProperty("an")
+ private String asName;
+
+ public void setAsName(String asName) {
+ this.asName = asName;
+ }
+
+ public String getAsName() {
+ return this.asName;
+ }
+
@JsonProperty("fs")
private String firstSeenMillis;
@@ -364,7 +375,7 @@ public class SummaryDocument extends Document {
public SummaryDocument(boolean isRelay, String nickname,
String fingerprint, List<String> addresses, long lastSeenMillis,
boolean running, SortedSet<String> relayFlags, long consensusWeight,
- String countryCode, long firstSeenMillis, String asNumber,
+ String countryCode, long firstSeenMillis, String asNumber, String asName,
String contact, SortedSet<String> familyFingerprints,
SortedSet<String> effectiveFamily, String version, String operatingSystem,
String hostName, List<String> verifiedHostNames,
@@ -380,6 +391,7 @@ public class SummaryDocument extends Document {
this.setCountryCode(countryCode);
this.setFirstSeenMillis(firstSeenMillis);
this.setAsNumber(asNumber);
+ this.setAsName(asName);
this.setContact(contact);
this.setFamilyFingerprints(familyFingerprints);
this.setEffectiveFamily(effectiveFamily);
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
index ed9ec44..d8b983a 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
@@ -85,6 +85,16 @@ class NodeIndex {
return relaysByAsNumber;
}
+ private Map<String, Set<String>> relaysByAsName = null;
+
+ public void setRelaysByAsName(Map<String, Set<String>> relaysByAsName) {
+ this.relaysByAsName = relaysByAsName;
+ }
+
+ public Map<String, Set<String>> getRelaysByAsName() {
+ return relaysByAsName;
+ }
+
private Map<String, Set<String>> relaysByFlag = null;
public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) {
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index 1f1d279..5a2ea8d 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -151,6 +151,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
new HashMap<>();
Map<String, Set<String>> newRelaysByCountryCode = new HashMap<>();
Map<String, Set<String>> newRelaysByAsNumber = new HashMap<>();
+ Map<String, Set<String>> newRelaysByAsName = new HashMap<>();
Map<String, Set<String>> newRelaysByFlag = new HashMap<>();
Map<String, Set<String>> newBridgesByFlag = new HashMap<>();
Map<String, Set<String>> newRelaysByContact = new HashMap<>();
@@ -230,6 +231,12 @@ public class NodeIndexer implements ServletContextListener, Runnable {
}
newRelaysByAsNumber.get(asNumber).add(fingerprint);
newRelaysByAsNumber.get(asNumber).add(hashedFingerprint);
+ String asName = entry.getAsName();
+ if (!newRelaysByAsName.containsKey(asName)) {
+ newRelaysByAsName.put(asName, new HashSet<>());
+ }
+ newRelaysByAsName.get(asName).add(fingerprint);
+ newRelaysByAsName.get(asName).add(hashedFingerprint);
for (String flag : entry.getRelayFlags()) {
String flagLowerCase = flag.toLowerCase();
if (!newRelaysByFlag.containsKey(flagLowerCase)) {
@@ -402,6 +409,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newBridgeFingerprintSummaryLines);
newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode);
newNodeIndex.setRelaysByAsNumber(newRelaysByAsNumber);
+ newNodeIndex.setRelaysByAsName(newRelaysByAsName);
newNodeIndex.setRelaysByFlag(newRelaysByFlag);
newNodeIndex.setBridgesByFlag(newBridgesByFlag);
newNodeIndex.setRelaysByContact(newRelaysByContact);
diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
index 344178b..e401ae9 100644
--- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java
+++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
@@ -78,6 +78,12 @@ public class RequestHandler {
this.as = as;
}
+ private String[] asName;
+
+ public void setAsName(String[] asName) {
+ this.asName = asName;
+ }
+
private String flag;
public void setFlag(String flag) {
@@ -177,6 +183,7 @@ public class RequestHandler {
this.filterByFingerprint();
this.filterByCountryCode();
this.filterByAsNumber();
+ this.filterByAsName();
this.filterByFlag();
this.filterNodesByFirstSeenDays();
this.filterNodesByLastSeenDays();
@@ -427,6 +434,28 @@ public class RequestHandler {
this.filteredBridges.clear();
}
+ private void filterByAsName() {
+ if (this.asName == null) {
+ /* Not filtering by AS name. */
+ return;
+ }
+ Set<String> removeRelays = new HashSet<>();
+ for (Map.Entry<String, Set<String>> e :
+ this.nodeIndex.getRelaysByAsName().entrySet()) {
+ String asName = e.getKey();
+ for (String asNamePart : this.asName) {
+ if (asName == null || !asName.contains(asNamePart.toLowerCase())) {
+ removeRelays.addAll(e.getValue());
+ break;
+ }
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ this.filteredBridges.clear();
+ }
+
private void filterByFlag() {
if (this.flag == null) {
/* Not filtering by relay flag. */
diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
index 534dc0c..cf480d3 100644
--- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
+++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
@@ -67,9 +67,9 @@ public class ResourceServlet extends HttpServlet {
private static Set<String> knownParameters = new HashSet<>(
Arrays.asList("type", "running", "search", "lookup", "fingerprint",
- "country", "as", "flag", "first_seen_days", "last_seen_days",
- "contact", "order", "limit", "offset", "fields", "family", "version",
- "os", "host_name", "recommended_version"));
+ "country", "as", "as_name", "flag", "first_seen_days",
+ "last_seen_days", "contact", "order", "limit", "offset", "fields",
+ "family", "version", "os", "host_name", "recommended_version"));
private static Set<String> illegalSearchQualifiers =
new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -246,6 +246,15 @@ public class ResourceServlet extends HttpServlet {
}
rh.setAs(asNumberParameter);
}
+ if (parameterMap.containsKey("as_name")) {
+ String[] asNameParameter = this.parseAsNameParameter(
+ parameterMap.get("as_name"));
+ if (null == asNameParameter) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setAsName(asNameParameter);
+ }
if (parameterMap.containsKey("flag")) {
String flagParameter = this.parseFlagParameter(
parameterMap.get("flag"));
@@ -503,6 +512,16 @@ public class ResourceServlet extends HttpServlet {
return parameter;
}
+ private String[] parseAsNameParameter(String parameter) {
+ for (char c : parameter.toCharArray()) {
+ if (c < 32 || c >= 127) {
+ /* Only accept printable ASCII. */
+ return null;
+ }
+ }
+ return parameter.toLowerCase().split(" ");
+ }
+
private static Pattern flagPattern =
Pattern.compile("^[a-zA-Z0-9]{1,20}$");
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
index 6d22aa2..baba85f 100644
--- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -492,6 +492,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
nodeStatus.getDefaultPolicy());
updatedNodeStatus.setPortList(nodeStatus.getPortList());
updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber());
+ updatedNodeStatus.setAsName(nodeStatus.getAsName());
updatedNodeStatus.setRecommendedVersion(
nodeStatus.getRecommendedVersion());
updatedNodeStatus.setVersion(nodeStatus.getVersion());
@@ -879,6 +880,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
detailsStatus.setAsName(lookupResult.getAsName());
nodeStatus.setCountryCode(lookupResult.getCountryCode());
nodeStatus.setAsNumber(lookupResult.getAsNumber());
+ if (null != lookupResult.getAsName()) {
+ nodeStatus.setAsName(lookupResult.getAsName().toLowerCase());
+ }
}
if (this.consensusWeightFractions.containsKey(fingerprint)) {
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
index dc6eba8..2bcc102 100644
--- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -87,6 +87,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
String countryCode = nodeStatus.getCountryCode();
long firstSeenMillis = nodeStatus.getFirstSeenMillis();
String asNumber = nodeStatus.getAsNumber();
+ String asName = nodeStatus.getAsName();
String contact = nodeStatus.getContact();
SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily();
SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
@@ -100,7 +101,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
- asNumber, contact, declaredFamily, effectiveFamily, version,
+ asNumber, asName, contact, declaredFamily, effectiveFamily, version,
operatingSystem, hostName, verifiedHostNames,
unverifiedHostNames, recommendedVersion);
if (this.documentStore.store(summaryDocument, fingerprint)) {
diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
index 405fff6..f81dc73 100644
--- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
@@ -21,6 +21,7 @@ public class SummaryDocumentTest {
false, new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), 20L, "de",
DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
+ "m-net telekommunikations gmbh",
"torkaz <klaus dot zufall at gmx dot de> "
+ "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
index 8100c00..37f7923 100644
--- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
+++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
@@ -140,6 +140,7 @@ public class ResourceServletTest {
false, new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), 20L, "de",
DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
+ "m-net telekommunikations gmbh",
"torkaz <klaus dot zufall at gmx dot de> "
+ "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
@@ -159,7 +160,8 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-24 12:00:00"), true,
new TreeSet<>(Arrays.asList(new String[] { "Fast", "Named",
"Running", "V2Dir", "Valid" })), 1140L, "us",
- DateTimeHelper.parse("2013-04-16 18:00:00"), "AS7922", null,
+ DateTimeHelper.parse("2013-04-16 18:00:00"), "AS7922",
+ "comcast cable communications, llc", null,
new TreeSet<String>(Arrays.asList(new String[] {
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
new TreeSet<>(Arrays.asList(new String[] {
@@ -178,6 +180,7 @@ public class ResourceServletTest {
new TreeSet<>(Arrays.asList(new String[] { "Fast",
"Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1",
DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
+ "liberty global operations b.v.",
"1024d/51e2a1c7 \"steven j. murdoch\" "
+ "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev",
@@ -191,7 +194,7 @@ public class ResourceServletTest {
Arrays.asList(new String[] { "10.199.7.176" }),
DateTimeHelper.parse("2013-04-21 18:07:03"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
- null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
+ null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, null,
null, null, "0.2.2.39", null, null, null, null, true);
this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
bridgeec2bridgercc7f31fe);
@@ -201,7 +204,7 @@ public class ResourceServletTest {
new String[] { "10.0.52.84" }),
DateTimeHelper.parse("2013-04-20 17:37:04"), false,
new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
- null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
+ null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, null,
null, null, null, null, null, null, null, null);
this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
bridgeUnnamed);
@@ -212,7 +215,7 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-04-24 01:07:04"), true,
new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), -1L, null,
- DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
+ DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, null,
null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false);
this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
bridgegummy);
@@ -1171,6 +1174,30 @@ public class ResourceServletTest {
}
@Test(timeout = 100)
+ public void testAsNameComcast() {
+ this.assertSummaryDocument("/summary?as_name=Comcast", 1, null, 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testAsNameComcastCable() {
+ this.assertSummaryDocument("/summary?as_name=Comcast Cable",
+ 1, null, 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testAsNameCableComcast() {
+ this.assertSummaryDocument("/summary?as_name=Cable Comcast",
+ 1, null, 0, null);
+ }
+
+ @Test(timeout = 100)
+ public void testAsNameMit() {
+ this.assertSummaryDocument(
+ "/summary?as_name=Massachusetts Institute of Technology",
+ 0, null, 0, null);
+ }
+
+ @Test(timeout = 100)
public void testFlagRunning() {
this.assertSummaryDocument(
"/summary?flag=Running", 3, null, 1, null);
diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
index 2b67206..c79565a 100644
--- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
+++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
@@ -35,6 +35,7 @@ public class SummaryDocumentComparatorTest {
false, new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), 20L, "de",
DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
+ "m-net telekommunikations gmbh",
"torkaz <klaus dot zufall at gmx dot de> "
+ "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
