commit a7d03e231499e37d82a47230886693a5d3127d20 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Jul 24 14:46:50 2018 +0200

Add new "as_name" parameter.

Implements another part of #23713. --- CHANGELOG.md | 1 + .../org/torproject/onionoo/docs/DocumentStore.java | 3 +- .../org/torproject/onionoo/docs/NodeStatus.java | 14 +++++++++ .../torproject/onionoo/docs/SummaryDocument.java | 14 ++++++++- .../org/torproject/onionoo/server/NodeIndex.java | 10 +++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 8 +++++ .../torproject/onionoo/server/RequestHandler.java | 29 ++++++++++++++++++ .../torproject/onionoo/server/ResourceServlet.java | 25 ++++++++++++++-- .../onionoo/updater/NodeDetailsStatusUpdater.java | 4 +++ .../onionoo/writer/SummaryDocumentWriter.java | 3 +- .../onionoo/docs/SummaryDocumentTest.java | 1 + .../onionoo/server/ResourceServletTest.java | 35 +++++++++++++++++++--- .../server/SummaryDocumentComparatorTest.java | 1 + 13 files changed, 138 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d0d091..89b36eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ bridges. - Add "as" field as a copy of the "as_number" field in preparation of removing the "as_number" field in the future. + - Add new "as_name" parameter to search relays by AS name.

# Changes in version 6.1-1.15.0 - 2018-07-16 diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index b044788..8cf76c8 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -415,6 +415,7 @@ public class DocumentStore { List<String> addresses = new ArrayList<>(); String countryCode = null; String asNumber = null; + String asName = null; String contact = null; for (String orAddressAndPort : detailsDocument.getOrAddresses()) { if (!orAddressAndPort.contains(":")) { @@ -451,7 +452,7 @@ public class DocumentStore { SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family, version, operatingSystem, + asNumber, asName, contact, family, family, version, operatingSystem, hostName, verifiedHostNames, unverifiedHostNames, recommendedVersion); return summaryDocument; diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 46da822..63720d9 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -387,6 +387,16 @@ public class NodeStatus extends Document { return this.asNumber; }

+ private String asName; + + public void setAsName(String asName) { + this.asName = asName; + } + + public String getAsName() { + return this.asName; + } + /* Reverse DNS lookup result */

private String hostName; @@ -618,6 +628,9 @@ public class NodeStatus extends Document { if (parts.length >= 26) { nodeStatus.setVersionStatus(TorVersionStatus.ofAbbreviation(parts[25])); } + if (parts.length >= 27) { + nodeStatus.setAsName(parts[26]); + } return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " @@ -688,6 +701,7 @@ public class NodeStatus extends Document { .append((this.getHostName() != null ? this.getHostName() : "")); sb.append("\t").append(null != this.getVersionStatus() ? this.getVersionStatus().getAbbreviation() : ""); + sb.append("\t").append((this.asName != null ? this.asName : "")); return sb.toString(); } } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 92ebe69..fc84ba6 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -190,6 +190,17 @@ public class SummaryDocument extends Document { return this.asNumber; }

+ @JsonProperty("an") + private String asName; + + public void setAsName(String asName) { + this.asName = asName; + } + + public String getAsName() { + return this.asName; + } + @JsonProperty("fs") private String firstSeenMillis;

@@ -364,7 +375,7 @@ public class SummaryDocument extends Document { public SummaryDocument(boolean isRelay, String nickname, String fingerprint, List<String> addresses, long lastSeenMillis, boolean running, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, long firstSeenMillis, String asNumber, + String countryCode, long firstSeenMillis, String asNumber, String asName, String contact, SortedSet<String> familyFingerprints, SortedSet<String> effectiveFamily, String version, String operatingSystem, String hostName, List<String> verifiedHostNames, @@ -380,6 +391,7 @@ public class SummaryDocument extends Document { this.setCountryCode(countryCode); this.setFirstSeenMillis(firstSeenMillis); this.setAsNumber(asNumber); + this.setAsName(asName); this.setContact(contact); this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index ed9ec44..d8b983a 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -85,6 +85,16 @@ class NodeIndex { return relaysByAsNumber; }

+ private Map<String, Set<String>> relaysByAsName = null; + + public void setRelaysByAsName(Map<String, Set<String>> relaysByAsName) { + this.relaysByAsName = relaysByAsName; + } + + public Map<String, Set<String>> getRelaysByAsName() { + return relaysByAsName; + } + private Map<String, Set<String>> relaysByFlag = null;

public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) { diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 1f1d279..5a2ea8d 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -151,6 +151,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { new HashMap<>(); Map<String, Set<String>> newRelaysByCountryCode = new HashMap<>(); Map<String, Set<String>> newRelaysByAsNumber = new HashMap<>(); + Map<String, Set<String>> newRelaysByAsName = new HashMap<>(); Map<String, Set<String>> newRelaysByFlag = new HashMap<>(); Map<String, Set<String>> newBridgesByFlag = new HashMap<>(); Map<String, Set<String>> newRelaysByContact = new HashMap<>(); @@ -230,6 +231,12 @@ public class NodeIndexer implements ServletContextListener, Runnable { } newRelaysByAsNumber.get(asNumber).add(fingerprint); newRelaysByAsNumber.get(asNumber).add(hashedFingerprint); + String asName = entry.getAsName(); + if (!newRelaysByAsName.containsKey(asName)) { + newRelaysByAsName.put(asName, new HashSet<>()); + } + newRelaysByAsName.get(asName).add(fingerprint); + newRelaysByAsName.get(asName).add(hashedFingerprint); for (String flag : entry.getRelayFlags()) { String flagLowerCase = flag.toLowerCase(); if (!newRelaysByFlag.containsKey(flagLowerCase)) { @@ -402,6 +409,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newBridgeFingerprintSummaryLines); newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode); newNodeIndex.setRelaysByAsNumber(newRelaysByAsNumber); + newNodeIndex.setRelaysByAsName(newRelaysByAsName); newNodeIndex.setRelaysByFlag(newRelaysByFlag); newNodeIndex.setBridgesByFlag(newBridgesByFlag); newNodeIndex.setRelaysByContact(newRelaysByContact); diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 344178b..e401ae9 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -78,6 +78,12 @@ public class RequestHandler { this.as = as; }

+ private String[] asName; + + public void setAsName(String[] asName) { + this.asName = asName; + } + private String flag;

public void setFlag(String flag) { @@ -177,6 +183,7 @@ public class RequestHandler { this.filterByFingerprint(); this.filterByCountryCode(); this.filterByAsNumber(); + this.filterByAsName(); this.filterByFlag(); this.filterNodesByFirstSeenDays(); this.filterNodesByLastSeenDays(); @@ -427,6 +434,28 @@ public class RequestHandler { this.filteredBridges.clear(); }

+ private void filterByAsName() { + if (this.asName == null) { + /* Not filtering by AS name. */ + return; + } + Set<String> removeRelays = new HashSet<>(); + for (Map.Entry<String, Set<String>> e : + this.nodeIndex.getRelaysByAsName().entrySet()) { + String asName = e.getKey(); + for (String asNamePart : this.asName) { + if (asName == null || !asName.contains(asNamePart.toLowerCase())) { + removeRelays.addAll(e.getValue()); + break; + } + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + private void filterByFlag() { if (this.flag == null) { /* Not filtering by relay flag. */ diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 534dc0c..cf480d3 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -67,9 +67,9 @@ public class ResourceServlet extends HttpServlet {

private static Set<String> knownParameters = new HashSet<>( Arrays.asList("type", "running", "search", "lookup", "fingerprint", - "country", "as", "flag", "first_seen_days", "last_seen_days", - "contact", "order", "limit", "offset", "fields", "family", "version", - "os", "host_name", "recommended_version")); + "country", "as", "as_name", "flag", "first_seen_days", + "last_seen_days", "contact", "order", "limit", "offset", "fields", + "family", "version", "os", "host_name", "recommended_version"));

private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -246,6 +246,15 @@ public class ResourceServlet extends HttpServlet { } rh.setAs(asNumberParameter); } + if (parameterMap.containsKey("as_name")) { + String[] asNameParameter = this.parseAsNameParameter( + parameterMap.get("as_name")); + if (null == asNameParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setAsName(asNameParameter); + } if (parameterMap.containsKey("flag")) { String flagParameter = this.parseFlagParameter( parameterMap.get("flag")); @@ -503,6 +512,16 @@ public class ResourceServlet extends HttpServlet { return parameter; }

+ private String[] parseAsNameParameter(String parameter) { + for (char c : parameter.toCharArray()) { + if (c < 32 || c >= 127) { + /* Only accept printable ASCII. */ + return null; + } + } + return parameter.toLowerCase().split(" "); + } + private static Pattern flagPattern = Pattern.compile("^[a-zA-Z0-9]{1,20}$");

diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 6d22aa2..baba85f 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -492,6 +492,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.getDefaultPolicy()); updatedNodeStatus.setPortList(nodeStatus.getPortList()); updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber()); + updatedNodeStatus.setAsName(nodeStatus.getAsName()); updatedNodeStatus.setRecommendedVersion( nodeStatus.getRecommendedVersion()); updatedNodeStatus.setVersion(nodeStatus.getVersion()); @@ -879,6 +880,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, detailsStatus.setAsName(lookupResult.getAsName()); nodeStatus.setCountryCode(lookupResult.getCountryCode()); nodeStatus.setAsNumber(lookupResult.getAsNumber()); + if (null != lookupResult.getAsName()) { + nodeStatus.setAsName(lookupResult.getAsName().toLowerCase()); + } }

if (this.consensusWeightFractions.containsKey(fingerprint)) { diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index dc6eba8..2bcc102 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -87,6 +87,7 @@ public class SummaryDocumentWriter implements DocumentWriter { String countryCode = nodeStatus.getCountryCode(); long firstSeenMillis = nodeStatus.getFirstSeenMillis(); String asNumber = nodeStatus.getAsNumber(); + String asName = nodeStatus.getAsName(); String contact = nodeStatus.getContact(); SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily(); SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); @@ -100,7 +101,7 @@ public class SummaryDocumentWriter implements DocumentWriter { SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, declaredFamily, effectiveFamily, version, + asNumber, asName, contact, declaredFamily, effectiveFamily, version, operatingSystem, hostName, verifiedHostNames, unverifiedHostNames, recommendedVersion); if (this.documentStore.store(summaryDocument, fingerprint)) { diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 405fff6..f81dc73 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -21,6 +21,7 @@ public class SummaryDocumentTest { false, new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), 20L, "de", DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767", + "m-net telekommunikations gmbh", "torkaz <klaus dot zufall at gmx dot de> " + "fb-token:np5_g_83jmf=", new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index 8100c00..37f7923 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -140,6 +140,7 @@ public class ResourceServletTest { false, new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), 20L, "de", DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767", + "m-net telekommunikations gmbh", "torkaz <klaus dot zufall at gmx dot de> " + "fb-token:np5_g_83jmf=", new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", @@ -159,7 +160,8 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-24 12:00:00"), true, new TreeSet<>(Arrays.asList(new String[] { "Fast", "Named", "Running", "V2Dir", "Valid" })), 1140L, "us", - DateTimeHelper.parse("2013-04-16 18:00:00"), "AS7922", null, + DateTimeHelper.parse("2013-04-16 18:00:00"), "AS7922", + "comcast cable communications, llc", null, new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { @@ -178,6 +180,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Fast", "Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1", DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", + "liberty global operations b.v.", "1024d/51e2a1c7 "steven j. murdoch" " + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm=", new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", @@ -191,7 +194,7 @@ public class ResourceServletTest { Arrays.asList(new String[] { "10.199.7.176" }), DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, - null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, + null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, null, null, null, "0.2.2.39", null, null, null, null, true); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); @@ -201,7 +204,7 @@ public class ResourceServletTest { new String[] { "10.0.52.84" }), DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, - null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, + null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, null, null, null, null, null, null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); @@ -212,7 +215,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-24 01:07:04"), true, new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, - DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, + DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, null, null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); @@ -1171,6 +1174,30 @@ public class ResourceServletTest { }

@Test(timeout = 100) + public void testAsNameComcast() { + this.assertSummaryDocument("/summary?as_name=Comcast", 1, null, 0, null); + } + + @Test(timeout = 100) + public void testAsNameComcastCable() { + this.assertSummaryDocument("/summary?as_name=Comcast Cable", + 1, null, 0, null); + } + + @Test(timeout = 100) + public void testAsNameCableComcast() { + this.assertSummaryDocument("/summary?as_name=Cable Comcast", + 1, null, 0, null); + } + + @Test(timeout = 100) + public void testAsNameMit() { + this.assertSummaryDocument( + "/summary?as_name=Massachusetts Institute of Technology", + 0, null, 0, null); + } + + @Test(timeout = 100) public void testFlagRunning() { this.assertSummaryDocument( "/summary?flag=Running", 3, null, 1, null); diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index 2b67206..c79565a 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -35,6 +35,7 @@ public class SummaryDocumentComparatorTest { false, new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), 20L, "de", DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767", + "m-net telekommunikations gmbh", "torkaz <klaus dot zufall at gmx dot de> " + "fb-token:np5_g_83jmf=", new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",