[tor-commits] [onionoo/master] Provides more accurate DNS results

commit 48e5ff84d067254339d2e303547a48cc7ef00183 Author: Iain R. Learmonth irl@fsfe.org Date: Wed Jul 4 16:43:23 2018 +0100

Provides more accurate DNS results

This commit adds two new fields: {un,}verified_host_names.

Whereas previously InetAddress was used to resolve reverse domain names, this instead changes the lookup mechanism to use JNDI allowing for a deeper view into the DNS. It also accounts for the fact that multiple PTR records are not forbidden in the DNS specification and are often used in shared hosting scenarios.

A host name is considered verified if it has a matching forward record. If a PTR value is found to have multiple A records, it will be considered verified if any one of the A records matches the original address. If no matching record is found, it will be reported as an unverified host name.

Previously, unverified host names were discarded internally by the InetAddress lookup mechanism and so this data could not be used.

To maintain "bug compatibility" with the previous implementation of the "host_name" field, which will now be deprecated, the IP address is returned when a lookup fails.

The host_name field continues to be used, but now will consider all verified and unverified host names. If finer grained filtering is needed, then a seperate ticket could be filed for that, but it is unclear that it is useful enough to justify the work.

Fixes: #18342 --- CHANGELOG.md | 7 +++ .../torproject/onionoo/docs/DetailsDocument.java | 68 ++++++++++++++++++++ .../org/torproject/onionoo/docs/DetailsStatus.java | 69 +++++++++++++++++++++ .../org/torproject/onionoo/docs/DocumentStore.java | 3 + .../org/torproject/onionoo/docs/NodeStatus.java | 21 +++++++ .../torproject/onionoo/docs/SummaryDocument.java | 25 ++++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 14 ++++- .../torproject/onionoo/server/ResponseBuilder.java | 4 ++ .../onionoo/updater/NodeDetailsStatusUpdater.java | 39 +++++++++++- .../onionoo/updater/RdnsLookupRequest.java | 72 +++++++++++++++++++--- .../onionoo/updater/RdnsLookupWorker.java | 20 +++++- .../onionoo/updater/ReverseDomainNameResolver.java | 35 ++++++++++- .../onionoo/writer/DetailsDocumentWriter.java | 4 ++ .../onionoo/writer/SummaryDocumentWriter.java | 5 +- .../onionoo/docs/SummaryDocumentTest.java | 2 +- .../onionoo/server/ResourceServletTest.java | 19 +++--- .../server/SummaryDocumentComparatorTest.java | 2 +- 17 files changed, 384 insertions(+), 25 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc8985..c9c9aba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 6.?-1.??.0 - 2018-??-?? + + * Medium changes + - Provide more accurate DNS results in "verified_host_names" and + "unverified_host_names". + + # Changes in version 6.0-1.14.0 - 2018-05-29

* Medium changes diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index e112efe..7dcf38e 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -242,6 +242,74 @@ public class DetailsDocument extends Document { return unescapeJson(this.host_name); }

+ private List<String> verified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the verified host names, unless the argument was null in + * which case the verified host names are just set to null. + */ + public void setVerifiedHostNames(List<String> verifiedHostNames) { + if (null == verifiedHostNames) { + this.verified_host_names = null; + return; + } + this.verified_host_names = new ArrayList<>(); + for (String hostName : verifiedHostNames) { + this.verified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getVerifiedHostNames() { + if (null == this.verified_host_names) { + return null; + } + List<String> verifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.verified_host_names) { + verifiedHostNames.add(unescapeJson(escapedHostName)); + } + return verifiedHostNames; + } + + private List<String> unverified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the unverified host names, unless the argument was null in + * which case the unverified host names are just set to null. + */ + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + if (null == unverifiedHostNames) { + this.unverified_host_names = null; + return; + } + this.unverified_host_names = new ArrayList<>(); + for (String hostName : unverifiedHostNames) { + this.unverified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getUnverifiedHostNames() { + if (null == this.unverified_host_names) { + return null; + } + List<String> unverifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.unverified_host_names) { + unverifiedHostNames.add(unescapeJson(escapedHostName)); + } + return unverifiedHostNames; + } + private String last_restarted;

public void setLastRestarted(Long lastRestarted) { diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 7ee0bb6..19a7380 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs;

import org.apache.commons.lang3.StringEscapeUtils;

+import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.SortedSet; @@ -531,6 +532,74 @@ public class DetailsStatus extends Document { return unescapeJson(this.host_name); }

+ private List<String> verified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the verified host names, unless the argument was null in + * which case the verified host names are just set to null. + */ + public void setVerifiedHostNames(List<String> verifiedHostNames) { + if (null == verifiedHostNames) { + this.verified_host_names = null; + return; + } + this.verified_host_names = new ArrayList<>(); + for (String hostName : verifiedHostNames) { + this.verified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getVerifiedHostNames() { + if (null == this.verified_host_names) { + return null; + } + List<String> verifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.verified_host_names) { + verifiedHostNames.add(unescapeJson(escapedHostName)); + } + return verifiedHostNames; + } + + private List<String> unverified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the unverified host names, unless the argument was null in + * which case the unverified host names are just set to null. + */ + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + if (null == unverifiedHostNames) { + this.unverified_host_names = null; + return; + } + this.unverified_host_names = new ArrayList<>(); + for (String hostName : unverifiedHostNames) { + this.unverified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getUnverifiedHostNames() { + if (null == this.unverified_host_names) { + return null; + } + List<String> unverifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.unverified_host_names) { + unverifiedHostNames.add(unescapeJson(escapedHostName)); + } + return unverifiedHostNames; + } + private List<String> advertised_or_addresses;

public void setAdvertisedOrAddresses(List<String> advertisedOrAddresses) { diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 3fac5c9..0d75bf9 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -452,11 +452,14 @@ public class DocumentStore { long consensusWeight = -1L; long firstSeenMillis = -1L; String hostName = null; + List<String> verifiedHostNames = null; + List<String> unverifiedHostNames = null; Boolean recommendedVersion = null; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, family, family, version, hostName, + verifiedHostNames, unverifiedHostNames, recommendedVersion); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 1ea8b98..64332da 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -15,6 +15,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -388,6 +389,26 @@ public class NodeStatus extends Document { return this.hostName; }

+ private List<String> verifiedHostNames; + + public void setVerifiedHostNames(List<String> verifiedHostNames) { + this.verifiedHostNames = verifiedHostNames; + } + + public List<String> getVerifiedHostNames() { + return this.verifiedHostNames; + } + + private List<String> unverifiedHostNames; + + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + this.unverifiedHostNames = unverifiedHostNames; + } + + public List<String> getUnverifiedHostNames() { + return this.unverifiedHostNames; + } + private long lastRdnsLookup = -1L;

public void setLastRdnsLookup(long lastRdnsLookup) { diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 31cd84c..8b5a5e1 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -309,6 +309,28 @@ public class SummaryDocument extends Document { return this.hostName; }

+ @JsonProperty("vh") + private List<String> verifiedHostNames; + + public void setVerifiedHostNames(List<String> verifiedHostNames) { + this.verifiedHostNames = verifiedHostNames; + } + + public List<String> getVerifiedHostNames() { + return this.verifiedHostNames; + } + + @JsonProperty("uh") + private List<String> unverifiedHostNames; + + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + this.unverifiedHostNames = unverifiedHostNames; + } + + public List<String> getUnverifiedHostNames() { + return this.unverifiedHostNames; + } + @JsonProperty("rv") private Boolean recommendedVersion;

@@ -334,6 +356,7 @@ public class SummaryDocument extends Document { String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, SortedSet<String> effectiveFamily, String version, String hostName, + List<String> verifiedHostNames, List<String> unverifiedHostNames, Boolean recommendedVersion) { this.setRelay(isRelay); this.setNickname(nickname); @@ -351,6 +374,8 @@ public class SummaryDocument extends Document { this.setEffectiveFamily(effectiveFamily); this.setVersion(version); this.setHostName(hostName); + this.setVerifiedHostNames(verifiedHostNames); + this.setUnverifiedHostNames(unverifiedHostNames); this.setRecommendedVersion(recommendedVersion); } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index e95eda9..c95818d 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -15,8 +15,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory;

import java.io.File; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -266,8 +268,16 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } - String hostName = entry.getHostName(); - if (null != hostName) { + List<String> allHostNames = new ArrayList<>(); + List<String> verifiedHostNames = entry.getVerifiedHostNames(); + if (null != verifiedHostNames) { + allHostNames.addAll(verifiedHostNames); + } + List<String> unverifiedHostNames = entry.getUnverifiedHostNames(); + if (null != unverifiedHostNames) { + allHostNames.addAll(unverifiedHostNames); + } + for (String hostName : allHostNames) { String hostNameLowerCase = hostName.toLowerCase(); if (!newRelaysByHostName.containsKey(hostNameLowerCase)) { newRelaysByHostName.put(hostNameLowerCase, new HashSet<>()); diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index 1dcfeeb..dff9899 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -295,6 +295,10 @@ public class ResponseBuilder { dd.setConsensusWeight(detailsDocument.getConsensusWeight()); } else if (field.equals("host_name")) { dd.setHostName(detailsDocument.getHostName()); + } else if (field.equals("verified_host_names")) { + dd.setVerifiedHostNames(detailsDocument.getVerifiedHostNames()); + } else if (field.equals("unverified_host_names")) { + dd.setUnverifiedHostNames(detailsDocument.getUnverifiedHostNames()); } else if (field.equals("last_restarted")) { dd.setLastRestarted(detailsDocument.getLastRestarted()); } else if (field.equals("bandwidth_rate")) { diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index da55475..ad2cac7 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -757,6 +757,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private long startedRdnsLookups = -1L;

private SortedMap<String, String> rdnsLookupResults = new TreeMap<>(); + private SortedMap<String, List<String>> rdnsVerifiedLookupResults = + new TreeMap<>(); + private SortedMap<String, List<String>> rdnsUnverifiedLookupResults = + new TreeMap<>();

private void finishReverseDomainNameLookups() { this.reverseDomainNameResolver.finishReverseDomainNameLookups(); @@ -764,11 +768,28 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.reverseDomainNameResolver.getLookupStartMillis(); Map<String, String> lookupResults = this.reverseDomainNameResolver.getLookupResults(); + Map<String, List<String>> verifiedLookupResults = + this.reverseDomainNameResolver.getVerifiedLookupResults(); + Map<String, List<String>> unverifiedLookupResults = + this.reverseDomainNameResolver.getUnverifiedLookupResults(); for (String fingerprint : this.currentRelays) { NodeStatus nodeStatus = this.knownNodes.get(fingerprint); String hostName = lookupResults.get(nodeStatus.getAddress()); - if (hostName != null) { + List<String> verifiedHostNames = + verifiedLookupResults.get(nodeStatus.getAddress()); + List<String> unverifiedHostNames = + unverifiedLookupResults.get(nodeStatus.getAddress()); + if (null != hostName) { this.rdnsLookupResults.put(fingerprint, hostName); + } else { + /* Maintains bug compatibility with previous implementation */ + this.rdnsLookupResults.put(fingerprint, nodeStatus.getAddress()); + } + if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) { + this.rdnsVerifiedLookupResults.put(fingerprint, verifiedHostNames); + } + if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) { + this.rdnsUnverifiedLookupResults.put(fingerprint, unverifiedHostNames); } } } @@ -878,6 +899,22 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); }

+ if (this.rdnsVerifiedLookupResults.containsKey(fingerprint)) { + List<String> verifiedHostNames = + this.rdnsVerifiedLookupResults.get(fingerprint); + detailsStatus.setVerifiedHostNames(verifiedHostNames); + nodeStatus.setVerifiedHostNames(verifiedHostNames); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + + if (this.rdnsUnverifiedLookupResults.containsKey(fingerprint)) { + List<String> unverifiedHostNames = + this.rdnsUnverifiedLookupResults.get(fingerprint); + detailsStatus.setUnverifiedHostNames(unverifiedHostNames); + nodeStatus.setUnverifiedHostNames(unverifiedHostNames); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + if (detailsStatus.getLastSeenMillis() < nodeStatus.getLastSeenMillis()) { if (this.lastSeenMeasured.containsKey(fingerprint)) { diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java index cb9169c..7f14350 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -3,18 +3,26 @@

package org.torproject.onionoo.updater;

-import java.net.InetAddress; -import java.net.UnknownHostException; +import org.torproject.onionoo.util.DomainNameSystem; + +import java.util.ArrayList; +import java.util.List; + +import javax.naming.NamingException;

class RdnsLookupRequest extends Thread {

- private final ReverseDomainNameResolver reverseDomainNameResolver; + private ReverseDomainNameResolver reverseDomainNameResolver; + + private DomainNameSystem domainNameSystem;

private RdnsLookupWorker parent;

private String address;

- private String hostName; + private List<String> verifiedHostNames; + + private List<String> unverifiedHostNames;

private long lookupStartedMillis = -1L;

@@ -24,6 +32,8 @@ class RdnsLookupRequest extends Thread { ReverseDomainNameResolver reverseDomainNameResolver, RdnsLookupWorker parent, String address) { this.reverseDomainNameResolver = reverseDomainNameResolver; + this.domainNameSystem = + this.reverseDomainNameResolver.getDomainNameSystemInstance(); this.parent = parent; this.address = address; } @@ -32,23 +42,65 @@ class RdnsLookupRequest extends Thread { public void run() { this.lookupStartedMillis = System.currentTimeMillis(); try { - String result = InetAddress.getByName(this.address).getHostName(); + final List<String> verifiedResults = new ArrayList<>(); + final List<String> unverifiedResults = new ArrayList<>(); + final String[] bytes = this.address.split("\."); + if (bytes.length == 4) { + final String reverseDnsDomain = + bytes[3] + + "." + bytes[2] + "." + bytes[1] + "." + bytes[0] + + ".in-addr.arpa"; + String[] reverseDnsRecords = + this.domainNameSystem.getRecords(reverseDnsDomain, "PTR"); + for (String reverseDnsRecord : reverseDnsRecords) { + boolean verified = false; + String[] forwardDnsRecords = + this.domainNameSystem.getRecords(reverseDnsRecord, "A"); + for (String forwardDnsRecord : forwardDnsRecords) { + if (forwardDnsRecord.equals(this.address)) { + verified = true; + break; + } + } + if (verified) { + verifiedResults.add(reverseDnsRecord.substring(0, + reverseDnsRecord.length() - 1)); + } else { + unverifiedResults.add(reverseDnsRecord.substring(0, + reverseDnsRecord.length() - 1)); + } + } + } synchronized (this) { - this.hostName = result; + this.verifiedHostNames = verifiedResults; + this.unverifiedHostNames = unverifiedResults; } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ + } catch (NamingException e) { + /* The Onionoo field is omitted for both lookup failure and absence of + * a host name. We'll try again the next time. */ } this.lookupCompletedMillis = System.currentTimeMillis(); this.parent.interrupt(); }

public synchronized String getHostName() { - return hostName; + List<String> verifiedHostNames = this.verifiedHostNames; + if (null != verifiedHostNames && !verifiedHostNames.isEmpty() ) { + return verifiedHostNames.get(0); + } else { + return null; + } + } + + public synchronized List<String> getVerifiedHostNames() { + return this.verifiedHostNames; + } + + public synchronized List<String> getUnverifiedHostNames() { + return this.unverifiedHostNames; }

public synchronized long getLookupMillis() { return this.lookupCompletedMillis - this.lookupStartedMillis; } } - diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java index 4c188ed..c94c334 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -3,6 +3,8 @@

package org.torproject.onionoo.updater;

+import java.util.List; + class RdnsLookupWorker extends Thread {

private final ReverseDomainNameResolver reverseDomainNameResolver; @@ -38,12 +40,28 @@ class RdnsLookupWorker extends Thread { /* Getting interrupted should be the default case. */ } String hostName = request.getHostName(); - if (hostName != null) { + if (null != hostName) { synchronized (this.reverseDomainNameResolver.rdnsLookupResults) { this.reverseDomainNameResolver.rdnsLookupResults.put( rdnsLookupJob, hostName); } } + List<String> verifiedHostNames = request.getVerifiedHostNames(); + if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) { + synchronized (this.reverseDomainNameResolver + .rdnsVerifiedLookupResults) { + this.reverseDomainNameResolver.rdnsVerifiedLookupResults.put( + rdnsLookupJob, verifiedHostNames); + } + } + List<String> unverifiedHostNames = request.getUnverifiedHostNames(); + if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) { + synchronized (this.reverseDomainNameResolver + .rdnsUnverifiedLookupResults) { + this.reverseDomainNameResolver.rdnsUnverifiedLookupResults.put( + rdnsLookupJob, unverifiedHostNames); + } + } long lookupMillis = request.getLookupMillis(); if (lookupMillis >= 0L) { synchronized (this.reverseDomainNameResolver.rdnsLookupMillis) { diff --git a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 3c82d14..698b637 100644 --- a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -3,6 +3,7 @@

package org.torproject.onionoo.updater;

+import org.torproject.onionoo.util.DomainNameSystem; import org.torproject.onionoo.util.FormattingUtils;

import java.util.ArrayList; @@ -24,12 +25,18 @@ public class ReverseDomainNameResolver {

private static final int RDNS_LOOKUP_WORKERS_NUM = 5;

+ private DomainNameSystem domainNameSystem; + private Map<String, Long> addressLastLookupTimes;

Set<String> rdnsLookupJobs;

Map<String, String> rdnsLookupResults;

+ Map<String, List<String>> rdnsVerifiedLookupResults; + + Map<String, List<String>> rdnsUnverifiedLookupResults; + List<Long> rdnsLookupMillis;

long startedRdnsLookups; @@ -43,6 +50,7 @@ public class ReverseDomainNameResolver { /** Starts reverse domain name lookups in one or more background * threads and returns immediately. */ public void startReverseDomainNameLookups() { + this.domainNameSystem = new DomainNameSystem(); this.startedRdnsLookups = System.currentTimeMillis(); this.rdnsLookupJobs = new HashSet<>(); for (Map.Entry<String, Long> e : @@ -53,6 +61,8 @@ public class ReverseDomainNameResolver { } } this.rdnsLookupResults = new HashMap<>(); + this.rdnsVerifiedLookupResults = new HashMap<>(); + this.rdnsUnverifiedLookupResults = new HashMap<>(); this.rdnsLookupMillis = new ArrayList<>(); this.rdnsLookupWorkers = new ArrayList<>(); for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { @@ -83,19 +93,42 @@ public class ReverseDomainNameResolver { } }

+ /** Returns reverse domain name verified lookup results. */ + public Map<String, List<String>> getVerifiedLookupResults() { + synchronized (this.rdnsVerifiedLookupResults) { + return new HashMap<>(this.rdnsVerifiedLookupResults); + } + } + + /** Returns reverse domain name unverified lookup results. */ + public Map<String, List<String>> getUnverifiedLookupResults() { + synchronized (this.rdnsUnverifiedLookupResults) { + return new HashMap<>(this.rdnsUnverifiedLookupResults); + } + } + /** Returns the time in milliseconds since the epoch when reverse domain * lookups have been started. */ public long getLookupStartMillis() { return this.startedRdnsLookups; }

+ public DomainNameSystem getDomainNameSystemInstance() { + return this.domainNameSystem; + } + /** Returns a string with the number of performed reverse domain name * lookups and some simple statistics on lookup time. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( rdnsLookupMillis.size()) + " lookups performed\n"); - if (rdnsLookupMillis.size() > 0) { + sb.append(" " + FormattingUtils.formatDecimalNumber( + rdnsVerifiedLookupResults.size()) + " verified results found\n"); + sb.append(" " + FormattingUtils.formatDecimalNumber( + rdnsUnverifiedLookupResults.size()) + + " unverified results found\n"); + if (!rdnsLookupMillis.isEmpty()) { Collections.sort(rdnsLookupMillis); sb.append(" " + FormattingUtils.formatMillis( rdnsLookupMillis.get(0)) + " minimum lookup time\n"); diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index a335916..bc5bf8c 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -76,6 +76,10 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setConsensusWeight( detailsStatus.getConsensusWeight()); detailsDocument.setHostName(detailsStatus.getHostName()); + detailsDocument.setVerifiedHostNames( + detailsStatus.getVerifiedHostNames()); + detailsDocument.setUnverifiedHostNames( + detailsStatus.getUnverifiedHostNames()); String defaultPolicy = detailsStatus.getDefaultPolicy(); String portList = detailsStatus.getPortList(); if (defaultPolicy != null && (defaultPolicy.equals("accept") diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 356feab..ed36d56 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -93,12 +93,15 @@ public class SummaryDocumentWriter implements DocumentWriter { String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); String hostName = nodeStatus.getHostName(); + List<String> verifiedHostNames = nodeStatus.getVerifiedHostNames(); + List<String> unverifiedHostNames = nodeStatus.getUnverifiedHostNames(); Boolean recommendedVersion = nodeStatus.getRecommendedVersion(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, declaredFamily, effectiveFamily, version, - hostName, recommendedVersion); + hostName, verifiedHostNames, unverifiedHostNames, + recommendedVersion); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index a832ca0..459d514 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -27,7 +27,7 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, true); + null, null, null, true); }

@Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index d64d59d..4e9a5f0 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; - import org.junit.Before; import org.junit.Test;

@@ -154,7 +153,10 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", true); + "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", + Arrays.asList( + new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }), + null, true); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = @@ -169,7 +171,10 @@ public class ResourceServletTest { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, - "c-68-38-171-200.hsd1.in.comcast.net", null); + "c-68-38-171-200.hsd1.in.comcast.net", + Arrays.asList( + new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}), + null, null); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", relayFerrari458); org.torproject.onionoo.docs.SummaryDocument relayTimMayTribute = @@ -183,7 +188,7 @@ public class ResourceServletTest { "1024d/51e2a1c7 "steven j. murdoch" " + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm=", new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null, - false); + null, null, false); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -194,7 +199,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, "0.2.2.39", null, true); + null, null, "0.2.2.39", null, null, null, true); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -204,7 +209,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null, null, null); + null, null, null, null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -215,7 +220,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, "0.2.4.4-alpha-dev", null, false); + null, "0.2.4.4-alpha-dev", null, null, null, false); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index e24e26f..569300d 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null); + null, null, null, null); }

/** Some values for running all comparison types. */