tor-commits July 2018

tor-commits@lists.torproject.org

17 participants
1737 discussions

[onionoo/master] Bump version to 6.1-1.15.0-dev.
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit 335a9ea2ec1449a1f9caa2c07ddab8cbe1a5a53a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 16 16:15:59 2018 +0200 Bump version to 6.1-1.15.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 1f3fa1d..1bc5734 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ <property name="implementation-title" value="Onionoo" /> <property name="onionoo.protocol.version" value="6.1"/> <property name="release.version" - value="${onionoo.protocol.version}-1.15.0"/> + value="${onionoo.protocol.version}-1.15.0-dev"/> <property name="metricslibversion" value="2.4.0"/> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile"

1 0

[onionoo/release] Bump version to 6.0-1.14.0-dev.
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit 13d53601b1ab2a769f888f00548ab6b692aa9d10 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 30 15:58:13 2018 +0200 Bump version to 6.0-1.14.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 596fb84..e1fb1ab 100644 --- a/build.xml +++ b/build.xml @@ -10,7 +10,7 @@ <property name="implementation-title" value="Onionoo" /> <property name="onionoo.protocol.version" value="6.0"/> <property name="release.version" - value="${onionoo.protocol.version}-1.14.0"/> + value="${onionoo.protocol.version}-1.14.0-dev"/> <property name="metricslibversion" value="2.4.0"/> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile"

1 0

[onionoo/release] Removes unnecessary cast
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit 5c8f0d533ff94b6a6098bf2acaf2dbd0560353bc Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Jul 4 16:09:30 2018 +0100 Removes unnecessary cast --- .../java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index b34d2ef..da55475 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -628,7 +628,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, boolean isExit = nodeStatus.getRelayFlags().contains("Exit") && !nodeStatus.getRelayFlags().contains("BadExit"); boolean isGuard = nodeStatus.getRelayFlags().contains("Guard"); - double consensusWeight = (double) nodeStatus.getConsensusWeight(); + double consensusWeight = nodeStatus.getConsensusWeight(); consensusWeights.put(fingerprint, consensusWeight); totalConsensusWeight += consensusWeight; if (consensusContainsBandwidthWeights) {

1 0

[onionoo/release] Adds a DomainNameSystem utility class
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit 5ba56c857b5bb519af3dca3afc06a06cce45a23c Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Jul 4 16:37:57 2018 +0100 Adds a DomainNameSystem utility class This class uses JNDI to look up resource record values from the Domain Name System using the system configured resolver. See: #18342 --- .../torproject/onionoo/util/DomainNameSystem.java | 66 ++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/main/java/org/torproject/onionoo/util/DomainNameSystem.java b/src/main/java/org/torproject/onionoo/util/DomainNameSystem.java new file mode 100644 index 0000000..b085963 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/util/DomainNameSystem.java @@ -0,0 +1,66 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Hashtable; +import java.util.Set; +import java.util.TreeSet; + +import javax.naming.Context; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +public class DomainNameSystem { + + private DirContext dnsContext; + private Logger log; + + /** Creates a new instance. */ + public DomainNameSystem() { + log = LoggerFactory.getLogger(DomainNameSystem.class); + Hashtable<String, String> envProps = new Hashtable<>(); + envProps.put(Context.INITIAL_CONTEXT_FACTORY, + "com.sun.jndi.dns.DnsContextFactory"); + try { + dnsContext = new InitialDirContext(envProps); + } catch (NamingException e) { + log.error( + "Unable to create directory context. " + + "Host name lookup will be disabled!"); + } + } + + /** + * Returns all the values of DNS resource records found for a given host name + * and type. + */ + public String[] getRecords(String hostName, String type) + throws NamingException { + if (dnsContext == null) { + /* Initial setup failed, so all lookups will fail. */ + throw new NamingException(); + } + Set<String> results = new TreeSet<String>(); + Attributes dnsEntries = + dnsContext.getAttributes(hostName, new String[] { type }); + if (dnsEntries != null) { + Attribute dnsAttribute = dnsEntries.get(type); + if (dnsAttribute != null) { + NamingEnumeration<?> dnsEntryIterator = dnsEntries.get(type).getAll(); + while (dnsEntryIterator.hasMoreElements()) { + results.add(dnsEntryIterator.next().toString()); + } + } + } + return results.toArray(new String[results.size()]); + } + +}

1 0

[onionoo/release] Provides more accurate DNS results
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit 48e5ff84d067254339d2e303547a48cc7ef00183 Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Jul 4 16:43:23 2018 +0100 Provides more accurate DNS results This commit adds two new fields: {un,}verified_host_names. Whereas previously InetAddress was used to resolve reverse domain names, this instead changes the lookup mechanism to use JNDI allowing for a deeper view into the DNS. It also accounts for the fact that multiple PTR records are not forbidden in the DNS specification and are often used in shared hosting scenarios. A host name is considered verified if it has a matching forward record. If a PTR value is found to have multiple A records, it will be considered verified if any one of the A records matches the original address. If no matching record is found, it will be reported as an unverified host name. Previously, unverified host names were discarded internally by the InetAddress lookup mechanism and so this data could not be used. To maintain "bug compatibility" with the previous implementation of the "host_name" field, which will now be deprecated, the IP address is returned when a lookup fails. The host_name field continues to be used, but now will consider all verified and unverified host names. If finer grained filtering is needed, then a seperate ticket could be filed for that, but it is unclear that it is useful enough to justify the work. Fixes: #18342 --- CHANGELOG.md | 7 +++ .../torproject/onionoo/docs/DetailsDocument.java | 68 ++++++++++++++++++++ .../org/torproject/onionoo/docs/DetailsStatus.java | 69 +++++++++++++++++++++ .../org/torproject/onionoo/docs/DocumentStore.java | 3 + .../org/torproject/onionoo/docs/NodeStatus.java | 21 +++++++ .../torproject/onionoo/docs/SummaryDocument.java | 25 ++++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 14 ++++- .../torproject/onionoo/server/ResponseBuilder.java | 4 ++ .../onionoo/updater/NodeDetailsStatusUpdater.java | 39 +++++++++++- .../onionoo/updater/RdnsLookupRequest.java | 72 +++++++++++++++++++--- .../onionoo/updater/RdnsLookupWorker.java | 20 +++++- .../onionoo/updater/ReverseDomainNameResolver.java | 35 ++++++++++- .../onionoo/writer/DetailsDocumentWriter.java | 4 ++ .../onionoo/writer/SummaryDocumentWriter.java | 5 +- .../onionoo/docs/SummaryDocumentTest.java | 2 +- .../onionoo/server/ResourceServletTest.java | 19 +++--- .../server/SummaryDocumentComparatorTest.java | 2 +- 17 files changed, 384 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc8985..c9c9aba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 6.?-1.??.0 - 2018-??-?? + + * Medium changes + - Provide more accurate DNS results in "verified_host_names" and + "unverified_host_names". + + # Changes in version 6.0-1.14.0 - 2018-05-29 * Medium changes diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index e112efe..7dcf38e 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -242,6 +242,74 @@ public class DetailsDocument extends Document { return unescapeJson(this.host_name); } + private List<String> verified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the verified host names, unless the argument was null in + * which case the verified host names are just set to null. + */ + public void setVerifiedHostNames(List<String> verifiedHostNames) { + if (null == verifiedHostNames) { + this.verified_host_names = null; + return; + } + this.verified_host_names = new ArrayList<>(); + for (String hostName : verifiedHostNames) { + this.verified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getVerifiedHostNames() { + if (null == this.verified_host_names) { + return null; + } + List<String> verifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.verified_host_names) { + verifiedHostNames.add(unescapeJson(escapedHostName)); + } + return verifiedHostNames; + } + + private List<String> unverified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the unverified host names, unless the argument was null in + * which case the unverified host names are just set to null. + */ + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + if (null == unverifiedHostNames) { + this.unverified_host_names = null; + return; + } + this.unverified_host_names = new ArrayList<>(); + for (String hostName : unverifiedHostNames) { + this.unverified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getUnverifiedHostNames() { + if (null == this.unverified_host_names) { + return null; + } + List<String> unverifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.unverified_host_names) { + unverifiedHostNames.add(unescapeJson(escapedHostName)); + } + return unverifiedHostNames; + } + private String last_restarted; public void setLastRestarted(Long lastRestarted) { diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 7ee0bb6..19a7380 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import org.apache.commons.lang3.StringEscapeUtils; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.SortedSet; @@ -531,6 +532,74 @@ public class DetailsStatus extends Document { return unescapeJson(this.host_name); } + private List<String> verified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the verified host names, unless the argument was null in + * which case the verified host names are just set to null. + */ + public void setVerifiedHostNames(List<String> verifiedHostNames) { + if (null == verifiedHostNames) { + this.verified_host_names = null; + return; + } + this.verified_host_names = new ArrayList<>(); + for (String hostName : verifiedHostNames) { + this.verified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getVerifiedHostNames() { + if (null == this.verified_host_names) { + return null; + } + List<String> verifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.verified_host_names) { + verifiedHostNames.add(unescapeJson(escapedHostName)); + } + return verifiedHostNames; + } + + private List<String> unverified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the unverified host names, unless the argument was null in + * which case the unverified host names are just set to null. + */ + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + if (null == unverifiedHostNames) { + this.unverified_host_names = null; + return; + } + this.unverified_host_names = new ArrayList<>(); + for (String hostName : unverifiedHostNames) { + this.unverified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List<String> getUnverifiedHostNames() { + if (null == this.unverified_host_names) { + return null; + } + List<String> unverifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.unverified_host_names) { + unverifiedHostNames.add(unescapeJson(escapedHostName)); + } + return unverifiedHostNames; + } + private List<String> advertised_or_addresses; public void setAdvertisedOrAddresses(List<String> advertisedOrAddresses) { diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 3fac5c9..0d75bf9 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -452,11 +452,14 @@ public class DocumentStore { long consensusWeight = -1L; long firstSeenMillis = -1L; String hostName = null; + List<String> verifiedHostNames = null; + List<String> unverifiedHostNames = null; Boolean recommendedVersion = null; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, family, family, version, hostName, + verifiedHostNames, unverifiedHostNames, recommendedVersion); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 1ea8b98..64332da 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -15,6 +15,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -388,6 +389,26 @@ public class NodeStatus extends Document { return this.hostName; } + private List<String> verifiedHostNames; + + public void setVerifiedHostNames(List<String> verifiedHostNames) { + this.verifiedHostNames = verifiedHostNames; + } + + public List<String> getVerifiedHostNames() { + return this.verifiedHostNames; + } + + private List<String> unverifiedHostNames; + + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + this.unverifiedHostNames = unverifiedHostNames; + } + + public List<String> getUnverifiedHostNames() { + return this.unverifiedHostNames; + } + private long lastRdnsLookup = -1L; public void setLastRdnsLookup(long lastRdnsLookup) { diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 31cd84c..8b5a5e1 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -309,6 +309,28 @@ public class SummaryDocument extends Document { return this.hostName; } + @JsonProperty("vh") + private List<String> verifiedHostNames; + + public void setVerifiedHostNames(List<String> verifiedHostNames) { + this.verifiedHostNames = verifiedHostNames; + } + + public List<String> getVerifiedHostNames() { + return this.verifiedHostNames; + } + + @JsonProperty("uh") + private List<String> unverifiedHostNames; + + public void setUnverifiedHostNames(List<String> unverifiedHostNames) { + this.unverifiedHostNames = unverifiedHostNames; + } + + public List<String> getUnverifiedHostNames() { + return this.unverifiedHostNames; + } + @JsonProperty("rv") private Boolean recommendedVersion; @@ -334,6 +356,7 @@ public class SummaryDocument extends Document { String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, SortedSet<String> effectiveFamily, String version, String hostName, + List<String> verifiedHostNames, List<String> unverifiedHostNames, Boolean recommendedVersion) { this.setRelay(isRelay); this.setNickname(nickname); @@ -351,6 +374,8 @@ public class SummaryDocument extends Document { this.setEffectiveFamily(effectiveFamily); this.setVersion(version); this.setHostName(hostName); + this.setVerifiedHostNames(verifiedHostNames); + this.setUnverifiedHostNames(unverifiedHostNames); this.setRecommendedVersion(recommendedVersion); } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index e95eda9..c95818d 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -15,8 +15,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -266,8 +268,16 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } - String hostName = entry.getHostName(); - if (null != hostName) { + List<String> allHostNames = new ArrayList<>(); + List<String> verifiedHostNames = entry.getVerifiedHostNames(); + if (null != verifiedHostNames) { + allHostNames.addAll(verifiedHostNames); + } + List<String> unverifiedHostNames = entry.getUnverifiedHostNames(); + if (null != unverifiedHostNames) { + allHostNames.addAll(unverifiedHostNames); + } + for (String hostName : allHostNames) { String hostNameLowerCase = hostName.toLowerCase(); if (!newRelaysByHostName.containsKey(hostNameLowerCase)) { newRelaysByHostName.put(hostNameLowerCase, new HashSet<>()); diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index 1dcfeeb..dff9899 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -295,6 +295,10 @@ public class ResponseBuilder { dd.setConsensusWeight(detailsDocument.getConsensusWeight()); } else if (field.equals("host_name")) { dd.setHostName(detailsDocument.getHostName()); + } else if (field.equals("verified_host_names")) { + dd.setVerifiedHostNames(detailsDocument.getVerifiedHostNames()); + } else if (field.equals("unverified_host_names")) { + dd.setUnverifiedHostNames(detailsDocument.getUnverifiedHostNames()); } else if (field.equals("last_restarted")) { dd.setLastRestarted(detailsDocument.getLastRestarted()); } else if (field.equals("bandwidth_rate")) { diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index da55475..ad2cac7 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -757,6 +757,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private long startedRdnsLookups = -1L; private SortedMap<String, String> rdnsLookupResults = new TreeMap<>(); + private SortedMap<String, List<String>> rdnsVerifiedLookupResults = + new TreeMap<>(); + private SortedMap<String, List<String>> rdnsUnverifiedLookupResults = + new TreeMap<>(); private void finishReverseDomainNameLookups() { this.reverseDomainNameResolver.finishReverseDomainNameLookups(); @@ -764,11 +768,28 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.reverseDomainNameResolver.getLookupStartMillis(); Map<String, String> lookupResults = this.reverseDomainNameResolver.getLookupResults(); + Map<String, List<String>> verifiedLookupResults = + this.reverseDomainNameResolver.getVerifiedLookupResults(); + Map<String, List<String>> unverifiedLookupResults = + this.reverseDomainNameResolver.getUnverifiedLookupResults(); for (String fingerprint : this.currentRelays) { NodeStatus nodeStatus = this.knownNodes.get(fingerprint); String hostName = lookupResults.get(nodeStatus.getAddress()); - if (hostName != null) { + List<String> verifiedHostNames = + verifiedLookupResults.get(nodeStatus.getAddress()); + List<String> unverifiedHostNames = + unverifiedLookupResults.get(nodeStatus.getAddress()); + if (null != hostName) { this.rdnsLookupResults.put(fingerprint, hostName); + } else { + /* Maintains bug compatibility with previous implementation */ + this.rdnsLookupResults.put(fingerprint, nodeStatus.getAddress()); + } + if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) { + this.rdnsVerifiedLookupResults.put(fingerprint, verifiedHostNames); + } + if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) { + this.rdnsUnverifiedLookupResults.put(fingerprint, unverifiedHostNames); } } } @@ -878,6 +899,22 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); } + if (this.rdnsVerifiedLookupResults.containsKey(fingerprint)) { + List<String> verifiedHostNames = + this.rdnsVerifiedLookupResults.get(fingerprint); + detailsStatus.setVerifiedHostNames(verifiedHostNames); + nodeStatus.setVerifiedHostNames(verifiedHostNames); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + + if (this.rdnsUnverifiedLookupResults.containsKey(fingerprint)) { + List<String> unverifiedHostNames = + this.rdnsUnverifiedLookupResults.get(fingerprint); + detailsStatus.setUnverifiedHostNames(unverifiedHostNames); + nodeStatus.setUnverifiedHostNames(unverifiedHostNames); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + if (detailsStatus.getLastSeenMillis() < nodeStatus.getLastSeenMillis()) { if (this.lastSeenMeasured.containsKey(fingerprint)) { diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java index cb9169c..7f14350 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -3,18 +3,26 @@ package org.torproject.onionoo.updater; -import java.net.InetAddress; -import java.net.UnknownHostException; +import org.torproject.onionoo.util.DomainNameSystem; + +import java.util.ArrayList; +import java.util.List; + +import javax.naming.NamingException; class RdnsLookupRequest extends Thread { - private final ReverseDomainNameResolver reverseDomainNameResolver; + private ReverseDomainNameResolver reverseDomainNameResolver; + + private DomainNameSystem domainNameSystem; private RdnsLookupWorker parent; private String address; - private String hostName; + private List<String> verifiedHostNames; + + private List<String> unverifiedHostNames; private long lookupStartedMillis = -1L; @@ -24,6 +32,8 @@ class RdnsLookupRequest extends Thread { ReverseDomainNameResolver reverseDomainNameResolver, RdnsLookupWorker parent, String address) { this.reverseDomainNameResolver = reverseDomainNameResolver; + this.domainNameSystem = + this.reverseDomainNameResolver.getDomainNameSystemInstance(); this.parent = parent; this.address = address; } @@ -32,23 +42,65 @@ class RdnsLookupRequest extends Thread { public void run() { this.lookupStartedMillis = System.currentTimeMillis(); try { - String result = InetAddress.getByName(this.address).getHostName(); + final List<String> verifiedResults = new ArrayList<>(); + final List<String> unverifiedResults = new ArrayList<>(); + final String[] bytes = this.address.split("\\."); + if (bytes.length == 4) { + final String reverseDnsDomain = + bytes[3] + + "." + bytes[2] + "." + bytes[1] + "." + bytes[0] + + ".in-addr.arpa"; + String[] reverseDnsRecords = + this.domainNameSystem.getRecords(reverseDnsDomain, "PTR"); + for (String reverseDnsRecord : reverseDnsRecords) { + boolean verified = false; + String[] forwardDnsRecords = + this.domainNameSystem.getRecords(reverseDnsRecord, "A"); + for (String forwardDnsRecord : forwardDnsRecords) { + if (forwardDnsRecord.equals(this.address)) { + verified = true; + break; + } + } + if (verified) { + verifiedResults.add(reverseDnsRecord.substring(0, + reverseDnsRecord.length() - 1)); + } else { + unverifiedResults.add(reverseDnsRecord.substring(0, + reverseDnsRecord.length() - 1)); + } + } + } synchronized (this) { - this.hostName = result; + this.verifiedHostNames = verifiedResults; + this.unverifiedHostNames = unverifiedResults; } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ + } catch (NamingException e) { + /* The Onionoo field is omitted for both lookup failure and absence of + * a host name. We'll try again the next time. */ } this.lookupCompletedMillis = System.currentTimeMillis(); this.parent.interrupt(); } public synchronized String getHostName() { - return hostName; + List<String> verifiedHostNames = this.verifiedHostNames; + if (null != verifiedHostNames && !verifiedHostNames.isEmpty() ) { + return verifiedHostNames.get(0); + } else { + return null; + } + } + + public synchronized List<String> getVerifiedHostNames() { + return this.verifiedHostNames; + } + + public synchronized List<String> getUnverifiedHostNames() { + return this.unverifiedHostNames; } public synchronized long getLookupMillis() { return this.lookupCompletedMillis - this.lookupStartedMillis; } } - diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java index 4c188ed..c94c334 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -3,6 +3,8 @@ package org.torproject.onionoo.updater; +import java.util.List; + class RdnsLookupWorker extends Thread { private final ReverseDomainNameResolver reverseDomainNameResolver; @@ -38,12 +40,28 @@ class RdnsLookupWorker extends Thread { /* Getting interrupted should be the default case. */ } String hostName = request.getHostName(); - if (hostName != null) { + if (null != hostName) { synchronized (this.reverseDomainNameResolver.rdnsLookupResults) { this.reverseDomainNameResolver.rdnsLookupResults.put( rdnsLookupJob, hostName); } } + List<String> verifiedHostNames = request.getVerifiedHostNames(); + if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) { + synchronized (this.reverseDomainNameResolver + .rdnsVerifiedLookupResults) { + this.reverseDomainNameResolver.rdnsVerifiedLookupResults.put( + rdnsLookupJob, verifiedHostNames); + } + } + List<String> unverifiedHostNames = request.getUnverifiedHostNames(); + if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) { + synchronized (this.reverseDomainNameResolver + .rdnsUnverifiedLookupResults) { + this.reverseDomainNameResolver.rdnsUnverifiedLookupResults.put( + rdnsLookupJob, unverifiedHostNames); + } + } long lookupMillis = request.getLookupMillis(); if (lookupMillis >= 0L) { synchronized (this.reverseDomainNameResolver.rdnsLookupMillis) { diff --git a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 3c82d14..698b637 100644 --- a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -3,6 +3,7 @@ package org.torproject.onionoo.updater; +import org.torproject.onionoo.util.DomainNameSystem; import org.torproject.onionoo.util.FormattingUtils; import java.util.ArrayList; @@ -24,12 +25,18 @@ public class ReverseDomainNameResolver { private static final int RDNS_LOOKUP_WORKERS_NUM = 5; + private DomainNameSystem domainNameSystem; + private Map<String, Long> addressLastLookupTimes; Set<String> rdnsLookupJobs; Map<String, String> rdnsLookupResults; + Map<String, List<String>> rdnsVerifiedLookupResults; + + Map<String, List<String>> rdnsUnverifiedLookupResults; + List<Long> rdnsLookupMillis; long startedRdnsLookups; @@ -43,6 +50,7 @@ public class ReverseDomainNameResolver { /** Starts reverse domain name lookups in one or more background * threads and returns immediately. */ public void startReverseDomainNameLookups() { + this.domainNameSystem = new DomainNameSystem(); this.startedRdnsLookups = System.currentTimeMillis(); this.rdnsLookupJobs = new HashSet<>(); for (Map.Entry<String, Long> e : @@ -53,6 +61,8 @@ public class ReverseDomainNameResolver { } } this.rdnsLookupResults = new HashMap<>(); + this.rdnsVerifiedLookupResults = new HashMap<>(); + this.rdnsUnverifiedLookupResults = new HashMap<>(); this.rdnsLookupMillis = new ArrayList<>(); this.rdnsLookupWorkers = new ArrayList<>(); for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { @@ -83,19 +93,42 @@ public class ReverseDomainNameResolver { } } + /** Returns reverse domain name verified lookup results. */ + public Map<String, List<String>> getVerifiedLookupResults() { + synchronized (this.rdnsVerifiedLookupResults) { + return new HashMap<>(this.rdnsVerifiedLookupResults); + } + } + + /** Returns reverse domain name unverified lookup results. */ + public Map<String, List<String>> getUnverifiedLookupResults() { + synchronized (this.rdnsUnverifiedLookupResults) { + return new HashMap<>(this.rdnsUnverifiedLookupResults); + } + } + /** Returns the time in milliseconds since the epoch when reverse domain * lookups have been started. */ public long getLookupStartMillis() { return this.startedRdnsLookups; } + public DomainNameSystem getDomainNameSystemInstance() { + return this.domainNameSystem; + } + /** Returns a string with the number of performed reverse domain name * lookups and some simple statistics on lookup time. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( rdnsLookupMillis.size()) + " lookups performed\n"); - if (rdnsLookupMillis.size() > 0) { + sb.append(" " + FormattingUtils.formatDecimalNumber( + rdnsVerifiedLookupResults.size()) + " verified results found\n"); + sb.append(" " + FormattingUtils.formatDecimalNumber( + rdnsUnverifiedLookupResults.size()) + + " unverified results found\n"); + if (!rdnsLookupMillis.isEmpty()) { Collections.sort(rdnsLookupMillis); sb.append(" " + FormattingUtils.formatMillis( rdnsLookupMillis.get(0)) + " minimum lookup time\n"); diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index a335916..bc5bf8c 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -76,6 +76,10 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setConsensusWeight( detailsStatus.getConsensusWeight()); detailsDocument.setHostName(detailsStatus.getHostName()); + detailsDocument.setVerifiedHostNames( + detailsStatus.getVerifiedHostNames()); + detailsDocument.setUnverifiedHostNames( + detailsStatus.getUnverifiedHostNames()); String defaultPolicy = detailsStatus.getDefaultPolicy(); String portList = detailsStatus.getPortList(); if (defaultPolicy != null && (defaultPolicy.equals("accept") diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 356feab..ed36d56 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -93,12 +93,15 @@ public class SummaryDocumentWriter implements DocumentWriter { String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); String hostName = nodeStatus.getHostName(); + List<String> verifiedHostNames = nodeStatus.getVerifiedHostNames(); + List<String> unverifiedHostNames = nodeStatus.getUnverifiedHostNames(); Boolean recommendedVersion = nodeStatus.getRecommendedVersion(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, declaredFamily, effectiveFamily, version, - hostName, recommendedVersion); + hostName, verifiedHostNames, unverifiedHostNames, + recommendedVersion); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index a832ca0..459d514 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -27,7 +27,7 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, true); + null, null, null, true); } @Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index d64d59d..4e9a5f0 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; - import org.junit.Before; import org.junit.Test; @@ -154,7 +153,10 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", true); + "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", + Arrays.asList( + new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }), + null, true); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = @@ -169,7 +171,10 @@ public class ResourceServletTest { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, - "c-68-38-171-200.hsd1.in.comcast.net", null); + "c-68-38-171-200.hsd1.in.comcast.net", + Arrays.asList( + new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}), + null, null); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", relayFerrari458); org.torproject.onionoo.docs.SummaryDocument relayTimMayTribute = @@ -183,7 +188,7 @@ public class ResourceServletTest { "1024d/51e2a1c7 \"steven j. murdoch\" " + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null, - false); + null, null, false); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -194,7 +199,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, "0.2.2.39", null, true); + null, null, "0.2.2.39", null, null, null, true); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -204,7 +209,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null, null, null); + null, null, null, null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -215,7 +220,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, "0.2.4.4-alpha-dev", null, false); + null, "0.2.4.4-alpha-dev", null, null, null, false); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index e24e26f..569300d 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null); + null, null, null, null); } /** Some values for running all comparison types. */

1 0

[onionoo/release] Index relays with no known country or AS
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit 641a939cde75fa3a5cddd32ee29a6508fd66b4c4 Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Tue Jul 10 11:51:23 2018 +0100 Index relays with no known country or AS Indexes are created for relays with no known country code or autonomous system number using the special values "xz" and "AS0" respectively. No changes are made to the summary or details documents. Fixes: #26665 --- CHANGELOG.md | 4 ++ .../org/torproject/onionoo/server/NodeIndexer.java | 43 ++++++++++++++-------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c9aba..ca0142b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ - Provide more accurate DNS results in "verified_host_names" and "unverified_host_names". + * Minor changes + - Index relays with no known country code or autonomous system + number using the special values "xz" and "AS0" respectively. + # Changes in version 6.0-1.14.0 - 2018-05-29 diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index c95818d..f6b84b8 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -198,23 +198,36 @@ public class NodeIndexer implements ServletContextListener, Runnable { .toUpperCase(); newRelayFingerprintSummaryLines.put(fingerprint, entry); newRelayFingerprintSummaryLines.put(hashedFingerprint, entry); - if (entry.getCountryCode() != null) { - String countryCode = entry.getCountryCode(); - if (!newRelaysByCountryCode.containsKey(countryCode)) { - newRelaysByCountryCode.put(countryCode, - new HashSet<String>()); - } - newRelaysByCountryCode.get(countryCode).add(fingerprint); - newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); + String countryCode; + if (null != entry.getCountryCode()) { + countryCode = entry.getCountryCode(); + } else { + /* The country code xz will never be assigned for use with ISO 3166-1 + * and is "user-assigned". Fun fact: UN/LOCODE assigns XZ to represent + * installations in international waters. */ + countryCode = "xz"; } - if (entry.getAsNumber() != null) { - String asNumber = entry.getAsNumber(); - if (!newRelaysByAsNumber.containsKey(asNumber)) { - newRelaysByAsNumber.put(asNumber, new HashSet<String>()); - } - newRelaysByAsNumber.get(asNumber).add(fingerprint); - newRelaysByAsNumber.get(asNumber).add(hashedFingerprint); + if (!newRelaysByCountryCode.containsKey(countryCode)) { + newRelaysByCountryCode.put(countryCode, + new HashSet<String>()); + } + newRelaysByCountryCode.get(countryCode).add(fingerprint); + newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); + String asNumber; + if (null != entry.getAsNumber()) { + asNumber = entry.getAsNumber(); + } else { + /* Autonomous system number 0 is reserved by RFC6707, to be used for + * networks that are not routed. The use of this number should be, + * and in the majority of cases probably is, filtered and so it + * shouldn't appear in any lookup databases. */ + asNumber = "AS0"; + } + if (!newRelaysByAsNumber.containsKey(asNumber)) { + newRelaysByAsNumber.put(asNumber, new HashSet<String>()); } + newRelaysByAsNumber.get(asNumber).add(fingerprint); + newRelaysByAsNumber.get(asNumber).add(hashedFingerprint); for (String flag : entry.getRelayFlags()) { String flagLowerCase = flag.toLowerCase(); if (!newRelaysByFlag.containsKey(flagLowerCase)) {

1 0

[onionoo/release] Allows filtering by operating system
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit e95b88f0c344546df2e23262e13e3714040c36ad Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Jul 11 13:08:25 2018 +0100 Allows filtering by operating system The filter is applicable to both bridges and relays. It uses basic string operations to extract the operating system from the platform line of descriptors. Fixes: #6946 --- CHANGELOG.md | 1 + .../org/torproject/onionoo/docs/DocumentStore.java | 5 ++- .../org/torproject/onionoo/docs/NodeStatus.java | 10 +++++ .../torproject/onionoo/docs/SummaryDocument.java | 18 ++++++-- .../org/torproject/onionoo/server/NodeIndex.java | 22 +++++++++ .../org/torproject/onionoo/server/NodeIndexer.java | 22 +++++++++ .../torproject/onionoo/server/RequestHandler.java | 30 +++++++++++++ .../torproject/onionoo/server/ResourceServlet.java | 21 ++++++++- .../onionoo/updater/NodeDetailsStatusUpdater.java | 6 +++ .../onionoo/writer/SummaryDocumentWriter.java | 5 ++- .../onionoo/docs/SummaryDocumentTest.java | 2 +- .../onionoo/server/ResourceServletTest.java | 52 +++++++++++++++++++--- .../server/SummaryDocumentComparatorTest.java | 2 +- 13 files changed, 179 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4cfb82..a9f21b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Medium changes - Provide more accurate DNS results in "verified_host_names" and "unverified_host_names". + - Allow filtering by operating system using the new "os" parameter. * Minor changes - Index relays with no known country code or autonomous system diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 0d75bf9..e474684 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -448,6 +448,7 @@ public class DocumentStore { SortedSet<String> relayFlags = new TreeSet<>(); SortedSet<String> family = null; String version = null; + String operatingSystem = null; long lastSeenMillis = -1L; long consensusWeight = -1L; long firstSeenMillis = -1L; @@ -458,8 +459,8 @@ public class DocumentStore { SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family, version, hostName, - verifiedHostNames, unverifiedHostNames, + asNumber, contact, family, family, version, operatingSystem, + hostName, verifiedHostNames, unverifiedHostNames, recommendedVersion); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 64332da..46da822 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -343,6 +343,16 @@ public class NodeStatus extends Document { return this.versionStatus; } + private String operatingSystem; + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + + public String getOperatingSystem() { + return this.operatingSystem; + } + /* From exit lists: */ private SortedSet<String> exitAddresses; diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 8b5a5e1..92ebe69 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -298,6 +298,17 @@ public class SummaryDocument extends Document { return this.version; } + @JsonProperty("o") + private String operatingSystem; + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + + public String getOperatingSystem() { + return this.operatingSystem; + } + @JsonProperty("h") private String hostName; @@ -355,9 +366,9 @@ public class SummaryDocument extends Document { boolean running, SortedSet<String> relayFlags, long consensusWeight, String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, - SortedSet<String> effectiveFamily, String version, String hostName, - List<String> verifiedHostNames, List<String> unverifiedHostNames, - Boolean recommendedVersion) { + SortedSet<String> effectiveFamily, String version, String operatingSystem, + String hostName, List<String> verifiedHostNames, + List<String> unverifiedHostNames, Boolean recommendedVersion) { this.setRelay(isRelay); this.setNickname(nickname); this.setFingerprint(fingerprint); @@ -373,6 +384,7 @@ public class SummaryDocument extends Document { this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); this.setVersion(version); + this.setOperatingSystem(operatingSystem); this.setHostName(hostName); this.setVerifiedHostNames(verifiedHostNames); this.setUnverifiedHostNames(unverifiedHostNames); diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index 77993a3..ed9ec44 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -190,6 +190,28 @@ class NodeIndex { return this.bridgesByVersion; } + private Map<String, Set<String>> relaysByOperatingSystem; + + public void setRelaysByOperatingSystem( + Map<String, Set<String>> relaysByOperatingSystem) { + this.relaysByOperatingSystem = relaysByOperatingSystem; + } + + public Map<String, Set<String>> getRelaysByOperatingSystem() { + return this.relaysByOperatingSystem; + } + + private Map<String, Set<String>> bridgesByOperatingSystem; + + public void setBridgesByOperatingSystem( + Map<String, Set<String>> bridgesByOperatingSystem) { + this.bridgesByOperatingSystem = bridgesByOperatingSystem; + } + + public Map<String, Set<String>> getBridgesByOperatingSystem() { + return this.bridgesByOperatingSystem; + } + private Map<String, Set<String>> relaysByHostName; public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) { diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index f6b84b8..1f1d279 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -157,6 +157,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { Map<String, Set<String>> newRelaysByFamily = new HashMap<>(); Map<String, Set<String>> newRelaysByVersion = new HashMap<>(); Map<String, Set<String>> newBridgesByVersion = new HashMap<>(); + Map<String, Set<String>> newRelaysByOperatingSystem = new HashMap<>(); + Map<String, Set<String>> newBridgesByOperatingSystem = new HashMap<>(); Map<String, Set<String>> newRelaysByHostName = new HashMap<>(); Map<Boolean, Set<String>> newRelaysByRecommendedVersion = new HashMap<>(); newRelaysByRecommendedVersion.put(true, new HashSet<>()); @@ -281,6 +283,14 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } + String operatingSystem = entry.getOperatingSystem(); + if (null != operatingSystem) { + if (!newRelaysByOperatingSystem.containsKey(operatingSystem)) { + newRelaysByOperatingSystem.put(operatingSystem, new HashSet<>()); + } + newRelaysByOperatingSystem.get(operatingSystem).add(fingerprint); + newRelaysByOperatingSystem.get(operatingSystem).add(hashedFingerprint); + } List<String> allHostNames = new ArrayList<>(); List<String> verifiedHostNames = entry.getVerifiedHostNames(); if (null != verifiedHostNames) { @@ -367,6 +377,16 @@ public class NodeIndexer implements ServletContextListener, Runnable { newBridgesByVersion.get(version).add(hashedFingerprint); newBridgesByVersion.get(version).add(hashedHashedFingerprint); } + String operatingSystem = entry.getOperatingSystem(); + if (null != operatingSystem) { + if (!newBridgesByOperatingSystem.containsKey(operatingSystem)) { + newBridgesByOperatingSystem.put(operatingSystem, new HashSet<>()); + } + newBridgesByOperatingSystem.get(operatingSystem) + .add(hashedFingerprint); + newBridgesByOperatingSystem.get(operatingSystem) + .add(hashedHashedFingerprint); + } Boolean recommendedVersion = entry.getRecommendedVersion(); if (null != recommendedVersion) { newBridgesByRecommendedVersion.get(recommendedVersion).add( @@ -394,6 +414,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis); newNodeIndex.setRelaysByVersion(newRelaysByVersion); newNodeIndex.setBridgesByVersion(newBridgesByVersion); + newNodeIndex.setRelaysByOperatingSystem(newRelaysByOperatingSystem); + newNodeIndex.setBridgesByOperatingSystem(newBridgesByOperatingSystem); newNodeIndex.setRelaysByHostName(newRelaysByHostName); newNodeIndex.setRelaysByRecommendedVersion(newRelaysByRecommendedVersion); newNodeIndex.setBridgesByRecommendedVersion(newBridgesByRecommendedVersion); diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 63b8fe9..b9d9b23 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -97,6 +97,12 @@ public class RequestHandler { this.version = version; } + private String operatingSystem; + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + private String hostName; public void setHostName(String hostName) { @@ -177,6 +183,7 @@ public class RequestHandler { this.filterByContact(); this.filterByFamily(); this.filterByVersion(); + this.filterByOperatingSystem(); this.filterByHostName(); this.filterByRecommendedVersion(); this.order(); @@ -558,6 +565,29 @@ public class RequestHandler { this.filteredBridges.keySet().retainAll(keepBridges); } + private void filterByOperatingSystem() { + if (null == this.operatingSystem) { + /* Not filtering by operating system. */ + return; + } + Set<String> keepRelays = new HashSet<>(); + for (Map.Entry<String, Set<String>> e + : this.nodeIndex.getRelaysByOperatingSystem().entrySet()) { + if (e.getKey().startsWith(this.operatingSystem)) { + keepRelays.addAll(e.getValue()); + } + } + this.filteredRelays.keySet().retainAll(keepRelays); + Set<String> keepBridges = new HashSet<>(); + for (Map.Entry<String, Set<String>> e + : this.nodeIndex.getBridgesByOperatingSystem().entrySet()) { + if (e.getKey().startsWith(this.operatingSystem)) { + keepBridges.addAll(e.getValue()); + } + } + this.filteredBridges.keySet().retainAll(keepBridges); + } + private void filterByHostName() { if (this.hostName == null) { /* Not filtering by host name. */ diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index a64b89c..e119518 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -69,7 +69,7 @@ public class ResourceServlet extends HttpServlet { Arrays.asList("type", "running", "search", "lookup", "fingerprint", "country", "as", "flag", "first_seen_days", "last_seen_days", "contact", "order", "limit", "offset", "fields", "family", "version", - "host_name", "recommended_version")); + "os", "host_name", "recommended_version")); private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -293,6 +293,15 @@ public class ResourceServlet extends HttpServlet { } rh.setVersion(versionParameter); } + if (parameterMap.containsKey("os")) { + String osParameter = this.parseOperatingSystemParameter( + parameterMap.get("os")); + if (null == osParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setOperatingSystem(osParameter); + } if (parameterMap.containsKey("host_name")) { String hostNameParameter = this.parseHostNameParameter( parameterMap.get("host_name")); @@ -607,6 +616,16 @@ public class ResourceServlet extends HttpServlet { return parameter; } + private String parseOperatingSystemParameter(String parameter) { + for (char c : parameter.toCharArray()) { + if (c < 32 || c >= 127) { + /* Only accept printable ASCII. */ + return null; + } + } + return parameter.toLowerCase(); + } + private static Pattern hostNameParameterPattern = Pattern.compile("^[0-9A-Za-z_\\.\\-]+$"); diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index ad2cac7..6d22aa2 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -807,6 +807,12 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } nodeStatus.setContact(detailsStatus.getContact()); + if (null != detailsStatus.getPlatform()) { + String[] platformParts = detailsStatus.getPlatform().split(" on "); + if (platformParts.length > 1) { + nodeStatus.setOperatingSystem(platformParts[1].toLowerCase()); + } + } /* Extract tor software version for bridges from their "platform" line. * (We already know this for relays from "v" lines in the consensus.) */ diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index ed36d56..dc6eba8 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -92,6 +92,7 @@ public class SummaryDocumentWriter implements DocumentWriter { SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); + String operatingSystem = nodeStatus.getOperatingSystem(); String hostName = nodeStatus.getHostName(); List<String> verifiedHostNames = nodeStatus.getVerifiedHostNames(); List<String> unverifiedHostNames = nodeStatus.getUnverifiedHostNames(); @@ -100,8 +101,8 @@ public class SummaryDocumentWriter implements DocumentWriter { nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, declaredFamily, effectiveFamily, version, - hostName, verifiedHostNames, unverifiedHostNames, - recommendedVersion); + operatingSystem, hostName, verifiedHostNames, + unverifiedHostNames, recommendedVersion); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 459d514..405fff6 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -27,7 +27,7 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null, null, true); + null, null, null, null, true); } @Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index 4e9a5f0..145619d 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -153,7 +153,7 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", + "0.2.3.25", "linux", "ppp-62-216-201-221.dynamic.mnet-online.de", Arrays.asList( new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }), null, true); @@ -170,7 +170,7 @@ public class ResourceServletTest { new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, null, "c-68-38-171-200.hsd1.in.comcast.net", Arrays.asList( new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}), @@ -187,8 +187,8 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024d/51e2a1c7 \"steven j. murdoch\" " + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", - new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null, - null, null, false); + new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", + "windows xp", null, null, null, false); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -199,7 +199,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, "0.2.2.39", null, null, null, true); + null, null, "0.2.2.39", null, null, null, null, true); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -209,7 +209,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null, null, null, null, null); + null, null, null, null, null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -220,7 +220,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, "0.2.4.4-alpha-dev", null, null, null, false); + null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } @@ -1648,6 +1648,44 @@ public class ResourceServletTest { this.assertErrorStatusCode("/summary?version=*", 400); } + @Test(timeout = 100) + public void testOperatingSystemLinux() { + this.assertSummaryDocument( + "/summary?os=linux", 1, new String[] {"TorkaZ"}, 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemLinuxMixedCaps() { + this.assertSummaryDocument( + "/summary?os=LiNUx", 1, new String[] {"TorkaZ"}, 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemLin() { + this.assertSummaryDocument( + "/summary?os=lin", 1, new String[] {"TorkaZ"}, 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemWindows() { + this.assertSummaryDocument( + "/summary?os=windows", 1, new String[] {"TimMayTribute"}, + 1, new String[] {"gummy"}); + } + + @Test(timeout = 100) + public void testOperatingSystemWindowsExperience() { + this.assertSummaryDocument( + "/summary?os=windows xp", 1, new String[] {"TimMayTribute"}, + 0, null); + } + + @Test(timeout = 100) + public void testOperatingSystemWindows7() { + this.assertSummaryDocument( + "/summary?os=windows 7", 0, null, 1, new String[] {"gummy"}); + } + @Test public void testHostNameDe() { this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null); diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index 569300d..2b67206 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null, null, null); + null, null, null, null, null); } /** Some values for running all comparison types. */

1 0

[onionoo/release] Avoid IllegalStateException when descriptors are missing.
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit b6e3a1d51722f02f2d391295a676c7b241090213 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 11 09:35:07 2018 +0200 Avoid IllegalStateException when descriptors are missing. A missing descriptor directory has caused an IllegalStateException when we attempted to write our parse history files. Let's be more careful about this case and skip this history file if there is no such directory. Fixes #26711. --- CHANGELOG.md | 2 ++ src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca0142b..d4cfb82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ * Minor changes - Index relays with no known country code or autonomous system number using the special values "xz" and "AS0" respectively. + - Avoid running into an IllegalStateException when CollecTor is + missing a whole descriptor directory. # Changes in version 6.0-1.14.0 - 2018-05-29 diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java index c19fabd..d00b8b2 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java @@ -108,6 +108,11 @@ class DescriptorQueue { if (this.historyFile == null) { return; } + if (null == this.descriptors) { + log.debug("Not writing history file {}, because we did not read a single " + + "descriptor from {}.", this.historyFile, this.directory); + return; + } SortedMap<String, Long> excludedAndParsedFiles = new TreeMap<>(); excludedAndParsedFiles.putAll( this.descriptorReader.getExcludedFiles());

1 0

[onionoo/release] Prepare for 6.1-1.15.0 release.
by karsten＠torproject.org 16 Jul '18

16 Jul '18

commit eee9cf8e12523b2baa0b6d3e30736b4b1ca12eb9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 16 15:27:36 2018 +0200 Prepare for 6.1-1.15.0 release. --- CHANGELOG.md | 2 +- build.xml | 4 ++-- src/main/java/org/torproject/onionoo/server/ResponseBuilder.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9f21b6..cd11502 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 6.?-1.??.0 - 2018-??-?? +# Changes in version 6.1-1.15.0 - 2018-07-16 * Medium changes - Provide more accurate DNS results in "verified_host_names" and diff --git a/build.xml b/build.xml index e1fb1ab..1f3fa1d 100644 --- a/build.xml +++ b/build.xml @@ -8,9 +8,9 @@ <property name="javadoc-title" value="Onionoo API Documentation"/> <property name="implementation-title" value="Onionoo" /> - <property name="onionoo.protocol.version" value="6.0"/> + <property name="onionoo.protocol.version" value="6.1"/> <property name="release.version" - value="${onionoo.protocol.version}-1.14.0-dev"/> + value="${onionoo.protocol.version}-1.15.0"/> <property name="metricslibversion" value="2.4.0"/> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index dff9899..decf559 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -132,7 +132,7 @@ public class ResponseBuilder { return this.charsWritten; } - private static final String PROTOCOL_VERSION = "6.0"; + private static final String PROTOCOL_VERSION = "6.1"; private static final String NEXT_MAJOR_VERSION_SCHEDULED = null;

1 0

[onionoo/master] Prepare for 6.1-1.15.0 release.
by karsten＠torproject.org 16 Jul '18

16 Jul '18

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits July 2018