commit b1cf0cab52b7155446c77b98cb6f7d696a053ee3 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Jun 18 16:10:21 2012 +0200 Perform reverse DNS lookups to learn relay host names. Implements #5247. --- src/org/torproject/onionoo/CurrentNodes.java | 33 +++++-- src/org/torproject/onionoo/DetailDataWriter.java | 115 ++++++++++++++++++++++ src/org/torproject/onionoo/Main.java | 2 + src/org/torproject/onionoo/Node.java | 18 +++- web/index.html | 8 ++ 5 files changed, 165 insertions(+), 11 deletions(-) diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java index 4da7133..0cfcbf6 100644 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ b/src/org/torproject/onionoo/CurrentNodes.java @@ -98,16 +98,22 @@ public class CurrentNodes { if (parts.length > 10) { countryCode = parts[10]; } + String hostName = null; + long lastRdnsLookup = -1L; + if (parts.length > 12) { + hostName = parts[11].equals("null") ? null : parts[11]; + lastRdnsLookup = Long.parseLong(parts[12]); + } if (isRelay) { this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedOrValidAfterMillis, orPort, dirPort, relayFlags, - consensusWeight, countryCode); + consensusWeight, countryCode, hostName, lastRdnsLookup); } else { this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedOrValidAfterMillis, orPort, dirPort, relayFlags, - consensusWeight, countryCode); + consensusWeight, countryCode, hostName, lastRdnsLookup); } } br.close(); @@ -166,10 +172,14 @@ public class CurrentNodes { entry.getConsensusWeight()); String countryCode = entry.getCountryCode() != null ? entry.getCountryCode() : "??"; + String hostName = entry.getHostName() != null + ? entry.getHostName() : "null"; + long lastRdnsLookup = entry.getLastRdnsLookup(); bw.write("r " + nickname + " " + fingerprint + " " + addressesBuilder.toString() + " " + validAfter + " " + orPort + " " + dirPort + " " + relayFlags + " " - + consensusWeight + " " + countryCode + "\n"); + + consensusWeight + " " + countryCode + " " + hostName + " " + + String.valueOf(lastRdnsLookup) + "\n"); } for (Node entry : this.currentBridges.values()) { String nickname = entry.getNickname(); @@ -194,7 +204,7 @@ public class CurrentNodes { String relayFlags = sb.toString().substring(1); bw.write("b " + nickname + " " + fingerprint + " " + addressesBuilder.toString() + " " + published + " " + orPort - + " " + dirPort + " " + relayFlags + " -1 ??\n"); + + " " + dirPort + " " + relayFlags + " -1 ?? null -1\n"); } bw.close(); } catch (IOException e) { @@ -256,7 +266,7 @@ public class CurrentNodes { long consensusWeight = entry.getBandwidth(); this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, null, validAfterMillis, orPort, dirPort, relayFlags, - consensusWeight, null); + consensusWeight, null, null, -1L); } } @@ -264,14 +274,15 @@ public class CurrentNodes { String address, SortedSet<String> orAddressesAndPorts, SortedSet<String> exitAddresses, long validAfterMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode) { + String countryCode, String hostname, long lastRdnsLookup) { if (validAfterMillis >= cutoff && (!this.currentRelays.containsKey(fingerprint) || this.currentRelays.get(fingerprint).getLastSeenMillis() < validAfterMillis)) { Node entry = new Node(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, validAfterMillis, orPort, - dirPort, relayFlags, consensusWeight, countryCode); + dirPort, relayFlags, consensusWeight, countryCode, hostname, + lastRdnsLookup); this.currentRelays.put(fingerprint, entry); if (validAfterMillis > this.lastValidAfterMillis) { this.lastValidAfterMillis = validAfterMillis; @@ -368,7 +379,8 @@ public class CurrentNodes { int dirPort = entry.getDirPort(); SortedSet<String> relayFlags = entry.getFlags(); this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, - null, publishedMillis, orPort, dirPort, relayFlags, -1, "??"); + null, publishedMillis, orPort, dirPort, relayFlags, -1, "??", + null, -1L); } } @@ -376,14 +388,15 @@ public class CurrentNodes { String address, SortedSet<String> orAddressesAndPorts, SortedSet<String> exitAddresses, long publishedMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode) { + String countryCode, String hostname, long lastRdnsLookup) { if (publishedMillis >= cutoff && (!this.currentBridges.containsKey(fingerprint) || this.currentBridges.get(fingerprint).getLastSeenMillis() < publishedMillis)) { Node entry = new Node(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedMillis, orPort, - dirPort, relayFlags, consensusWeight, countryCode); + dirPort, relayFlags, consensusWeight, countryCode, hostname, + lastRdnsLookup); this.currentBridges.put(fingerprint, entry); if (publishedMillis > this.lastPublishedMillis) { this.lastPublishedMillis = publishedMillis; diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java index 6a724d8..3d03981 100644 --- a/src/org/torproject/onionoo/DetailDataWriter.java +++ b/src/org/torproject/onionoo/DetailDataWriter.java @@ -8,11 +8,15 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -50,6 +54,112 @@ public class DetailDataWriter { this.bridges = currentBridges; } + private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = 10L * 1000L; + private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = 5L * 60L + * 1000L; + private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = 12L * 60L * 60L + * 1000L; + private static final int RDNS_LOOKUP_WORKERS_NUM = 5; + private Set<String> rdnsLookupJobs; + private Map<String, String> rdnsLookupResults; + private long startedRdnsLookups; + private List<RdnsLookupWorker> rdnsLookupWorkers; + public void startReverseDomainNameLookups() { + this.startedRdnsLookups = System.currentTimeMillis(); + this.rdnsLookupJobs = new HashSet<String>(); + for (Node relay : relays.values()) { + if (relay.getLastRdnsLookup() < this.startedRdnsLookups + - RDNS_LOOKUP_MAX_AGE_MILLIS) { + this.rdnsLookupJobs.add(relay.getAddress()); + } + } + this.rdnsLookupResults = new HashMap<String, String>(); + this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); + for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { + RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); + this.rdnsLookupWorkers.add(rdnsLookupWorker); + rdnsLookupWorker.setDaemon(true); + rdnsLookupWorker.start(); + } + } + + public void finishReverseDomainNameLookups() { + for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { + try { + rdnsLookupWorker.join(); + } catch (InterruptedException e) { + /* This is not something that we can take care of. Just leave the + * worker thread alone. */ + } + } + synchronized (this.rdnsLookupResults) { + for (Node relay : relays.values()) { + if (this.rdnsLookupResults.containsKey(relay.getAddress())) { + relay.setHostName(this.rdnsLookupResults.get( + relay.getAddress())); + relay.setLastRdnsLookup(this.startedRdnsLookups); + } + } + } + } + + private class RdnsLookupWorker extends Thread { + public void run() { + while (System.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS + <= startedRdnsLookups) { + String rdnsLookupJob = null; + synchronized (rdnsLookupJobs) { + for (String job : rdnsLookupJobs) { + rdnsLookupJob = job; + rdnsLookupJobs.remove(job); + break; + } + } + if (rdnsLookupJob == null) { + break; + } + RdnsLookupRequest request = new RdnsLookupRequest(this, + rdnsLookupJob); + request.setDaemon(true); + request.start(); + try { + Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); + } catch (InterruptedException e) { + /* Getting interrupted should be the default case. */ + } + String hostName = request.getHostName(); + if (hostName != null) { + synchronized (rdnsLookupResults) { + rdnsLookupResults.put(rdnsLookupJob, hostName); + } + } + } + } + } + + private class RdnsLookupRequest extends Thread { + RdnsLookupWorker parent; + String address, hostName; + public RdnsLookupRequest(RdnsLookupWorker parent, String address) { + this.parent = parent; + this.address = address; + } + public void run() { + try { + String result = InetAddress.getByName(this.address).getHostName(); + synchronized (this) { + this.hostName = result; + } + } catch (UnknownHostException e) { + /* We'll try again the next time. */ + } + this.parent.interrupt(); + } + public synchronized String getHostName() { + return hostName; + } + } + private Map<String, ServerDescriptor> relayServerDescriptors = new HashMap<String, ServerDescriptor>(); public void readRelayServerDescriptors() { @@ -331,6 +441,7 @@ public class DetailDataWriter { String aSNumber = entry.getASNumber(); String aSName = entry.getASName(); long consensusWeight = entry.getConsensusWeight(); + String hostName = entry.getHostName(); StringBuilder sb = new StringBuilder(); sb.append("{\"version\":1,\n" + "\"nickname\":\"" + nickname + "\",\n" @@ -381,6 +492,10 @@ public class DetailDataWriter { sb.append(",\n\"consensus_weight\":" + String.valueOf(consensusWeight)); } + if (hostName != null) { + sb.append(",\n\"host_name\":\"" + + StringEscapeUtils.escapeJavaScript(hostName) + "\""); + } /* Add exit addresses if at least one of them is distinct from the * onion-routing addresses. */ diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 2c906db..5cac8de 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -22,10 +22,12 @@ public class Main { DetailDataWriter ddw = new DetailDataWriter(); ddw.setCurrentRelays(cn.getCurrentRelays()); ddw.setCurrentBridges(cn.getCurrentBridges()); + ddw.startReverseDomainNameLookups(); ddw.readRelayServerDescriptors(); ddw.readExitLists(); ddw.readBridgeServerDescriptors(); ddw.readBridgePoolAssignments(); + ddw.finishReverseDomainNameLookups(); ddw.writeDetailDataFiles(); printStatus("Updating bandwidth data."); diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java index 0bb4a23..88ca71c 100644 --- a/src/org/torproject/onionoo/Node.java +++ b/src/org/torproject/onionoo/Node.java @@ -33,11 +33,13 @@ public class Node { private SortedSet<String> relayFlags; private long consensusWeight; private boolean running; + private String hostName; + private long lastRdnsLookup = -1L; public Node(String nickname, String fingerprint, String address, SortedSet<String> orAddressesAndPorts, SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode) { + String countryCode, String hostName, long lastRdnsLookup) { this.nickname = nickname; this.fingerprint = fingerprint; try { @@ -66,6 +68,8 @@ public class Node { this.relayFlags = relayFlags; this.consensusWeight = consensusWeight; this.countryCode = countryCode; + this.hostName = hostName; + this.lastRdnsLookup = lastRdnsLookup; } public String getFingerprint() { return this.fingerprint; @@ -178,5 +182,17 @@ public class Node { public boolean getRunning() { return this.running; } + public void setHostName(String hostName) { + this.hostName = hostName; + } + public String getHostName() { + return this.hostName; + } + public void setLastRdnsLookup(long lastRdnsLookup) { + this.lastRdnsLookup = lastRdnsLookup; + } + public long getLastRdnsLookup() { + return this.lastRdnsLookup; + } } diff --git a/web/index.html b/web/index.html index d9ac6e8..fe937de 100755 --- a/web/index.html +++ b/web/index.html @@ -247,6 +247,14 @@ the directory authorities. The unit is arbitrary; currently it's kilobytes per second, but that might change in the future. Required field.</li> +<li><b><font color="blue">"host_name":</font></b> Host name as found in a +reverse DNS lookup of the relay IP address. +This field is updated at most once in 12 hours, unless the relay IP +address changes. +Optional field. +Omitted if the relay IP address was not looked up or if no lookup request +was successful yet. +<font color="blue">Added field on June 18, 2012.</font></li> <li><b>"last_restarted":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when the relay was last (re-)started. Optional field.</li>