commit 513b48093fc104201f7f4b31f5a17543297e3c85 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Mar 27 19:26:54 2013 +0100 Add last_changed_address_or_port field to details. Implements Onionoo side of #8374. --- src/org/torproject/onionoo/CurrentNodes.java | 168 +++++++++++++++------- src/org/torproject/onionoo/DetailDataWriter.java | 4 + src/org/torproject/onionoo/Node.java | 31 ++++- web/index.html | 14 ++ 4 files changed, 161 insertions(+), 56 deletions(-) diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java index 487cf4d..d98eb4e 100644 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ b/src/org/torproject/onionoo/CurrentNodes.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -118,18 +119,25 @@ public class CurrentNodes { firstSeenMillis = dateTimeFormat.parse(parts[15] + " " + parts[16]).getTime(); } + long lastChangedAddresses = publishedOrValidAfterMillis; + if (parts.length > 18 && !parts[17].equals("null")) { + lastChangedAddresses = dateTimeFormat.parse(parts[17] + " " + + parts[18]).getTime(); + } if (isRelay) { this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedOrValidAfterMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, - defaultPolicy, portList, firstSeenMillis); + defaultPolicy, portList, firstSeenMillis, + lastChangedAddresses); } else { this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedOrValidAfterMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, - defaultPolicy, portList, firstSeenMillis); + defaultPolicy, portList, firstSeenMillis, + lastChangedAddresses); } } br.close(); @@ -158,7 +166,13 @@ public class CurrentNodes { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + long cutoff = System.currentTimeMillis() + - 7L * 24L * 60L * 60L * 1000L; for (Node entry : this.currentRelays.values()) { + long lastSeenMillis = entry.getLastSeenMillis(); + if (lastSeenMillis < cutoff) { + continue; + } String nickname = entry.getNickname(); String fingerprint = entry.getFingerprint(); String address = entry.getAddress(); @@ -175,8 +189,7 @@ public class CurrentNodes { addressesBuilder.append((written++ > 0 ? "+" : "") + exitAddress); } - String validAfter = dateTimeFormat.format( - entry.getLastSeenMillis()); + String lastSeen = dateTimeFormat.format(lastSeenMillis); String orPort = String.valueOf(entry.getOrPort()); String dirPort = String.valueOf(entry.getDirPort()); StringBuilder flagsBuilder = new StringBuilder(); @@ -197,12 +210,15 @@ public class CurrentNodes { ? entry.getPortList() : "null"; String firstSeen = dateTimeFormat.format( entry.getFirstSeenMillis()); + String lastChangedAddresses = dateTimeFormat.format( + entry.getLastChangedOrAddress()); bw.write("r " + nickname + " " + fingerprint + " " - + addressesBuilder.toString() + " " + validAfter + " " + + addressesBuilder.toString() + " " + lastSeen + " " + orPort + " " + dirPort + " " + flagsBuilder.toString() + " " + consensusWeight + " " + countryCode + " " + hostName + " " + String.valueOf(lastRdnsLookup) + " " + defaultPolicy + " " - + portList + " " + firstSeen + "\n"); + + portList + " " + firstSeen + " " + lastChangedAddresses + + "\n"); } for (Node entry : this.currentBridges.values()) { String nickname = entry.getNickname(); @@ -230,7 +246,7 @@ public class CurrentNodes { bw.write("b " + nickname + " " + fingerprint + " " + addressesBuilder.toString() + " " + published + " " + orPort + " " + dirPort + " " + flagsBuilder.toString() - + " -1 ?? null -1 null null " + firstSeen + "\n"); + + " -1 ?? null -1 null null " + firstSeen + " null null\n"); } bw.close(); } catch (IOException e) { @@ -245,9 +261,6 @@ public class CurrentNodes { private long lastValidAfterMillis = 0L; private long lastPublishedMillis = 0L; - private long cutoff = System.currentTimeMillis() - - 7L * 24L * 60L * 60L * 1000L; - public void readRelayNetworkConsensuses() { DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); @@ -304,7 +317,7 @@ public class CurrentNodes { this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, null, validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, null, null, -1L, defaultPolicy, portList, - validAfterMillis); + validAfterMillis, validAfterMillis); } if (this.lastValidAfterMillis == validAfterMillis) { this.lastBandwidthWeights = consensus.getBandwidthWeights(); @@ -313,33 +326,62 @@ public class CurrentNodes { public void addRelay(String nickname, String fingerprint, String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long validAfterMillis, int orPort, + SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis) { - if (validAfterMillis >= cutoff && - (!this.currentRelays.containsKey(fingerprint) || - this.currentRelays.get(fingerprint).getLastSeenMillis() < - validAfterMillis)) { - Node previousRelay = this.currentRelays.containsKey(fingerprint) - ? this.currentRelays.get(fingerprint) : null; - if (previousRelay != null && hostName == null && - previousRelay.getAddress().equals(address)) { - hostName = previousRelay.getHostName(); - lastRdnsLookup = previousRelay.getLastRdnsLookup(); - } - if (previousRelay != null) { - firstSeenMillis = Math.min(firstSeenMillis, - previousRelay.getFirstSeenMillis()); + String defaultPolicy, String portList, long firstSeenMillis, + long lastChangedAddresses) { + /* Remember addresses and OR/dir ports that the relay advertised at + * the given time. */ + SortedMap<Long, Set<String>> lastAddresses = + new TreeMap<Long, Set<String>>(Collections.reverseOrder()); + Set<String> addresses = new HashSet<String>(); + addresses.add(address + ":" + orPort); + if (dirPort > 0) { + addresses.add(address + ":" + dirPort); + } + addresses.addAll(orAddressesAndPorts); + lastAddresses.put(lastChangedAddresses, addresses); + /* See if there's already an entry for this relay. */ + if (this.currentRelays.containsKey(fingerprint)) { + Node existingEntry = this.currentRelays.get(fingerprint); + if (lastSeenMillis < existingEntry.getLastSeenMillis()) { + /* Use latest information for nickname, current addresses, etc. */ + nickname = existingEntry.getNickname(); + address = existingEntry.getAddress(); + orAddressesAndPorts = existingEntry.getOrAddressesAndPorts(); + exitAddresses = existingEntry.getExitAddresses(); + lastSeenMillis = existingEntry.getLastSeenMillis(); + orPort = existingEntry.getOrPort(); + dirPort = existingEntry.getDirPort(); + relayFlags = existingEntry.getRelayFlags(); + consensusWeight = existingEntry.getConsensusWeight(); + countryCode = existingEntry.getCountryCode(); + defaultPolicy = existingEntry.getDefaultPolicy(); + portList = existingEntry.getPortList(); } - Node entry = new Node(nickname, fingerprint, address, - orAddressesAndPorts, exitAddresses, validAfterMillis, orPort, - dirPort, relayFlags, consensusWeight, countryCode, hostName, - lastRdnsLookup, defaultPolicy, portList, firstSeenMillis); - this.currentRelays.put(fingerprint, entry); - if (validAfterMillis > this.lastValidAfterMillis) { - this.lastValidAfterMillis = validAfterMillis; + if (hostName == null && + existingEntry.getAddress().equals(address)) { + /* Re-use reverse DNS lookup results if available. */ + hostName = existingEntry.getHostName(); + lastRdnsLookup = existingEntry.getLastRdnsLookup(); } + /* Update relay-history fields. */ + firstSeenMillis = Math.min(firstSeenMillis, + existingEntry.getFirstSeenMillis()); + lastAddresses.putAll(existingEntry.getLastAddresses()); + } + /* Add or update entry. */ + Node entry = new Node(nickname, fingerprint, address, + orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, + dirPort, relayFlags, consensusWeight, countryCode, hostName, + lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, + lastAddresses); + this.currentRelays.put(fingerprint, entry); + /* If this entry comes from a new consensus, update our global last + * valid-after time. */ + if (lastSeenMillis > this.lastValidAfterMillis) { + this.lastValidAfterMillis = lastSeenMillis; } } @@ -673,7 +715,8 @@ public class CurrentNodes { } } if (addressNumberASN.containsKey(addressNumber)) { - String[] parts = addressNumberASN.get(addressNumber).split(" ", 2); + String[] parts = addressNumberASN.get(addressNumber).split(" ", + 2); relay.setASNumber(parts[0]); relay.setASName(parts[1]); } @@ -726,34 +769,49 @@ public class CurrentNodes { SortedSet<String> relayFlags = entry.getFlags(); this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, null, publishedMillis, orPort, dirPort, relayFlags, -1, "??", - null, -1L, null, null, publishedMillis); + null, -1L, null, null, publishedMillis, -1L); } } public void addBridge(String nickname, String fingerprint, String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long publishedMillis, int orPort, + SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostname, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis) { - if (publishedMillis >= cutoff && - (!this.currentBridges.containsKey(fingerprint) || - this.currentBridges.get(fingerprint).getLastSeenMillis() < - publishedMillis)) { - Node previousBridge = this.currentBridges.containsKey(fingerprint) - ? this.currentBridges.get(fingerprint) : null; - if (previousBridge != null) { - firstSeenMillis = Math.min(firstSeenMillis, - previousBridge.getFirstSeenMillis()); - } - Node entry = new Node(nickname, fingerprint, address, - orAddressesAndPorts, exitAddresses, publishedMillis, orPort, - dirPort, relayFlags, consensusWeight, countryCode, hostname, - lastRdnsLookup, defaultPolicy, portList, firstSeenMillis); - this.currentBridges.put(fingerprint, entry); - if (publishedMillis > this.lastPublishedMillis) { - this.lastPublishedMillis = publishedMillis; + String defaultPolicy, String portList, long firstSeenMillis, + long lastChangedAddresses) { + /* See if there's already an entry for this bridge. */ + if (this.currentBridges.containsKey(fingerprint)) { + Node existingEntry = this.currentBridges.get(fingerprint); + if (lastSeenMillis < existingEntry.getLastSeenMillis()) { + /* Use latest information for nickname, current addresses, etc. */ + nickname = existingEntry.getNickname(); + address = existingEntry.getAddress(); + orAddressesAndPorts = existingEntry.getOrAddressesAndPorts(); + exitAddresses = existingEntry.getExitAddresses(); + lastSeenMillis = existingEntry.getLastSeenMillis(); + orPort = existingEntry.getOrPort(); + dirPort = existingEntry.getDirPort(); + relayFlags = existingEntry.getRelayFlags(); + consensusWeight = existingEntry.getConsensusWeight(); + countryCode = existingEntry.getCountryCode(); + defaultPolicy = existingEntry.getDefaultPolicy(); + portList = existingEntry.getPortList(); } + /* Update relay-history fields. */ + firstSeenMillis = Math.min(firstSeenMillis, + existingEntry.getFirstSeenMillis()); + } + /* Add or update entry. */ + Node entry = new Node(nickname, fingerprint, address, + orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, + dirPort, relayFlags, consensusWeight, countryCode, hostname, + lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null); + this.currentBridges.put(fingerprint, entry); + /* If this entry comes from a new status, update our global last + * published time. */ + if (lastSeenMillis > this.lastPublishedMillis) { + this.lastPublishedMillis = lastSeenMillis; } } diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java index dcea869..8236d8b 100644 --- a/src/org/torproject/onionoo/DetailDataWriter.java +++ b/src/org/torproject/onionoo/DetailDataWriter.java @@ -575,6 +575,8 @@ public class DetailDataWriter { String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); String firstSeen = dateTimeFormat.format( entry.getFirstSeenMillis()); + String lastChangedOrAddress = dateTimeFormat.format( + entry.getLastChangedOrAddress()); String running = entry.getRunning() ? "true" : "false"; int dirPort = entry.getDirPort(); String countryCode = entry.getCountryCode(); @@ -607,6 +609,8 @@ public class DetailDataWriter { } sb.append(",\n\"last_seen\":\"" + lastSeen + "\""); sb.append(",\n\"first_seen\":\"" + firstSeen + "\""); + sb.append(",\n\"last_changed_address_or_port\":\"" + + lastChangedOrAddress + "\""); sb.append(",\n\"running\":" + running); SortedSet<String> relayFlags = entry.getRelayFlags(); if (!relayFlags.isEmpty()) { diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java index 0857b4c..0cadd38 100644 --- a/src/org/torproject/onionoo/Node.java +++ b/src/org/torproject/onionoo/Node.java @@ -2,8 +2,12 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.util.Map; +import java.util.Set; import java.util.SortedSet; +import java.util.SortedMap; import java.util.TreeSet; +import java.util.TreeMap; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -43,12 +47,14 @@ public class Node { private double exitProbability = -1.0; private String defaultPolicy; private String portList; + private SortedMap<Long, Set<String>> lastAddresses; public Node(String nickname, String fingerprint, String address, SortedSet<String> orAddressesAndPorts, SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis) { + String defaultPolicy, String portList, long firstSeenMillis, + SortedMap<Long, Set<String>> lastAddresses) { this.nickname = nickname; this.fingerprint = fingerprint; try { @@ -82,6 +88,7 @@ public class Node { this.defaultPolicy = defaultPolicy; this.portList = portList; this.firstSeenMillis = firstSeenMillis; + this.lastAddresses = lastAddresses; } public String getFingerprint() { return this.fingerprint; @@ -246,5 +253,27 @@ public class Node { public String getPortList() { return this.portList; } + public SortedMap<Long, Set<String>> getLastAddresses() { + return this.lastAddresses == null ? null : + new TreeMap<Long, Set<String>>(this.lastAddresses); + } + public long getLastChangedOrAddress() { + long lastChangedAddressesMillis = -1L; + if (this.lastAddresses != null) { + Set<String> lastAddresses = null; + for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { + if (lastAddresses != null) { + for (String address : e.getValue()) { + if (!lastAddresses.contains(address)) { + return lastChangedAddressesMillis; + } + } + } + lastChangedAddressesMillis = e.getKey(); + lastAddresses = e.getValue(); + } + } + return lastChangedAddressesMillis; + } } diff --git a/web/index.html b/web/index.html index 4c3491c..e3bbdb7 100755 --- a/web/index.html +++ b/web/index.html @@ -130,6 +130,20 @@ Omitted if the relay does not accept directory connections.</li> <li><b>"last_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this relay was last seen in a network status consensus. Required field.</li> +<li><font color="blue"><b>"last_changed_address_or_port":</b></font> +UTC timestamp (YYYY-MM-DD +hh:mm:ss) when this relay last stopped announcing an IPv4 or IPv6 address +or TCP port where it previously accepted onion-routing or directory +connections. +This timestamp can serve as indicator whether this relay would be a +suitable fallback directory. +Note that this timestamp is reset when a relay drops out of the consensus +for more than 7 days, so that whenever that relay rejoins, it would seem +as if it changed its address(es) at that time; this is a known limitation +of the current Onionoo design. +Required field. +<font color="blue">Added on March 27.</font> +</li> <li><b>"first_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this relay was first seen in a network status consensus. Note that this timestamp is reset when a relay drops out of the consensus