commit 49595204ad73da7fca1a5285e693400195d66247 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Apr 9 11:08:00 2013 +0200 Remember nodes even if they drop out for 7+ days. Implements step 2 of #6509. --- src/org/torproject/onionoo/CurrentNodes.java | 93 ++++++++++++++----------- src/org/torproject/onionoo/Main.java | 5 +- web/index.html | 27 ++++---- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java index f2bbac7..db4305d 100644 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ b/src/org/torproject/onionoo/CurrentNodes.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -36,16 +37,12 @@ import org.torproject.descriptor.RelayNetworkStatusConsensus; * days. */ public class CurrentNodes { - private File internalRelaySearchDataFile; - /* Read the internal relay search data file from disk. */ - public void readRelaySearchDataFile(File internalRelaySearchDataFile) { - this.internalRelaySearchDataFile = internalRelaySearchDataFile; - if (this.internalRelaySearchDataFile.exists() && - !this.internalRelaySearchDataFile.isDirectory()) { + public void readRelaySearchDataFile(File summaryFile) { + if (summaryFile.exists() && !summaryFile.isDirectory()) { try { BufferedReader br = new BufferedReader(new FileReader( - this.internalRelaySearchDataFile)); + summaryFile)); String line; SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); @@ -55,7 +52,7 @@ public class CurrentNodes { boolean isRelay = parts[0].equals("r"); if (parts.length < 9) { System.err.println("Line '" + line + "' in '" - + this.internalRelaySearchDataFile.getAbsolutePath() + + summaryFile.getAbsolutePath() + "' is invalid. Exiting."); System.exit(1); } @@ -69,7 +66,7 @@ public class CurrentNodes { String[] addressParts = addresses.split(";", -1); if (addressParts.length != 3) { System.err.println("Line '" + line + "' in '" - + this.internalRelaySearchDataFile.getAbsolutePath() + + summaryFile.getAbsolutePath() + " is invalid. Exiting."); System.exit(1); } @@ -147,14 +144,12 @@ public class CurrentNodes { br.close(); } catch (IOException e) { System.err.println("Could not read " - + this.internalRelaySearchDataFile.getAbsolutePath() - + ". Exiting."); + + summaryFile.getAbsolutePath() + ". Exiting."); e.printStackTrace(); System.exit(1); } catch (ParseException e) { System.err.println("Could not read " - + this.internalRelaySearchDataFile.getAbsolutePath() - + ". Exiting."); + + summaryFile.getAbsolutePath() + ". Exiting."); e.printStackTrace(); System.exit(1); } @@ -162,21 +157,18 @@ public class CurrentNodes { } /* Write the internal relay search data file to disk. */ - public void writeRelaySearchDataFile() { + public void writeRelaySearchDataFile(File summaryFile, + boolean includeOldNodes) { try { - this.internalRelaySearchDataFile.getParentFile().mkdirs(); + summaryFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( - this.internalRelaySearchDataFile)); + summaryFile)); SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - long cutoff = System.currentTimeMillis() - - 7L * 24L * 60L * 60L * 1000L; - for (Node entry : this.currentRelays.values()) { - long lastSeenMillis = entry.getLastSeenMillis(); - if (lastSeenMillis < cutoff) { - continue; - } + Collection<Node> relays = includeOldNodes + ? this.knownRelays.values() : this.getCurrentRelays().values(); + for (Node entry : relays) { String nickname = entry.getNickname(); String fingerprint = entry.getFingerprint(); String address = entry.getAddress(); @@ -193,7 +185,8 @@ public class CurrentNodes { addressesBuilder.append((written++ > 0 ? "+" : "") + exitAddress); } - String lastSeen = dateTimeFormat.format(lastSeenMillis); + String lastSeen = dateTimeFormat.format( + entry.getLastSeenMillis()); String orPort = String.valueOf(entry.getOrPort()); String dirPort = String.valueOf(entry.getDirPort()); StringBuilder flagsBuilder = new StringBuilder(); @@ -226,7 +219,10 @@ public class CurrentNodes { + portList + " " + firstSeen + " " + lastChangedAddresses + " " + aSNumber + "\n"); } - for (Node entry : this.currentBridges.values()) { + Collection<Node> bridges = includeOldNodes + ? this.knownBridges.values() + : this.getCurrentBridges().values(); + for (Node entry : bridges) { String nickname = entry.getNickname(); String fingerprint = entry.getFingerprint(); String published = dateTimeFormat.format( @@ -258,8 +254,7 @@ public class CurrentNodes { bw.close(); } catch (IOException e) { System.err.println("Could not write '" - + this.internalRelaySearchDataFile.getAbsolutePath() - + "' to disk. Exiting."); + + summaryFile.getAbsolutePath() + "' to disk. Exiting."); e.printStackTrace(); System.exit(1); } @@ -294,7 +289,7 @@ public class CurrentNodes { public void setRelayRunningBits() { if (this.lastValidAfterMillis > 0L) { - for (Node entry : this.currentRelays.values()) { + for (Node entry : this.knownRelays.values()) { entry.setRunning(entry.getLastSeenMillis() == this.lastValidAfterMillis); } @@ -350,8 +345,8 @@ public class CurrentNodes { addresses.addAll(orAddressesAndPorts); lastAddresses.put(lastChangedAddresses, addresses); /* See if there's already an entry for this relay. */ - if (this.currentRelays.containsKey(fingerprint)) { - Node existingEntry = this.currentRelays.get(fingerprint); + if (this.knownRelays.containsKey(fingerprint)) { + Node existingEntry = this.knownRelays.get(fingerprint); if (lastSeenMillis < existingEntry.getLastSeenMillis()) { /* Use latest information for nickname, current addresses, etc. */ nickname = existingEntry.getNickname(); @@ -384,7 +379,7 @@ public class CurrentNodes { dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, lastAddresses, aSNumber); - this.currentRelays.put(fingerprint, entry); + this.knownRelays.put(fingerprint, entry); /* If this entry comes from a new consensus, update our global last * valid-after time. */ if (lastSeenMillis > this.lastValidAfterMillis) { @@ -436,7 +431,7 @@ public class CurrentNodes { /* Obtain a map from relay IP address strings to numbers. */ Map<String, Long> addressStringNumbers = new HashMap<String, Long>(); Pattern ipv4Pattern = Pattern.compile("^[0-9\\.]{7,15}$"); - for (Node relay : this.currentRelays.values()) { + for (Node relay : this.knownRelays.values()) { String addressString = relay.getAddress(); long addressNumber = -1L; if (ipv4Pattern.matcher(addressString).matches()) { @@ -695,7 +690,7 @@ public class CurrentNodes { } /* Finally, set relays' city and ASN information. */ - for (Node relay : currentRelays.values()) { + for (Node relay : knownRelays.values()) { String addressString = relay.getAddress(); if (addressStringNumbers.containsKey(addressString)) { long addressNumber = addressStringNumbers.get(addressString); @@ -756,7 +751,7 @@ public class CurrentNodes { public void setBridgeRunningBits() { if (this.lastPublishedMillis > 0L) { - for (Node entry : this.currentBridges.values()) { + for (Node entry : this.knownBridges.values()) { entry.setRunning(entry.getRelayFlags().contains("Running") && entry.getLastSeenMillis() == this.lastPublishedMillis); } @@ -788,8 +783,8 @@ public class CurrentNodes { String defaultPolicy, String portList, long firstSeenMillis, long lastChangedAddresses, String aSNumber) { /* See if there's already an entry for this bridge. */ - if (this.currentBridges.containsKey(fingerprint)) { - Node existingEntry = this.currentBridges.get(fingerprint); + if (this.knownBridges.containsKey(fingerprint)) { + Node existingEntry = this.knownBridges.get(fingerprint); if (lastSeenMillis < existingEntry.getLastSeenMillis()) { /* Use latest information for nickname, current addresses, etc. */ nickname = existingEntry.getNickname(); @@ -816,7 +811,7 @@ public class CurrentNodes { dirPort, relayFlags, consensusWeight, countryCode, hostname, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null, aSNumber); - this.currentBridges.put(fingerprint, entry); + this.knownBridges.put(fingerprint, entry); /* If this entry comes from a new status, update our global last * published time. */ if (lastSeenMillis > this.lastPublishedMillis) { @@ -824,16 +819,32 @@ public class CurrentNodes { } } - private SortedMap<String, Node> currentRelays = + private SortedMap<String, Node> knownRelays = new TreeMap<String, Node>(); public SortedMap<String, Node> getCurrentRelays() { - return new TreeMap<String, Node>(this.currentRelays); + long cutoff = System.currentTimeMillis() + - 7L * 24L * 60L * 60L * 1000L; + SortedMap<String, Node> currentRelays = new TreeMap<String, Node>(); + for (Map.Entry<String, Node> e : this.knownRelays.entrySet()) { + if (e.getValue().getLastSeenMillis() >= cutoff) { + currentRelays.put(e.getKey(), e.getValue()); + } + } + return currentRelays; } - private SortedMap<String, Node> currentBridges = + private SortedMap<String, Node> knownBridges = new TreeMap<String, Node>(); public SortedMap<String, Node> getCurrentBridges() { - return new TreeMap<String, Node>(this.currentBridges); + long cutoff = System.currentTimeMillis() + - 7L * 24L * 60L * 60L * 1000L; + SortedMap<String, Node> currentBridges = new TreeMap<String, Node>(); + for (Map.Entry<String, Node> e : this.knownBridges.entrySet()) { + if (e.getValue().getLastSeenMillis() >= cutoff) { + currentBridges.put(e.getKey(), e.getValue()); + } + } + return currentBridges; } public long getLastValidAfterMillis() { diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index e3e7c5b..af45221 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -11,12 +11,13 @@ public class Main { printStatus("Updating internal node list."); CurrentNodes cn = new CurrentNodes(); - cn.readRelaySearchDataFile(new File("out/summary")); + cn.readRelaySearchDataFile(new File("status/summary")); cn.readRelayNetworkConsensuses(); cn.setRelayRunningBits(); cn.lookUpCitiesAndASes(); cn.readBridgeNetworkStatuses(); cn.setBridgeRunningBits(); + cn.writeRelaySearchDataFile(new File("status/summary"), true); printStatus("Updating detail data."); DetailDataWriter ddw = new DetailDataWriter(); @@ -47,7 +48,7 @@ public class Main { wdw.deleteObsoleteWeightsDataFiles(); printStatus("Updating summary data."); - cn.writeRelaySearchDataFile(); + cn.writeRelaySearchDataFile(new File("out/summary"), false); printStatus("Terminating."); } diff --git a/web/index.html b/web/index.html index 152cec6..cfd857d 100755 --- a/web/index.html +++ b/web/index.html @@ -137,20 +137,19 @@ or TCP port where it previously accepted onion-routing or directory connections. This timestamp can serve as indicator whether this relay would be a suitable fallback directory. -Note that this timestamp is reset when a relay drops out of the consensus -for more than 7 days, so that whenever that relay rejoins, it would seem -as if it changed its address(es) at that time; this is a known limitation -of the current Onionoo design. Required field. <font color="blue">Added on March 27.</font> +<font color="blue">On April 9, fixed limitation that timestamp is reset +when a relay drops out of the consensus for more than 7 days and +rejoins.</font> </li> <li><b>"first_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this relay was first seen in a network status consensus. -Note that this timestamp is reset when a relay drops out of the consensus -for more than 7 days, so that whenever that relay rejoins, it would seem -as if it joined for the first time; this is a known limitation of the -current Onionoo design. -Required field.</li> +Required field. +<font color="blue">On April 9, fixed limitation that timestamp is reset +when a relay drops out of the consensus for more than 7 days and +rejoins.</font> +</li> <li><b>"running":</b> Boolean field saying whether this relay was listed as running in the last relay network status consensus. Required field.</li> @@ -340,11 +339,11 @@ was last seen in a bridge network status. Required field.</li> <li><b>"first_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this bridge was first seen in a bridge network status. -Note that this timestamp is reset when a bridge drops out of the network -status for more than 7 days, so that whenever that bridge rejoins, it -would seem as if it joined for the first time; this is a known limitation -of the current Onionoo design. -Required field.</li> +Required field. +<font color="blue">On April 9, fixed limitation that timestamp is reset +when a bridge drops out of the network status for more than 7 days and +rejoins.</font> +</li> <li><b>"running":</b> Boolean field saying whether this bridge was listed as running in the last bridge network status. Required field.</li>