commit 3e9b3d34a5286850f1d8f2d65fa96661d8fa99ed Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Fri Jun 15 11:48:17 2012 +0200 Include exit addresses in summary and search. Implements #5251. --- src/org/torproject/onionoo/CurrentNodes.java | 131 ++++++++++++---------- src/org/torproject/onionoo/DetailDataWriter.java | 11 +-- src/org/torproject/onionoo/Node.java | 19 +++- src/org/torproject/onionoo/ResourceServlet.java | 16 ++- web/index.html | 12 +- 5 files changed, 114 insertions(+), 75 deletions(-) diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java index 63fb8c6..c6b4d8a 100644 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ b/src/org/torproject/onionoo/CurrentNodes.java @@ -50,53 +50,57 @@ public class CurrentNodes { "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); while ((line = br.readLine()) != null) { - if (line.startsWith("r ")) { - String[] parts = line.split(" "); - if (parts.length < 9) { + String[] parts = line.split(" "); + boolean isRelay = parts[0].equals("r"); + if (parts.length < 9) { + System.err.println("Line '" + line + "' in '" + + this.internalRelaySearchDataFile.getAbsolutePath() + + " is invalid. Exiting."); + System.exit(1); + } + String nickname = parts[1]; + String fingerprint = parts[2]; + String addresses = parts[3]; + String address; + SortedSet<String> exitAddresses = new TreeSet<String>(); + if (addresses.contains(";")) { + String[] addressParts = addresses.split(";", -1); + if (addressParts.length != 3) { System.err.println("Line '" + line + "' in '" + this.internalRelaySearchDataFile.getAbsolutePath() + " is invalid. Exiting."); System.exit(1); } - String nickname = parts[1]; - String fingerprint = parts[2]; - String address = parts[3]; - long validAfterMillis = dateTimeFormat.parse(parts[4] + " " - + parts[5]).getTime(); - int orPort = Integer.parseInt(parts[6]); - int dirPort = Integer.parseInt(parts[7]); - SortedSet<String> relayFlags = new TreeSet<String>( - Arrays.asList(parts[8].split(","))); - long consensusWeight = -1L; - if (parts.length > 9) { - consensusWeight = Long.parseLong(parts[9]); - } - String countryCode = "??"; - if (parts.length > 10) { - countryCode = parts[10]; + address = addressParts[0]; + if (addressParts[2].length() > 0) { + exitAddresses.addAll(Arrays.asList( + addressParts[2].split("\\+"))); } - this.addRelay(nickname, fingerprint, address, - validAfterMillis, orPort, dirPort, relayFlags, + } else { + address = addresses; + } + long publishedOrValidAfterMillis = dateTimeFormat.parse( + parts[4] + " " + parts[5]).getTime(); + int orPort = Integer.parseInt(parts[6]); + int dirPort = Integer.parseInt(parts[7]); + SortedSet<String> relayFlags = new TreeSet<String>( + Arrays.asList(parts[8].split(","))); + long consensusWeight = -1L; + if (parts.length > 9) { + consensusWeight = Long.parseLong(parts[9]); + } + String countryCode = "??"; + if (parts.length > 10) { + countryCode = parts[10]; + } + if (isRelay) { + this.addRelay(nickname, fingerprint, address, exitAddresses, + publishedOrValidAfterMillis, orPort, dirPort, relayFlags, + consensusWeight, countryCode); + } else { + this.addBridge(nickname, fingerprint, address, exitAddresses, + publishedOrValidAfterMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode); - } else if (line.startsWith("b ")) { - String[] parts = line.split(" "); - if (parts.length < 9) { - System.err.println("Line '" + line + "' in '" - + this.internalRelaySearchDataFile.getAbsolutePath() - + " is invalid. Exiting."); - System.exit(1); - } - String nickname = parts[1]; - String hashedFingerprint = parts[2]; - String address = parts[3]; - long publishedMillis = dateTimeFormat.parse(parts[4] + " " - + parts[5]).getTime(); - int orPort = Integer.parseInt(parts[6]); - int dirPort = Integer.parseInt(parts[7]); - SortedSet<String> relayFlags = new TreeSet<String>( - Arrays.asList(parts[8].split(","))); - this.addBridge(nickname, hashedFingerprint, address, - publishedMillis, orPort, dirPort, relayFlags); } } br.close(); @@ -129,6 +133,13 @@ public class CurrentNodes { String nickname = entry.getNickname(); String fingerprint = entry.getFingerprint(); String address = entry.getAddress(); + StringBuilder addressesBuilder = new StringBuilder(); + addressesBuilder.append(address + ";;"); + int written = 0; + for (String exitAddress : entry.getExitAddresses()) { + addressesBuilder.append((written++ > 0 ? "+" : "") + + exitAddress); + } String validAfter = dateTimeFormat.format( entry.getLastSeenMillis()); String orPort = String.valueOf(entry.getOrPort()); @@ -142,16 +153,17 @@ public class CurrentNodes { entry.getConsensusWeight()); String countryCode = entry.getCountryCode() != null ? entry.getCountryCode() : "??"; - bw.write("r " + nickname + " " + fingerprint + " " + address + " " - + validAfter + " " + orPort + " " + dirPort + " " + relayFlags - + " " + consensusWeight + " " + countryCode + "\n"); + bw.write("r " + nickname + " " + fingerprint + " " + + addressesBuilder.toString() + " " + validAfter + " " + + orPort + " " + dirPort + " " + relayFlags + " " + + consensusWeight + " " + countryCode + "\n"); } for (Node entry : this.currentBridges.values()) { String nickname = entry.getNickname(); String fingerprint = entry.getFingerprint(); String published = dateTimeFormat.format( entry.getLastSeenMillis()); - String address = String.valueOf(entry.getAddress()); + String address = entry.getAddress(); String orPort = String.valueOf(entry.getOrPort()); String dirPort = String.valueOf(entry.getDirPort()); StringBuilder sb = new StringBuilder(); @@ -159,9 +171,9 @@ public class CurrentNodes { sb.append("," + relayFlag); } String relayFlags = sb.toString().substring(1); - bw.write("b " + nickname + " " + fingerprint + " " + address + " " - + published + " " + orPort + " " + dirPort + " " + relayFlags - + " -1 ??\n"); + bw.write("b " + nickname + " " + fingerprint + " " + address + + ";; " + published + " " + orPort + " " + dirPort + " " + + relayFlags + " -1 ??\n"); } bw.close(); } catch (IOException e) { @@ -219,20 +231,22 @@ public class CurrentNodes { int dirPort = entry.getDirPort(); SortedSet<String> relayFlags = entry.getFlags(); long consensusWeight = entry.getBandwidth(); - this.addRelay(nickname, fingerprint, address, validAfterMillis, - orPort, dirPort, relayFlags, consensusWeight, null); + this.addRelay(nickname, fingerprint, address, null, + validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, + null); } } public void addRelay(String nickname, String fingerprint, - String address, long validAfterMillis, int orPort, int dirPort, + String address, SortedSet<String> exitAddresses, + long validAfterMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode) { if (validAfterMillis >= cutoff && (!this.currentRelays.containsKey(fingerprint) || this.currentRelays.get(fingerprint).getLastSeenMillis() < validAfterMillis)) { - Node entry = new Node(nickname, fingerprint, address, + Node entry = new Node(nickname, fingerprint, address, exitAddresses, validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode); this.currentRelays.put(fingerprint, entry); @@ -328,20 +342,23 @@ public class CurrentNodes { int orPort = entry.getOrPort(); int dirPort = entry.getDirPort(); SortedSet<String> relayFlags = entry.getFlags(); - this.addBridge(nickname, fingerprint, address, publishedMillis, - orPort, dirPort, relayFlags); + this.addBridge(nickname, fingerprint, address, null, + publishedMillis, orPort, dirPort, relayFlags, -1, "??"); } } public void addBridge(String nickname, String fingerprint, - String address, long publishedMillis, int orPort, int dirPort, - SortedSet<String> relayFlags) { + String address, SortedSet<String> exitAddresses, + long publishedMillis, int orPort, int dirPort, + SortedSet<String> relayFlags, long consensusWeight, + String countryCode) { if (publishedMillis >= cutoff && (!this.currentBridges.containsKey(fingerprint) || this.currentBridges.get(fingerprint).getLastSeenMillis() < publishedMillis)) { - Node entry = new Node(nickname, fingerprint, address, - publishedMillis, orPort, dirPort, relayFlags, -1L, null); + Node entry = new Node(nickname, fingerprint, address, exitAddresses, + publishedMillis, orPort, dirPort, relayFlags, consensusWeight, + countryCode); this.currentBridges.put(fingerprint, entry); if (publishedMillis > this.lastPublishedMillis) { this.lastPublishedMillis = publishedMillis; diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java index bb4dbfa..cb4ebe6 100644 --- a/src/org/torproject/onionoo/DetailDataWriter.java +++ b/src/org/torproject/onionoo/DetailDataWriter.java @@ -19,7 +19,6 @@ import java.util.SortedMap; import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; -import java.util.TreeSet; import org.apache.commons.lang.StringEscapeUtils; @@ -374,20 +373,16 @@ public class DetailDataWriter { /* Add exit addresses if at least one of them is distinct from the * onion-routing addresses. */ - SortedSet<String> exitAddresses = new TreeSet<String>(); if (exitListEntries.containsKey(fingerprint)) { for (ExitListEntry exitListEntry : exitListEntries.get(fingerprint)) { - String exitAddress = exitListEntry.getExitAddress(); - if (!exitAddress.equals(address)) { - exitAddresses.add(exitAddress); - } + entry.addExitAddress(exitListEntry.getExitAddress()); } } - if (!exitAddresses.isEmpty()) { + if (!entry.getExitAddresses().isEmpty()) { sb.append(",\n\"exit_addresses\":["); int written = 0; - for (String exitAddress : exitAddresses) { + for (String exitAddress : entry.getExitAddresses()) { sb.append((written++ > 0 ? "," : "") + "\"" + exitAddress + "\""); } diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java index 887a600..f4e5e67 100644 --- a/src/org/torproject/onionoo/Node.java +++ b/src/org/torproject/onionoo/Node.java @@ -3,6 +3,7 @@ package org.torproject.onionoo; import java.util.SortedSet; +import java.util.TreeSet; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -15,6 +16,7 @@ public class Node { private String hashedFingerprint; private String nickname; private String address; + private SortedSet<String> exitAddresses; private String latitude; private String longitude; private String countryCode; @@ -30,8 +32,8 @@ public class Node { private long consensusWeight; private boolean running; public Node(String nickname, String fingerprint, String address, - long lastSeenMillis, int orPort, int dirPort, - SortedSet<String> relayFlags, long consensusWeight, + SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, + int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode) { this.nickname = nickname; this.fingerprint = fingerprint; @@ -43,6 +45,11 @@ public class Node { + "' is not a valid fingerprint."); } this.address = address; + this.exitAddresses = new TreeSet<String>(); + if (exitAddresses != null) { + this.exitAddresses.addAll(exitAddresses); + } + this.exitAddresses.remove(this.address); this.lastSeenMillis = lastSeenMillis; this.orPort = orPort; this.dirPort = dirPort; @@ -62,6 +69,14 @@ public class Node { public String getAddress() { return this.address; } + public void addExitAddress(String exitAddress) { + if (exitAddress.length() > 0 && !this.address.equals(exitAddress)) { + this.exitAddresses.add(exitAddress); + } + } + public SortedSet<String> getExitAddresses() { + return new TreeSet<String>(this.exitAddresses); + } public void setLatitude(String latitude) { this.latitude = latitude; } diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 8744150..66644d3 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -16,7 +16,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import java.util.TimeZone; +import java.util.TreeSet; import java.util.regex.Pattern; import javax.servlet.ServletConfig; @@ -112,10 +114,18 @@ public class ResourceServlet extends HttpServlet { entry.getNickname() : null; String fingerprint = entry.getFingerprint(); String running = entry.getRunning() ? "true" : "false"; - String address = entry.getAddress(); - return String.format("{%s\"f\":\"%s\",\"a\":[\"%s\"],\"r\":%s}", + SortedSet<String> addresses = new TreeSet<String>(); + addresses.add(entry.getAddress()); + addresses.addAll(entry.getExitAddresses()); + StringBuilder addressesBuilder = new StringBuilder(); + int written = 0; + for (String address : addresses) { + addressesBuilder.append((written++ > 0 ? "," : "") + "\"" + address + + "\""); + } + return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}", (nickname == null ? "" : "\"n\":\"" + nickname + "\","), - fingerprint, address, running); + fingerprint, addressesBuilder.toString(), running); } private String formatBridgeSummaryLine(Node entry) { diff --git a/web/index.html b/web/index.html index 589d347..bd44108 100755 --- a/web/index.html +++ b/web/index.html @@ -58,8 +58,11 @@ Omitted if the relay nickname is <i>"Unnamed"</i>.</li> characters. Required field.</li> <li><b>"a":</b> Array of IPv4 or IPv6 addresses where the relay accepts -onion-routing connections. -Required field.</li> +onion-routing connections or which the relay used to exit to the Internet +in the past 24 hours. +Required field. +<font color="blue">Extended field to also contain exit addresses on June +15, 2012.</font></li> <li><b>"r":</b> Boolean field saying whether this relay was listed as running in the last relay network status consensus. Required field.</li> @@ -182,9 +185,8 @@ Required field.</li> <li><b>"or_addresses":</b> Array of IPv4 or IPv6 addresses and TCP ports or port lists where the relay accepts onion-routing connections. Required field.</li> -<li><b>"exit_addresses":</b> Array of IPv4 or IPv6 addresses and TCP ports -or port lists that the relay used to exit to the Internet in the past 24 -hours. +<li><b>"exit_addresses":</b> Array of IPv4 or IPv6 addresses that the +relay used to exit to the Internet in the past 24 hours. Only those addresses are listed that are different from onion-routing addresses. Optional field.