[tor-commits] [onionoo/master] Include exit addresses in summary and search.

karsten at torproject.org karsten at torproject.org
Fri Jun 15 09:52:38 UTC 2012


commit 3e9b3d34a5286850f1d8f2d65fa96661d8fa99ed
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Fri Jun 15 11:48:17 2012 +0200

    Include exit addresses in summary and search.
    
    Implements #5251.
---
 src/org/torproject/onionoo/CurrentNodes.java     |  131 ++++++++++++----------
 src/org/torproject/onionoo/DetailDataWriter.java |   11 +--
 src/org/torproject/onionoo/Node.java             |   19 +++-
 src/org/torproject/onionoo/ResourceServlet.java  |   16 ++-
 web/index.html                                   |   12 +-
 5 files changed, 114 insertions(+), 75 deletions(-)

diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java
index 63fb8c6..c6b4d8a 100644
--- a/src/org/torproject/onionoo/CurrentNodes.java
+++ b/src/org/torproject/onionoo/CurrentNodes.java
@@ -50,53 +50,57 @@ public class CurrentNodes {
             "yyyy-MM-dd HH:mm:ss");
         dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
         while ((line = br.readLine()) != null) {
-          if (line.startsWith("r ")) {
-            String[] parts = line.split(" ");
-            if (parts.length < 9) {
+          String[] parts = line.split(" ");
+          boolean isRelay = parts[0].equals("r");
+          if (parts.length < 9) {
+            System.err.println("Line '" + line + "' in '"
+                + this.internalRelaySearchDataFile.getAbsolutePath()
+                + " is invalid.  Exiting.");
+            System.exit(1);
+          }
+          String nickname = parts[1];
+          String fingerprint = parts[2];
+          String addresses = parts[3];
+          String address;
+          SortedSet<String> exitAddresses = new TreeSet<String>();
+          if (addresses.contains(";")) {
+            String[] addressParts = addresses.split(";", -1);
+            if (addressParts.length != 3) {
               System.err.println("Line '" + line + "' in '"
                   + this.internalRelaySearchDataFile.getAbsolutePath()
                   + " is invalid.  Exiting.");
               System.exit(1);
             }
-            String nickname = parts[1];
-            String fingerprint = parts[2];
-            String address = parts[3];
-            long validAfterMillis = dateTimeFormat.parse(parts[4] + " "
-               + parts[5]).getTime();
-            int orPort = Integer.parseInt(parts[6]);
-            int dirPort = Integer.parseInt(parts[7]);
-            SortedSet<String> relayFlags = new TreeSet<String>(
-                Arrays.asList(parts[8].split(",")));
-            long consensusWeight = -1L;
-            if (parts.length > 9) {
-              consensusWeight = Long.parseLong(parts[9]);
-            }
-            String countryCode = "??";
-            if (parts.length > 10) {
-              countryCode = parts[10];
+            address = addressParts[0];
+            if (addressParts[2].length() > 0) {
+              exitAddresses.addAll(Arrays.asList(
+                  addressParts[2].split("\\+")));
             }
-            this.addRelay(nickname, fingerprint, address,
-                validAfterMillis, orPort, dirPort, relayFlags,
+          } else {
+            address = addresses;
+          }
+          long publishedOrValidAfterMillis = dateTimeFormat.parse(
+              parts[4] + " " + parts[5]).getTime();
+          int orPort = Integer.parseInt(parts[6]);
+          int dirPort = Integer.parseInt(parts[7]);
+          SortedSet<String> relayFlags = new TreeSet<String>(
+              Arrays.asList(parts[8].split(",")));
+          long consensusWeight = -1L;
+          if (parts.length > 9) {
+            consensusWeight = Long.parseLong(parts[9]);
+          }
+          String countryCode = "??";
+          if (parts.length > 10) {
+            countryCode = parts[10];
+          }
+          if (isRelay) {
+            this.addRelay(nickname, fingerprint, address, exitAddresses,
+                publishedOrValidAfterMillis, orPort, dirPort, relayFlags,
+                consensusWeight, countryCode);
+          } else {
+            this.addBridge(nickname, fingerprint, address, exitAddresses,
+                publishedOrValidAfterMillis, orPort, dirPort, relayFlags,
                 consensusWeight, countryCode);
-          } else if (line.startsWith("b ")) {
-            String[] parts = line.split(" ");
-            if (parts.length < 9) {
-              System.err.println("Line '" + line + "' in '"
-                  + this.internalRelaySearchDataFile.getAbsolutePath()
-                  + " is invalid.  Exiting.");
-              System.exit(1);
-            }
-            String nickname = parts[1];
-            String hashedFingerprint = parts[2];
-            String address = parts[3];
-            long publishedMillis = dateTimeFormat.parse(parts[4] + " "
-                + parts[5]).getTime();
-            int orPort = Integer.parseInt(parts[6]);
-            int dirPort = Integer.parseInt(parts[7]);
-            SortedSet<String> relayFlags = new TreeSet<String>(
-                Arrays.asList(parts[8].split(",")));
-            this.addBridge(nickname, hashedFingerprint, address,
-                publishedMillis, orPort, dirPort, relayFlags);
           }
         }
         br.close();
@@ -129,6 +133,13 @@ public class CurrentNodes {
         String nickname = entry.getNickname();
         String fingerprint = entry.getFingerprint();
         String address = entry.getAddress();
+        StringBuilder addressesBuilder = new StringBuilder();
+        addressesBuilder.append(address + ";;");
+        int written = 0;
+        for (String exitAddress : entry.getExitAddresses()) {
+          addressesBuilder.append((written++ > 0 ? "+" : "")
+              + exitAddress);
+        }
         String validAfter = dateTimeFormat.format(
             entry.getLastSeenMillis());
         String orPort = String.valueOf(entry.getOrPort());
@@ -142,16 +153,17 @@ public class CurrentNodes {
             entry.getConsensusWeight());
         String countryCode = entry.getCountryCode() != null
             ? entry.getCountryCode() : "??";
-        bw.write("r " + nickname + " " + fingerprint + " " + address + " "
-            + validAfter + " " + orPort + " " + dirPort + " " + relayFlags
-            + " " + consensusWeight + " " + countryCode + "\n");
+        bw.write("r " + nickname + " " + fingerprint + " "
+            + addressesBuilder.toString() + " " + validAfter + " "
+            + orPort + " " + dirPort + " " + relayFlags + " "
+            + consensusWeight + " " + countryCode + "\n");
       }
       for (Node entry : this.currentBridges.values()) {
         String nickname = entry.getNickname();
         String fingerprint = entry.getFingerprint();
         String published = dateTimeFormat.format(
             entry.getLastSeenMillis());
-        String address = String.valueOf(entry.getAddress());
+        String address = entry.getAddress();
         String orPort = String.valueOf(entry.getOrPort());
         String dirPort = String.valueOf(entry.getDirPort());
         StringBuilder sb = new StringBuilder();
@@ -159,9 +171,9 @@ public class CurrentNodes {
           sb.append("," + relayFlag);
         }
         String relayFlags = sb.toString().substring(1);
-        bw.write("b " + nickname + " " + fingerprint + " " + address + " "
-            + published + " " + orPort + " " + dirPort + " " + relayFlags
-            + " -1 ??\n");
+        bw.write("b " + nickname + " " + fingerprint + " " + address
+            + ";; " + published + " " + orPort + " " + dirPort + " "
+            + relayFlags + " -1 ??\n");
       }
       bw.close();
     } catch (IOException e) {
@@ -219,20 +231,22 @@ public class CurrentNodes {
       int dirPort = entry.getDirPort();
       SortedSet<String> relayFlags = entry.getFlags();
       long consensusWeight = entry.getBandwidth();
-      this.addRelay(nickname, fingerprint, address, validAfterMillis,
-          orPort, dirPort, relayFlags, consensusWeight, null);
+      this.addRelay(nickname, fingerprint, address, null,
+          validAfterMillis, orPort, dirPort, relayFlags, consensusWeight,
+          null);
     }
   }
 
   public void addRelay(String nickname, String fingerprint,
-      String address, long validAfterMillis, int orPort, int dirPort,
+      String address, SortedSet<String> exitAddresses,
+      long validAfterMillis, int orPort, int dirPort,
       SortedSet<String> relayFlags, long consensusWeight,
       String countryCode) {
     if (validAfterMillis >= cutoff &&
         (!this.currentRelays.containsKey(fingerprint) ||
         this.currentRelays.get(fingerprint).getLastSeenMillis() <
         validAfterMillis)) {
-      Node entry = new Node(nickname, fingerprint, address,
+      Node entry = new Node(nickname, fingerprint, address, exitAddresses,
           validAfterMillis, orPort, dirPort, relayFlags, consensusWeight,
           countryCode);
       this.currentRelays.put(fingerprint, entry);
@@ -328,20 +342,23 @@ public class CurrentNodes {
       int orPort = entry.getOrPort();
       int dirPort = entry.getDirPort();
       SortedSet<String> relayFlags = entry.getFlags();
-      this.addBridge(nickname, fingerprint, address, publishedMillis,
-          orPort, dirPort, relayFlags);
+      this.addBridge(nickname, fingerprint, address, null,
+          publishedMillis, orPort, dirPort, relayFlags, -1, "??");
     }
   }
 
   public void addBridge(String nickname, String fingerprint,
-      String address, long publishedMillis, int orPort, int dirPort,
-      SortedSet<String> relayFlags) {
+      String address, SortedSet<String> exitAddresses,
+      long publishedMillis, int orPort, int dirPort,
+      SortedSet<String> relayFlags, long consensusWeight,
+      String countryCode) {
     if (publishedMillis >= cutoff &&
         (!this.currentBridges.containsKey(fingerprint) ||
         this.currentBridges.get(fingerprint).getLastSeenMillis() <
         publishedMillis)) {
-      Node entry = new Node(nickname, fingerprint, address,
-          publishedMillis, orPort, dirPort, relayFlags, -1L, null);
+      Node entry = new Node(nickname, fingerprint, address, exitAddresses,
+          publishedMillis, orPort, dirPort, relayFlags, consensusWeight,
+          countryCode);
       this.currentBridges.put(fingerprint, entry);
       if (publishedMillis > this.lastPublishedMillis) {
         this.lastPublishedMillis = publishedMillis;
diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java
index bb4dbfa..cb4ebe6 100644
--- a/src/org/torproject/onionoo/DetailDataWriter.java
+++ b/src/org/torproject/onionoo/DetailDataWriter.java
@@ -19,7 +19,6 @@ import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TimeZone;
 import java.util.TreeMap;
-import java.util.TreeSet;
 
 import org.apache.commons.lang.StringEscapeUtils;
 
@@ -374,20 +373,16 @@ public class DetailDataWriter {
 
       /* Add exit addresses if at least one of them is distinct from the
        * onion-routing addresses. */
-      SortedSet<String> exitAddresses = new TreeSet<String>();
       if (exitListEntries.containsKey(fingerprint)) {
         for (ExitListEntry exitListEntry :
             exitListEntries.get(fingerprint)) {
-          String exitAddress = exitListEntry.getExitAddress();
-          if (!exitAddress.equals(address)) {
-            exitAddresses.add(exitAddress);
-          }
+          entry.addExitAddress(exitListEntry.getExitAddress());
         }
       }
-      if (!exitAddresses.isEmpty()) {
+      if (!entry.getExitAddresses().isEmpty()) {
         sb.append(",\n\"exit_addresses\":[");
         int written = 0;
-        for (String exitAddress : exitAddresses) {
+        for (String exitAddress : entry.getExitAddresses()) {
           sb.append((written++ > 0 ? "," : "") + "\"" + exitAddress
               + "\"");
         }
diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java
index 887a600..f4e5e67 100644
--- a/src/org/torproject/onionoo/Node.java
+++ b/src/org/torproject/onionoo/Node.java
@@ -3,6 +3,7 @@
 package org.torproject.onionoo;
 
 import java.util.SortedSet;
+import java.util.TreeSet;
 
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
@@ -15,6 +16,7 @@ public class Node {
   private String hashedFingerprint;
   private String nickname;
   private String address;
+  private SortedSet<String> exitAddresses;
   private String latitude;
   private String longitude;
   private String countryCode;
@@ -30,8 +32,8 @@ public class Node {
   private long consensusWeight;
   private boolean running;
   public Node(String nickname, String fingerprint, String address,
-      long lastSeenMillis, int orPort, int dirPort,
-      SortedSet<String> relayFlags, long consensusWeight,
+      SortedSet<String> exitAddresses, long lastSeenMillis, int orPort,
+      int dirPort, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode) {
     this.nickname = nickname;
     this.fingerprint = fingerprint;
@@ -43,6 +45,11 @@ public class Node {
           + "' is not a valid fingerprint.");
     }
     this.address = address;
+    this.exitAddresses = new TreeSet<String>();
+    if (exitAddresses != null) {
+      this.exitAddresses.addAll(exitAddresses);
+    }
+    this.exitAddresses.remove(this.address);
     this.lastSeenMillis = lastSeenMillis;
     this.orPort = orPort;
     this.dirPort = dirPort;
@@ -62,6 +69,14 @@ public class Node {
   public String getAddress() {
     return this.address;
   }
+  public void addExitAddress(String exitAddress) {
+    if (exitAddress.length() > 0 && !this.address.equals(exitAddress)) {
+      this.exitAddresses.add(exitAddress);
+    }
+  }
+  public SortedSet<String> getExitAddresses() {
+    return new TreeSet<String>(this.exitAddresses);
+  }
   public void setLatitude(String latitude) {
     this.latitude = latitude;
   }
diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
index 8744150..66644d3 100644
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ b/src/org/torproject/onionoo/ResourceServlet.java
@@ -16,7 +16,9 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.TimeZone;
+import java.util.TreeSet;
 import java.util.regex.Pattern;
 
 import javax.servlet.ServletConfig;
@@ -112,10 +114,18 @@ public class ResourceServlet extends HttpServlet {
         entry.getNickname() : null;
     String fingerprint = entry.getFingerprint();
     String running = entry.getRunning() ? "true" : "false";
-    String address = entry.getAddress();
-    return String.format("{%s\"f\":\"%s\",\"a\":[\"%s\"],\"r\":%s}",
+    SortedSet<String> addresses = new TreeSet<String>();
+    addresses.add(entry.getAddress());
+    addresses.addAll(entry.getExitAddresses());
+    StringBuilder addressesBuilder = new StringBuilder();
+    int written = 0;
+    for (String address : addresses) {
+      addressesBuilder.append((written++ > 0 ? "," : "") + "\"" + address
+          + "\"");
+    }
+    return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}",
         (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
-        fingerprint, address, running);
+        fingerprint, addressesBuilder.toString(), running);
   }
 
   private String formatBridgeSummaryLine(Node entry) {
diff --git a/web/index.html b/web/index.html
index 589d347..bd44108 100755
--- a/web/index.html
+++ b/web/index.html
@@ -58,8 +58,11 @@ Omitted if the relay nickname is <i>"Unnamed"</i>.</li>
 characters.
 Required field.</li>
 <li><b>"a":</b> Array of IPv4 or IPv6 addresses where the relay accepts
-onion-routing connections.
-Required field.</li>
+onion-routing connections or which the relay used to exit to the Internet
+in the past 24 hours.
+Required field.
+<font color="blue">Extended field to also contain exit addresses on June
+15, 2012.</font></li>
 <li><b>"r":</b> Boolean field saying whether this relay was listed as
 running in the last relay network status consensus.
 Required field.</li>
@@ -182,9 +185,8 @@ Required field.</li>
 <li><b>"or_addresses":</b> Array of IPv4 or IPv6 addresses and TCP ports
 or port lists where the relay accepts onion-routing connections.
 Required field.</li>
-<li><b>"exit_addresses":</b> Array of IPv4 or IPv6 addresses and TCP ports
-or port lists that the relay used to exit to the Internet in the past 24
-hours.
+<li><b>"exit_addresses":</b> Array of IPv4 or IPv6 addresses that the
+relay used to exit to the Internet in the past 24 hours.
 Only those addresses are listed that are different from onion-routing
 addresses.
 Optional field.



More information about the tor-commits mailing list