[tor-commits] [onionoo/master] Use Gson to format JSON details status files.

karsten at torproject.org karsten at torproject.org
Fri May 9 06:35:54 UTC 2014


commit 677c5a3af88f8b7862c47d2148df6f0f9523062e
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Sun Apr 20 19:02:24 2014 +0200

    Use Gson to format JSON details status files.
---
 src/org/torproject/onionoo/DetailsStatus.java      |  130 +++++++++++++-
 src/org/torproject/onionoo/DocumentStore.java      |   29 +++-
 .../onionoo/NodeDetailsStatusUpdater.java          |  179 +++++---------------
 3 files changed, 201 insertions(+), 137 deletions(-)

diff --git a/src/org/torproject/onionoo/DetailsStatus.java b/src/org/torproject/onionoo/DetailsStatus.java
index 1ebb100..1a497e4 100644
--- a/src/org/torproject/onionoo/DetailsStatus.java
+++ b/src/org/torproject/onionoo/DetailsStatus.java
@@ -1,7 +1,133 @@
-/* Copyright 2013 The Tor Project
+/* Copyright 2013--2014 The Tor Project
  * See LICENSE for licensing information */
 package org.torproject.onionoo;
 
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringEscapeUtils;
+
 class DetailsStatus extends Document {
-}
 
+  /* We must ensure that details files only contain ASCII characters
+   * and no UTF-8 characters.  While UTF-8 characters are perfectly
+   * valid in JSON, this would break compatibility with existing files
+   * pretty badly.  We do this by escaping non-ASCII characters, e.g.,
+   * \u00F2.  Gson won't treat this as UTF-8, but will think that we want
+   * to write six characters '\', 'u', '0', '0', 'F', '2'.  The only thing
+   * we'll have to do is to change back the '\\' that Gson writes for the
+   * '\'. */
+  private static String escapeJSON(String s) {
+    return s == null ? null :
+        StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
+  }
+  private static String unescapeJSON(String s) {
+    return s == null ? null :
+        StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
+  }
+
+  private String desc_published;
+  public void setDescPublished(String descPublished) {
+    this.desc_published = descPublished;
+  }
+  public String getDescPublished() {
+    return this.desc_published;
+  }
+
+  private String last_restarted;
+  public void setLastRestarted(String lastRestarted) {
+    this.last_restarted = lastRestarted;
+  }
+  public String getLastRestarted() {
+    return this.last_restarted;
+  }
+
+  private Integer bandwidth_rate;
+  public void setBandwidthRate(Integer bandwidthRate) {
+    this.bandwidth_rate = bandwidthRate;
+  }
+  public Integer getBandwidthRate() {
+    return this.bandwidth_rate;
+  }
+
+  private Integer bandwidth_burst;
+  public void setBandwidthBurst(Integer bandwidthBurst) {
+    this.bandwidth_burst = bandwidthBurst;
+  }
+  public Integer getBandwidthBurst() {
+    return this.bandwidth_burst;
+  }
+
+  private Integer observed_bandwidth;
+  public void setObservedBandwidth(Integer observedBandwidth) {
+    this.observed_bandwidth = observedBandwidth;
+  }
+  public Integer getObservedBandwidth() {
+    return this.observed_bandwidth;
+  }
+
+  private Integer advertised_bandwidth;
+  public void setAdvertisedBandwidth(Integer advertisedBandwidth) {
+    this.advertised_bandwidth = advertisedBandwidth;
+  }
+  public Integer getAdvertisedBandwidth() {
+    return this.advertised_bandwidth;
+  }
+
+  private List<String> exit_policy;
+  public void setExitPolicy(List<String> exitPolicy) {
+    this.exit_policy = exitPolicy;
+  }
+  public List<String> getExitPolicy() {
+    return this.exit_policy;
+  }
+
+  private String contact;
+  public void setContact(String contact) {
+    this.contact = escapeJSON(contact);
+  }
+  public String getContact() {
+    return unescapeJSON(this.contact);
+  }
+
+  private String platform;
+  public void setPlatform(String platform) {
+    this.platform = escapeJSON(platform);
+  }
+  public String getPlatform() {
+    return unescapeJSON(this.platform);
+  }
+
+  private List<String> family;
+  public void setFamily(List<String> family) {
+    this.family = family;
+  }
+  public List<String> getFamily() {
+    return this.family;
+  }
+
+  private Map<String, List<String>> exit_policy_v6_summary;
+  public void setExitPolicyV6Summary(
+      Map<String, List<String>> exitPolicyV6Summary) {
+    this.exit_policy_v6_summary = exitPolicyV6Summary;
+  }
+  public Map<String, List<String>> getExitPolicyV6Summary() {
+    return this.exit_policy_v6_summary;
+  }
+
+  private Boolean hibernating;
+  public void setHibernating(Boolean hibernating) {
+    this.hibernating = hibernating;
+  }
+  public Boolean getHibernating() {
+    return this.hibernating;
+  }
+
+  private String pool_assignment;
+  public void setPoolAssignment(String poolAssignment) {
+    this.pool_assignment = poolAssignment;
+  }
+  public String getPoolAssignment() {
+    return this.pool_assignment;
+  }
+}
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
index e98fb4e..d80999e 100644
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ b/src/org/torproject/onionoo/DocumentStore.java
@@ -21,6 +21,7 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import com.google.gson.JsonParseException;
 
 // TODO For later migration from disk to database, do the following:
@@ -187,13 +188,34 @@ public class DocumentStore {
     String documentString;
     if (document.getDocumentString() != null) {
       documentString = document.getDocumentString();
-    } else if (document instanceof DetailsDocument ||
-          document instanceof BandwidthDocument ||
+    } else if (document instanceof BandwidthDocument ||
           document instanceof WeightsDocument ||
           document instanceof ClientsDocument ||
           document instanceof UptimeDocument) {
       Gson gson = new Gson();
       documentString = gson.toJson(document);
+    } else if (document instanceof DetailsStatus ||
+        document instanceof DetailsDocument) {
+      /* Don't escape HTML characters, like < and >, contained in
+       * strings. */
+      Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+      /* We must ensure that details files only contain ASCII characters
+       * and no UTF-8 characters.  While UTF-8 characters are perfectly
+       * valid in JSON, this would break compatibility with existing files
+       * pretty badly.  We already make sure that all strings in details
+       * objects are escaped JSON, e.g., \u00F2.  When Gson serlializes
+       * this string, it escapes the \ to \\, hence writes \\u00F2.  We
+       * need to undo this and change \\u00F2 back to \u00F2. */
+      documentString = gson.toJson(document).replaceAll("\\\\\\\\u",
+          "\\\\u");
+      /* Existing details statuses don't contain opening and closing curly
+       * brackets, so we should remove them from new details statuses,
+       * too. */
+       if (document instanceof DetailsStatus) {
+         documentString = documentString.substring(
+             documentString.indexOf("{") + 1,
+             documentString.lastIndexOf("}"));
+       }
     } else if (document instanceof BandwidthStatus ||
         document instanceof WeightsStatus ||
         document instanceof ClientsStatus ||
@@ -380,6 +402,9 @@ public class DocumentStore {
         documentType.equals(ClientsStatus.class) ||
         documentType.equals(UptimeStatus.class)) {
       return this.retrieveParsedStatusFile(documentType, documentString);
+    } else if (documentType.equals(DetailsStatus.class)) {
+      return this.retrieveParsedDocumentFile(documentType, "{"
+          + documentString + "}");
     } else {
       System.err.println("Parsing is not supported for type "
           + documentType.getName() + ".");
diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
index 4cfdd33..030a8ec 100644
--- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
+++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
@@ -5,15 +5,14 @@ package org.torproject.onionoo;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
-import java.util.Scanner;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
-import org.apache.commons.lang.StringEscapeUtils;
 import org.torproject.descriptor.BridgeNetworkStatus;
 import org.torproject.descriptor.BridgePoolAssignment;
 import org.torproject.descriptor.Descriptor;
@@ -291,27 +290,16 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
       ServerDescriptor descriptor) {
     String fingerprint = descriptor.getFingerprint();
     DetailsStatus detailsStatus = this.documentStore.retrieve(
-        DetailsStatus.class, false, fingerprint);
+        DetailsStatus.class, true, fingerprint);
     String publishedDateTime =
         DateTimeHelper.format(descriptor.getPublishedMillis());
-    if (detailsStatus != null) {
-      String detailsString = detailsStatus.getDocumentString();
-      String descPublishedLine = "\"desc_published\":\""
-          + publishedDateTime + "\",";
-      Scanner s = new Scanner(detailsString);
-      while (s.hasNextLine()) {
-        String line = s.nextLine();
-        if (line.startsWith("\"desc_published\":\"")) {
-          if (descPublishedLine.compareTo(line) < 0) {
-            return;
-          } else {
-            break;
-          }
-        }
-      }
-      s.close();
+    if (detailsStatus == null) {
+      detailsStatus = new DetailsStatus();
+    } else if (detailsStatus.getDescPublished() != null &&
+        publishedDateTime.compareTo(
+            detailsStatus.getDescPublished()) < 0) {
+      return;
     }
-    StringBuilder sb = new StringBuilder();
     String lastRestartedString = DateTimeHelper.format(
         descriptor.getPublishedMillis() - descriptor.getUptime()
         * DateTimeHelper.ONE_SECOND);
@@ -320,55 +308,31 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
     int observedBandwidth = descriptor.getBandwidthObserved();
     int advertisedBandwidth = Math.min(bandwidthRate,
         Math.min(bandwidthBurst, observedBandwidth));
-    sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
-        + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
-        + "\"bandwidth_rate\":" + bandwidthRate + ",\n"
-        + "\"bandwidth_burst\":" + bandwidthBurst + ",\n"
-        + "\"observed_bandwidth\":" + observedBandwidth + ",\n"
-        + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
-        + "\"exit_policy\":[");
-    int written = 0;
-    for (String exitPolicyLine : descriptor.getExitPolicyLines()) {
-      sb.append((written++ > 0 ? "," : "") + "\n  \"" + exitPolicyLine
-          + "\"");
-    }
-    sb.append("\n]");
-    if (descriptor.getContact() != null) {
-      sb.append(",\n\"contact\":\""
-          + escapeJSON(descriptor.getContact()) + "\"");
-    }
-    if (descriptor.getPlatform() != null) {
-      sb.append(",\n\"platform\":\""
-          + escapeJSON(descriptor.getPlatform()) + "\"");
-    }
-    if (descriptor.getFamilyEntries() != null) {
-      sb.append(",\n\"family\":[");
-      written = 0;
-      for (String familyEntry : descriptor.getFamilyEntries()) {
-        sb.append((written++ > 0 ? "," : "") + "\n  \"" + familyEntry
-            + "\"");
-      }
-      sb.append("\n]");
-    }
+    detailsStatus.setDescPublished(publishedDateTime);
+    detailsStatus.setLastRestarted(lastRestartedString);
+    detailsStatus.setBandwidthRate(bandwidthRate);
+    detailsStatus.setBandwidthBurst(bandwidthBurst);
+    detailsStatus.setObservedBandwidth(observedBandwidth);
+    detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+    detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
+    detailsStatus.setContact(descriptor.getContact());
+    detailsStatus.setPlatform(descriptor.getPlatform());
+    detailsStatus.setFamily(descriptor.getFamilyEntries());
     if (descriptor.getIpv6DefaultPolicy() != null &&
         (descriptor.getIpv6DefaultPolicy().equals("accept") ||
         descriptor.getIpv6DefaultPolicy().equals("reject")) &&
         descriptor.getIpv6PortList() != null) {
-      sb.append(",\n\"exit_policy_v6_summary\":{\""
-          + descriptor.getIpv6DefaultPolicy() + "\":[");
-      int portsWritten = 0;
-      for (String portOrPortRange :
-          descriptor.getIpv6PortList().split(",")) {
-        sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange
-            + "\"");
-      }
-      sb.append("]}");
+      Map<String, List<String>> exitPolicyV6Summary =
+          new HashMap<String, List<String>>();
+      List<String> portsOrPortRanges = Arrays.asList(
+          descriptor.getIpv6PortList().split(","));
+      exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(),
+          portsOrPortRanges);
+      detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary);
     }
     if (descriptor.isHibernating()) {
-      sb.append(",\n\"hibernating\":true");
+      detailsStatus.setHibernating(true);
     }
-    detailsStatus = new DetailsStatus();
-    detailsStatus.setDocumentString(sb.toString());
     this.documentStore.store(detailsStatus, fingerprint);
     if (descriptor.getContact() != null) {
       this.contacts.put(fingerprint, descriptor.getContact());
@@ -379,79 +343,43 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
       ServerDescriptor descriptor) {
     String fingerprint = descriptor.getFingerprint();
     DetailsStatus detailsStatus = this.documentStore.retrieve(
-        DetailsStatus.class, false, fingerprint);
+        DetailsStatus.class, true, fingerprint);
     String publishedDateTime =
         DateTimeHelper.format(descriptor.getPublishedMillis());
-    String poolAssignmentLine = null;
-    if (detailsStatus != null) {
-      String detailsString = detailsStatus.getDocumentString();
-      String descPublishedLine = "\"desc_published\":\""
-          + publishedDateTime + "\",";
-      Scanner s = new Scanner(detailsString);
-      while (s.hasNextLine()) {
-        String line = s.nextLine();
-        if (line.startsWith("\"pool_assignment\":")) {
-          poolAssignmentLine = line;
-        } else if (line.startsWith("\"desc_published\":") &&
-            descPublishedLine.compareTo(line) < 0) {
-          return;
-        }
-      }
-      s.close();
+    if (detailsStatus == null) {
+      detailsStatus = new DetailsStatus();
+    } else if (detailsStatus.getDescPublished() != null &&
+        publishedDateTime.compareTo(
+            detailsStatus.getDescPublished()) < 0) {
+      return;
     }
-    StringBuilder sb = new StringBuilder();
     String lastRestartedString = DateTimeHelper.format(
         descriptor.getPublishedMillis() - descriptor.getUptime()
         * DateTimeHelper.ONE_SECOND);
     int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
         Math.min(descriptor.getBandwidthBurst(),
         descriptor.getBandwidthObserved()));
-    sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
-        + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
-        + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
-        + "\"platform\":\"" + escapeJSON(descriptor.getPlatform())
-        + "\"");
-    if (poolAssignmentLine != null) {
-      sb.append(",\n" + poolAssignmentLine);
-    }
-    detailsStatus = new DetailsStatus();
-    detailsStatus.setDocumentString(sb.toString());
+    detailsStatus.setDescPublished(publishedDateTime);
+    detailsStatus.setLastRestarted(lastRestartedString);
+    detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+    detailsStatus.setPlatform(descriptor.getPlatform());
     this.documentStore.store(detailsStatus, fingerprint);
   }
 
-  private static String escapeJSON(String s) {
-    return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
-  }
-
   private void processBridgePoolAssignment(
       BridgePoolAssignment bridgePoolAssignment) {
     for (Map.Entry<String, String> e :
         bridgePoolAssignment.getEntries().entrySet()) {
       String fingerprint = e.getKey();
       String details = e.getValue();
-      StringBuilder sb = new StringBuilder();
       DetailsStatus detailsStatus = this.documentStore.retrieve(
-          DetailsStatus.class, false, fingerprint);
-      if (detailsStatus != null) {
-        String detailsString = detailsStatus.getDocumentString();
-        Scanner s = new Scanner(detailsString);
-        int linesWritten = 0;
-        boolean endsWithComma = false;
-        while (s.hasNextLine()) {
-          String line = s.nextLine();
-          if (!line.startsWith("\"pool_assignment\":")) {
-            sb.append((linesWritten++ > 0 ? "\n" : "") + line);
-            endsWithComma = line.endsWith(",");
-          }
-        }
-        s.close();
-        if (sb.length() > 0) {
-          sb.append((endsWithComma ? "" : ",") + "\n");
-        }
+          DetailsStatus.class, true, fingerprint);
+      if (detailsStatus == null) {
+        detailsStatus = new DetailsStatus();
+      } else if (details.equals(detailsStatus.getPoolAssignment())) {
+        return;
       }
-      sb.append("\"pool_assignment\":\"" + details + "\"");
-      detailsStatus = new DetailsStatus();
-      detailsStatus.setDocumentString(sb.toString());
+      detailsStatus.setPoolAssignment(details);
       this.documentStore.store(detailsStatus, fingerprint);
     }
   }
@@ -540,25 +468,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
           !relay.getRelayFlags().contains("BadExit");
       boolean isGuard = relay.getRelayFlags().contains("Guard");
       DetailsStatus detailsStatus = this.documentStore.retrieve(
-          DetailsStatus.class, false, fingerprint);
+          DetailsStatus.class, true, fingerprint);
       if (detailsStatus != null) {
-        double advertisedBandwidth = -1.0;
-        String detailsString = detailsStatus.getDocumentString();
-        Scanner s = new Scanner(detailsString);
-        while (s.hasNextLine()) {
-          String line = s.nextLine();
-          if (!line.startsWith("\"advertised_bandwidth\":")) {
-            continue;
-          }
-          try {
-            advertisedBandwidth = (double) Integer.parseInt(
-                line.split(":")[1].replaceAll(",", ""));
-          } catch (NumberFormatException ex) {
-            /* Handle below. */
-          }
-          break;
-        }
-        s.close();
+        double advertisedBandwidth =
+            detailsStatus.getAdvertisedBandwidth();
         if (advertisedBandwidth >= 0.0) {
           advertisedBandwidths.put(fingerprint, advertisedBandwidth);
           totalAdvertisedBandwidth += advertisedBandwidth;





More information about the tor-commits mailing list