commit 677c5a3af88f8b7862c47d2148df6f0f9523062e Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun Apr 20 19:02:24 2014 +0200 Use Gson to format JSON details status files. --- src/org/torproject/onionoo/DetailsStatus.java | 130 +++++++++++++- src/org/torproject/onionoo/DocumentStore.java | 29 +++- .../onionoo/NodeDetailsStatusUpdater.java | 179 +++++--------------- 3 files changed, 201 insertions(+), 137 deletions(-) diff --git a/src/org/torproject/onionoo/DetailsStatus.java b/src/org/torproject/onionoo/DetailsStatus.java index 1ebb100..1a497e4 100644 --- a/src/org/torproject/onionoo/DetailsStatus.java +++ b/src/org/torproject/onionoo/DetailsStatus.java @@ -1,7 +1,133 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringEscapeUtils; + class DetailsStatus extends Document { -} + /* We must ensure that details files only contain ASCII characters + * and no UTF-8 characters. While UTF-8 characters are perfectly + * valid in JSON, this would break compatibility with existing files + * pretty badly. We do this by escaping non-ASCII characters, e.g., + * \u00F2. Gson won't treat this as UTF-8, but will think that we want + * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing + * we'll have to do is to change back the '\\' that Gson writes for the + * '\'. */ + private static String escapeJSON(String s) { + return s == null ? null : + StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); + } + private static String unescapeJSON(String s) { + return s == null ? null : + StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); + } + + private String desc_published; + public void setDescPublished(String descPublished) { + this.desc_published = descPublished; + } + public String getDescPublished() { + return this.desc_published; + } + + private String last_restarted; + public void setLastRestarted(String lastRestarted) { + this.last_restarted = lastRestarted; + } + public String getLastRestarted() { + return this.last_restarted; + } + + private Integer bandwidth_rate; + public void setBandwidthRate(Integer bandwidthRate) { + this.bandwidth_rate = bandwidthRate; + } + public Integer getBandwidthRate() { + return this.bandwidth_rate; + } + + private Integer bandwidth_burst; + public void setBandwidthBurst(Integer bandwidthBurst) { + this.bandwidth_burst = bandwidthBurst; + } + public Integer getBandwidthBurst() { + return this.bandwidth_burst; + } + + private Integer observed_bandwidth; + public void setObservedBandwidth(Integer observedBandwidth) { + this.observed_bandwidth = observedBandwidth; + } + public Integer getObservedBandwidth() { + return this.observed_bandwidth; + } + + private Integer advertised_bandwidth; + public void setAdvertisedBandwidth(Integer advertisedBandwidth) { + this.advertised_bandwidth = advertisedBandwidth; + } + public Integer getAdvertisedBandwidth() { + return this.advertised_bandwidth; + } + + private List<String> exit_policy; + public void setExitPolicy(List<String> exitPolicy) { + this.exit_policy = exitPolicy; + } + public List<String> getExitPolicy() { + return this.exit_policy; + } + + private String contact; + public void setContact(String contact) { + this.contact = escapeJSON(contact); + } + public String getContact() { + return unescapeJSON(this.contact); + } + + private String platform; + public void setPlatform(String platform) { + this.platform = escapeJSON(platform); + } + public String getPlatform() { + return unescapeJSON(this.platform); + } + + private List<String> family; + public void setFamily(List<String> family) { + this.family = family; + } + public List<String> getFamily() { + return this.family; + } + + private Map<String, List<String>> exit_policy_v6_summary; + public void setExitPolicyV6Summary( + Map<String, List<String>> exitPolicyV6Summary) { + this.exit_policy_v6_summary = exitPolicyV6Summary; + } + public Map<String, List<String>> getExitPolicyV6Summary() { + return this.exit_policy_v6_summary; + } + + private Boolean hibernating; + public void setHibernating(Boolean hibernating) { + this.hibernating = hibernating; + } + public Boolean getHibernating() { + return this.hibernating; + } + + private String pool_assignment; + public void setPoolAssignment(String poolAssignment) { + this.pool_assignment = poolAssignment; + } + public String getPoolAssignment() { + return this.pool_assignment; + } +} diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index e98fb4e..d80999e 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -21,6 +21,7 @@ import java.util.TreeMap; import java.util.TreeSet; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; // TODO For later migration from disk to database, do the following: @@ -187,13 +188,34 @@ public class DocumentStore { String documentString; if (document.getDocumentString() != null) { documentString = document.getDocumentString(); - } else if (document instanceof DetailsDocument || - document instanceof BandwidthDocument || + } else if (document instanceof BandwidthDocument || document instanceof WeightsDocument || document instanceof ClientsDocument || document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(document); + } else if (document instanceof DetailsStatus || + document instanceof DetailsDocument) { + /* Don't escape HTML characters, like < and >, contained in + * strings. */ + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + /* We must ensure that details files only contain ASCII characters + * and no UTF-8 characters. While UTF-8 characters are perfectly + * valid in JSON, this would break compatibility with existing files + * pretty badly. We already make sure that all strings in details + * objects are escaped JSON, e.g., \u00F2. When Gson serlializes + * this string, it escapes the \ to \\, hence writes \\u00F2. We + * need to undo this and change \\u00F2 back to \u00F2. */ + documentString = gson.toJson(document).replaceAll("\\\\\\\\u", + "\\\\u"); + /* Existing details statuses don't contain opening and closing curly + * brackets, so we should remove them from new details statuses, + * too. */ + if (document instanceof DetailsStatus) { + documentString = documentString.substring( + documentString.indexOf("{") + 1, + documentString.lastIndexOf("}")); + } } else if (document instanceof BandwidthStatus || document instanceof WeightsStatus || document instanceof ClientsStatus || @@ -380,6 +402,9 @@ public class DocumentStore { documentType.equals(ClientsStatus.class) || documentType.equals(UptimeStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); + } else if (documentType.equals(DetailsStatus.class)) { + return this.retrieveParsedDocumentFile(documentType, "{" + + documentString + "}"); } else { System.err.println("Parsing is not supported for type " + documentType.getName() + "."); diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index 4cfdd33..030a8ec 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -5,15 +5,14 @@ package org.torproject.onionoo; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; -import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; -import org.apache.commons.lang.StringEscapeUtils; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; @@ -291,27 +290,16 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, ServerDescriptor descriptor) { String fingerprint = descriptor.getFingerprint(); DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); + DetailsStatus.class, true, fingerprint); String publishedDateTime = DateTimeHelper.format(descriptor.getPublishedMillis()); - if (detailsStatus != null) { - String detailsString = detailsStatus.getDocumentString(); - String descPublishedLine = "\"desc_published\":\"" - + publishedDateTime + "\","; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith("\"desc_published\":\"")) { - if (descPublishedLine.compareTo(line) < 0) { - return; - } else { - break; - } - } - } - s.close(); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (detailsStatus.getDescPublished() != null && + publishedDateTime.compareTo( + detailsStatus.getDescPublished()) < 0) { + return; } - StringBuilder sb = new StringBuilder(); String lastRestartedString = DateTimeHelper.format( descriptor.getPublishedMillis() - descriptor.getUptime() * DateTimeHelper.ONE_SECOND); @@ -320,55 +308,31 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, int observedBandwidth = descriptor.getBandwidthObserved(); int advertisedBandwidth = Math.min(bandwidthRate, Math.min(bandwidthBurst, observedBandwidth)); - sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" - + "\"last_restarted\":\"" + lastRestartedString + "\",\n" - + "\"bandwidth_rate\":" + bandwidthRate + ",\n" - + "\"bandwidth_burst\":" + bandwidthBurst + ",\n" - + "\"observed_bandwidth\":" + observedBandwidth + ",\n" - + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" - + "\"exit_policy\":["); - int written = 0; - for (String exitPolicyLine : descriptor.getExitPolicyLines()) { - sb.append((written++ > 0 ? "," : "") + "\n \"" + exitPolicyLine - + "\""); - } - sb.append("\n]"); - if (descriptor.getContact() != null) { - sb.append(",\n\"contact\":\"" - + escapeJSON(descriptor.getContact()) + "\""); - } - if (descriptor.getPlatform() != null) { - sb.append(",\n\"platform\":\"" - + escapeJSON(descriptor.getPlatform()) + "\""); - } - if (descriptor.getFamilyEntries() != null) { - sb.append(",\n\"family\":["); - written = 0; - for (String familyEntry : descriptor.getFamilyEntries()) { - sb.append((written++ > 0 ? "," : "") + "\n \"" + familyEntry - + "\""); - } - sb.append("\n]"); - } + detailsStatus.setDescPublished(publishedDateTime); + detailsStatus.setLastRestarted(lastRestartedString); + detailsStatus.setBandwidthRate(bandwidthRate); + detailsStatus.setBandwidthBurst(bandwidthBurst); + detailsStatus.setObservedBandwidth(observedBandwidth); + detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); + detailsStatus.setExitPolicy(descriptor.getExitPolicyLines()); + detailsStatus.setContact(descriptor.getContact()); + detailsStatus.setPlatform(descriptor.getPlatform()); + detailsStatus.setFamily(descriptor.getFamilyEntries()); if (descriptor.getIpv6DefaultPolicy() != null && (descriptor.getIpv6DefaultPolicy().equals("accept") || descriptor.getIpv6DefaultPolicy().equals("reject")) && descriptor.getIpv6PortList() != null) { - sb.append(",\n\"exit_policy_v6_summary\":{\"" - + descriptor.getIpv6DefaultPolicy() + "\":["); - int portsWritten = 0; - for (String portOrPortRange : - descriptor.getIpv6PortList().split(",")) { - sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange - + "\""); - } - sb.append("]}"); + Map<String, List<String>> exitPolicyV6Summary = + new HashMap<String, List<String>>(); + List<String> portsOrPortRanges = Arrays.asList( + descriptor.getIpv6PortList().split(",")); + exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(), + portsOrPortRanges); + detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); } if (descriptor.isHibernating()) { - sb.append(",\n\"hibernating\":true"); + detailsStatus.setHibernating(true); } - detailsStatus = new DetailsStatus(); - detailsStatus.setDocumentString(sb.toString()); this.documentStore.store(detailsStatus, fingerprint); if (descriptor.getContact() != null) { this.contacts.put(fingerprint, descriptor.getContact()); @@ -379,79 +343,43 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, ServerDescriptor descriptor) { String fingerprint = descriptor.getFingerprint(); DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); + DetailsStatus.class, true, fingerprint); String publishedDateTime = DateTimeHelper.format(descriptor.getPublishedMillis()); - String poolAssignmentLine = null; - if (detailsStatus != null) { - String detailsString = detailsStatus.getDocumentString(); - String descPublishedLine = "\"desc_published\":\"" - + publishedDateTime + "\","; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith("\"pool_assignment\":")) { - poolAssignmentLine = line; - } else if (line.startsWith("\"desc_published\":") && - descPublishedLine.compareTo(line) < 0) { - return; - } - } - s.close(); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (detailsStatus.getDescPublished() != null && + publishedDateTime.compareTo( + detailsStatus.getDescPublished()) < 0) { + return; } - StringBuilder sb = new StringBuilder(); String lastRestartedString = DateTimeHelper.format( descriptor.getPublishedMillis() - descriptor.getUptime() * DateTimeHelper.ONE_SECOND); int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), Math.min(descriptor.getBandwidthBurst(), descriptor.getBandwidthObserved())); - sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" - + "\"last_restarted\":\"" + lastRestartedString + "\",\n" - + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" - + "\"platform\":\"" + escapeJSON(descriptor.getPlatform()) - + "\""); - if (poolAssignmentLine != null) { - sb.append(",\n" + poolAssignmentLine); - } - detailsStatus = new DetailsStatus(); - detailsStatus.setDocumentString(sb.toString()); + detailsStatus.setDescPublished(publishedDateTime); + detailsStatus.setLastRestarted(lastRestartedString); + detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); + detailsStatus.setPlatform(descriptor.getPlatform()); this.documentStore.store(detailsStatus, fingerprint); } - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); - } - private void processBridgePoolAssignment( BridgePoolAssignment bridgePoolAssignment) { for (Map.Entry<String, String> e : bridgePoolAssignment.getEntries().entrySet()) { String fingerprint = e.getKey(); String details = e.getValue(); - StringBuilder sb = new StringBuilder(); DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null) { - String detailsString = detailsStatus.getDocumentString(); - Scanner s = new Scanner(detailsString); - int linesWritten = 0; - boolean endsWithComma = false; - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"pool_assignment\":")) { - sb.append((linesWritten++ > 0 ? "\n" : "") + line); - endsWithComma = line.endsWith(","); - } - } - s.close(); - if (sb.length() > 0) { - sb.append((endsWithComma ? "" : ",") + "\n"); - } + DetailsStatus.class, true, fingerprint); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (details.equals(detailsStatus.getPoolAssignment())) { + return; } - sb.append("\"pool_assignment\":\"" + details + "\""); - detailsStatus = new DetailsStatus(); - detailsStatus.setDocumentString(sb.toString()); + detailsStatus.setPoolAssignment(details); this.documentStore.store(detailsStatus, fingerprint); } } @@ -540,25 +468,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, !relay.getRelayFlags().contains("BadExit"); boolean isGuard = relay.getRelayFlags().contains("Guard"); DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); + DetailsStatus.class, true, fingerprint); if (detailsStatus != null) { - double advertisedBandwidth = -1.0; - String detailsString = detailsStatus.getDocumentString(); - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"advertised_bandwidth\":")) { - continue; - } - try { - advertisedBandwidth = (double) Integer.parseInt( - line.split(":")[1].replaceAll(",", "")); - } catch (NumberFormatException ex) { - /* Handle below. */ - } - break; - } - s.close(); + double advertisedBandwidth = + detailsStatus.getAdvertisedBandwidth(); if (advertisedBandwidth >= 0.0) { advertisedBandwidths.put(fingerprint, advertisedBandwidth); totalAdvertisedBandwidth += advertisedBandwidth;