[tor-commits] [onionoo/master] Use Gson to format JSON uptime documents.

9 May 2014

commit 7b05cc2bc7e94424ffdbccceaa6d2102c0e2aeae
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Sun Apr 20 14:51:44 2014 +0200
Use Gson to format JSON uptime documents.
---
 src/org/torproject/onionoo/GraphHistory.java       |   56 +++++
 src/org/torproject/onionoo/ResponseBuilder.java    |    4 +-
 src/org/torproject/onionoo/UptimeDocument.java     |   14 ++
 .../torproject/onionoo/UptimeDocumentWriter.java   |   67 +++---
 .../onionoo/UptimeDocumentWriterTest.java          |  253 ++++++++++++++++++++
 5 files changed, 355 insertions(+), 39 deletions(-)

diff --git a/src/org/torproject/onionoo/GraphHistory.java b/src/org/torproject/onionoo/GraphHistory.java
new file mode 100644
index 0000000..f03be58
--- /dev/null
+++ b/src/org/torproject/onionoo/GraphHistory.java
@@ -0,0 +1,56 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import java.util.List;
+
+public class GraphHistory {
+
+  private String first;
+  public void setFirst(String first) {
+    this.first = first;
+  }
+  public String getFirst() {
+    return this.first;
+  }
+
+  private String last;
+  public void setLast(String last) {
+    this.last = last;
+  }
+  public String getLast() {
+    return this.last;
+  }
+
+  private Integer interval;
+  public void setInterval(Integer interval) {
+    this.interval = interval;
+  }
+  public Integer getInterval() {
+    return this.interval;
+  }
+
+  private Double factor;
+  public void setFactor(Double factor) {
+    this.factor = factor;
+  }
+  public Double getFactor() {
+    return this.factor;
+  }
+
+  private Integer count;
+  public void setCount(Integer count) {
+    this.count = count;
+  }
+  public Integer getCount() {
+    return this.count;
+  }
+
+  private List<Integer> values;
+  public void setValues(List<Integer> values) {
+    this.values = values;
+  }
+  public List<Integer> getValues() {
+    return this.values;
+  }
+}
diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java
index c4f3f07..dbd8cc7 100644
--- a/src/org/torproject/onionoo/ResponseBuilder.java
+++ b/src/org/torproject/onionoo/ResponseBuilder.java
@@ -236,9 +236,7 @@ public class ResponseBuilder {
         UptimeDocument.class, false, fingerprint);
     if (uptimeDocument != null &&
         uptimeDocument.getDocumentString() != null) {
-      String uptimeLines = uptimeDocument.getDocumentString();
-      uptimeLines = uptimeLines.substring(0, uptimeLines.length() - 1);
-      return uptimeLines;
+      return uptimeDocument.getDocumentString();
     } else {
       return "{"fingerprint":"" + fingerprint.toUpperCase() + ""}";
     }
diff --git a/src/org/torproject/onionoo/UptimeDocument.java b/src/org/torproject/onionoo/UptimeDocument.java
index f71cb87..b2df03b 100644
--- a/src/org/torproject/onionoo/UptimeDocument.java
+++ b/src/org/torproject/onionoo/UptimeDocument.java
@@ -2,7 +2,21 @@
  * See LICENSE for licensing information */
 package org.torproject.onionoo;
+import java.util.Map;
+
 class UptimeDocument extends Document {
+  private String fingerprint;
+  public void setFingerprint(String fingerprint) {
+    this.fingerprint = fingerprint;
+  }
+
+  private Map<String, GraphHistory> uptime;
+  public void setUptime(Map<String, GraphHistory> uptime) {
+    this.uptime = uptime;
+  }
+  public Map<String, GraphHistory> getUptime() {
+    return this.uptime;
+  }
 }
diff --git a/src/org/torproject/onionoo/UptimeDocumentWriter.java b/src/org/torproject/onionoo/UptimeDocumentWriter.java
index 14c800d..8293f77 100644
--- a/src/org/torproject/onionoo/UptimeDocumentWriter.java
+++ b/src/org/torproject/onionoo/UptimeDocumentWriter.java
@@ -3,8 +3,9 @@
 package org.torproject.onionoo;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Locale;
+import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -70,9 +71,8 @@ public class UptimeDocumentWriter implements FingerprintListener,
       SortedSet<UptimeHistory> history = relay
           ? uptimeStatus.getRelayHistory()
           : uptimeStatus.getBridgeHistory();
-      UptimeDocument uptimeDocument = new UptimeDocument();
-      uptimeDocument.setDocumentString(this.formatHistoryString(relay,
-          fingerprint, history, knownStatuses));
+      UptimeDocument uptimeDocument = this.compileUptimeDocument(relay,
+          fingerprint, history, knownStatuses);
       this.documentStore.store(uptimeDocument, fingerprint);
       this.writtenDocuments++;
     }
@@ -99,31 +99,29 @@ public class UptimeDocumentWriter implements FingerprintListener,
       DateTimeHelper.TWO_DAYS,
       DateTimeHelper.TEN_DAYS };
-  private String formatHistoryString(boolean relay, String fingerprint,
-      SortedSet<UptimeHistory> history,
+  private UptimeDocument compileUptimeDocument(boolean relay,
+      String fingerprint, SortedSet<UptimeHistory> history,
       SortedSet<UptimeHistory> knownStatuses) {
-    StringBuilder sb = new StringBuilder();
-    sb.append("{"fingerprint":"" + fingerprint + """);
-    sb.append(",\n"uptime":{");
-    int graphIntervalsWritten = 0;
+    UptimeDocument uptimeDocument = new UptimeDocument();
+    uptimeDocument.setFingerprint(fingerprint);
+    Map<String, GraphHistory> uptime =
+        new LinkedHashMap<String, GraphHistory>();
     for (int graphIntervalIndex = 0; graphIntervalIndex <
         this.graphIntervals.length; graphIntervalIndex++) {
-      String timeline = this.formatTimeline(graphIntervalIndex, relay,
-          history, knownStatuses);
-      if (timeline != null) {
-        sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n"
-            + timeline);
+      String graphName = this.graphNames[graphIntervalIndex];
+      GraphHistory graphHistory = this.compileUptimeHistory(
+          graphIntervalIndex, relay, history, knownStatuses);
+      if (graphHistory != null) {
+        uptime.put(graphName, graphHistory);
       }
     }
-    sb.append("}");
-    sb.append("\n}\n");
-    return sb.toString();
+    uptimeDocument.setUptime(uptime);
+    return uptimeDocument;
   }
-  private String formatTimeline(int graphIntervalIndex, boolean relay,
-      SortedSet<UptimeHistory> history,
+  private GraphHistory compileUptimeHistory(int graphIntervalIndex,
+      boolean relay, SortedSet<UptimeHistory> history,
       SortedSet<UptimeHistory> knownStatuses) {
-    String graphName = this.graphNames[graphIntervalIndex];
     long graphInterval = this.graphIntervals[graphIntervalIndex];
     long dataPointInterval =
         this.dataPointIntervals[graphIntervalIndex];
@@ -253,18 +251,17 @@ public class UptimeDocumentWriter implements FingerprintListener,
     }
     long lastDataPointMillis = firstDataPointMillis
         + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
-    double factor = 1.0 / 999.0;
     int count = lastNonNullIndex - firstNonNullIndex + 1;
-    StringBuilder sb = new StringBuilder();
-    sb.append(""" + graphName + "":{"
-        + ""first":"" + DateTimeHelper.format(firstDataPointMillis)
-        + "","last":"" + DateTimeHelper.format(lastDataPointMillis)
-        + "","interval":" + String.valueOf(dataPointInterval
-        / DateTimeHelper.ONE_SECOND)
-        + ","factor":" + String.format(Locale.US, "%.9f", factor)
-        + ","count":" + String.valueOf(count) + ","values":[");
-    int dataPointsWritten = 0, previousNonNullIndex = -2;
+    GraphHistory graphHistory = new GraphHistory();
+    graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
+    graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
+    graphHistory.setInterval((int) (dataPointInterval
+        / DateTimeHelper.ONE_SECOND));
+    graphHistory.setFactor(1.0 / 999.0);
+    graphHistory.setCount(count);
+    int previousNonNullIndex = -2;
     boolean foundTwoAdjacentDataPoints = false;
+    List<Integer> values = new ArrayList<Integer>();
     for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
         lastNonNullIndex; dataPointIndex++) {
       double dataPoint = dataPoints.get(dataPointIndex);
@@ -274,13 +271,11 @@ public class UptimeDocumentWriter implements FingerprintListener,
         }
         previousNonNullIndex = dataPointIndex;
       }
-      sb.append((dataPointsWritten++ > 0 ? "," : "")
-          + (dataPoint < -0.5 ? "null" :
-          String.valueOf((long) (dataPoint * 999.0))));
+      values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0)));
     }
-    sb.append("]}");
+    graphHistory.setValues(values);
     if (foundTwoAdjacentDataPoints) {
-      return sb.toString();
+      return graphHistory;
     } else {
       return null;
     }
diff --git a/test/org/torproject/onionoo/UptimeDocumentWriterTest.java b/test/org/torproject/onionoo/UptimeDocumentWriterTest.java
new file mode 100644
index 0000000..5065e4d
--- /dev/null
+++ b/test/org/torproject/onionoo/UptimeDocumentWriterTest.java
@@ -0,0 +1,253 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class UptimeDocumentWriterTest {
+
+  private static final long TEST_TIME = DateTimeHelper.parse(
+      "2014-03-23 12:00:00");
+
+  private DummyTime dummyTime;
+
+  @Before
+  public void createDummyTime() {
+    this.dummyTime = new DummyTime(TEST_TIME);
+    ApplicationFactory.setTime(this.dummyTime);
+  }
+
+  private DummyDescriptorSource descriptorSource;
+
+  @Before
+  public void createDummyDescriptorSource() {
+    this.descriptorSource = new DummyDescriptorSource();
+    ApplicationFactory.setDescriptorSource(this.descriptorSource);
+  }
+
+  private DummyDocumentStore documentStore;
+
+  @Before
+  public void createDummyDocumentStore() {
+    this.documentStore = new DummyDocumentStore();
+    ApplicationFactory.setDocumentStore(this.documentStore);
+  }
+
+  @Test
+  public void testNoStatuses() {
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    writer.writeDocuments();
+    assertEquals("Without providing any data, nothing should be written "
+        + "to disk.", 0,
+        this.documentStore.getPerformedStoreOperations());
+  }
+
+  private static final String ALL_RELAYS_FINGERPRINT = null;
+
+  private static final String GABELMOO_FINGERPRINT =
+      "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281";
+
+  private void addStatusOneWeekSample(String allRelaysUptime,
+      String gabelmooUptime) {
+    UptimeStatus status = new UptimeStatus();
+    status.fromDocumentString(allRelaysUptime);
+    this.documentStore.addDocument(status, ALL_RELAYS_FINGERPRINT);
+    status = new UptimeStatus();
+    status.fromDocumentString(gabelmooUptime);
+    this.documentStore.addDocument(status, GABELMOO_FINGERPRINT);
+    this.descriptorSource.addFingerprint(DescriptorType.RELAY_CONSENSUSES,
+        GABELMOO_FINGERPRINT);
+  }
+
+  private void assertOneWeekGraph(UptimeDocument document, int graphs,
+      String first, String last, int count, List<Integer> values) {
+    this.assertGraph(document, graphs, "1_week", first, last,
+        (int) (DateTimeHelper.ONE_HOUR / DateTimeHelper.ONE_SECOND),
+        count, values);
+  }
+
+  private void assertOneMonthGraph(UptimeDocument document, int graphs,
+      String first, String last, int count, List<Integer> values) {
+    this.assertGraph(document, graphs, "1_month", first, last,
+        (int) (DateTimeHelper.FOUR_HOURS / DateTimeHelper.ONE_SECOND),
+        count, values);
+  }
+
+  private void assertGraph(UptimeDocument document, int graphs,
+      String graphName, String first, String last, int interval,
+      int count, List<Integer> values) {
+    assertEquals("Should contain exactly " + graphs + " graphs.", graphs,
+        document.getUptime().size());
+    assertTrue("Should contain a graph for " + graphName + ".",
+        document.getUptime().containsKey(graphName));
+    GraphHistory history = document.getUptime().get(graphName);
+    assertEquals("First data point should be " + first + ".", first,
+        history.getFirst());
+    assertEquals("Last data point should be " + last + ".", last,
+        history.getLast());
+    assertEquals("Interval should be " + interval + " seconds.", interval,
+        (int) history.getInterval());
+    assertEquals("Factor should be 1.0 / 999.0.", 1.0 / 999.0,
+        (double) history.getFactor(), 0.01);
+    assertEquals("There should be one data point per hour.", count,
+        (int) history.getCount());
+    assertEquals("Count should be the same as the number of values.",
+        count, history.getValues().size());
+    if (values == null) {
+      for (int value : history.getValues()) {
+        assertEquals("All values should be 999.", 999, value);
+      }
+    } else {
+      assertEquals("Values are not as expected.", values,
+          history.getValues());
+    }
+  }
+
+  @Test
+  public void testOneHourUptime() {
+    this.addStatusOneWeekSample("r 2014-03-23-11 1\n",
+        "r 2014-03-23-11 1\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    assertEquals("Should not contain any graph.", 0,
+        document.getUptime().size());
+  }
+
+  @Test
+  public void testTwoHoursUptime() {
+    this.addStatusOneWeekSample("r 2014-03-23-10 2\n",
+        "r 2014-03-23-10 2\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneWeekGraph(document, 1, "2014-03-23 10:30:00",
+        "2014-03-23 11:30:00", 2, null);
+  }
+
+  @Test
+  public void testTwoHoursUptimeSeparatedByNull() {
+    this.addStatusOneWeekSample("r 2014-03-23-09 1\nr 2014-03-23-11 1\n",
+        "r 2014-03-23-09 1\nr 2014-03-23-11 1\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    assertEquals("Should not contain any graph.", 0,
+        document.getUptime().size());
+  }
+
+  @Test
+  public void testTwoHoursUptimeSeparatedByZero() {
+    this.addStatusOneWeekSample("r 2014-03-23-09 3\n",
+        "r 2014-03-23-09 1\nr 2014-03-23-11 1\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneWeekGraph(document, 1, "2014-03-23 09:30:00",
+        "2014-03-23 11:30:00", 3,
+        Arrays.asList(new Integer[] { 999, 0, 999 }));
+  }
+
+  @Test
+  public void testTwoHoursUptimeThenDowntime() {
+    this.addStatusOneWeekSample("r 2014-03-23-09 3\n",
+        "r 2014-03-23-09 2\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneWeekGraph(document, 1, "2014-03-23 09:30:00",
+        "2014-03-23 11:30:00", 3,
+        Arrays.asList(new Integer[] { 999, 999, 0 }));
+  }
+
+  @Test
+  public void testOneWeekUptime() {
+    this.addStatusOneWeekSample("r 2014-03-16-12 168\n",
+        "r 2014-03-16-12 168\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneWeekGraph(document, 1, "2014-03-16 12:30:00",
+        "2014-03-23 11:30:00", 168, null);
+  }
+
+  @Test
+  public void testOneWeekOneHourUptime() {
+    this.addStatusOneWeekSample("r 2014-03-16-11 169\n",
+        "r 2014-03-16-11 169\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneWeekGraph(document, 2, "2014-03-16 12:30:00",
+        "2014-03-23 11:30:00", 168, null);
+    this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00",
+        "2014-03-23 10:00:00", 43, null);
+  }
+
+  @Test
+  public void testOneMonthPartialIntervalOnline() {
+    this.addStatusOneWeekSample("r 2014-03-16-08 8\n",
+        "r 2014-03-16-11 5\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00",
+        "2014-03-16 14:00:00", 2, null);
+  }
+
+  @Test
+  public void testOneMonthPartialIntervalOnOff() {
+    this.addStatusOneWeekSample("r 2014-03-16-08 8\n",
+        "r 2014-03-16-10 1\nr 2014-03-16-12 1\n");
+    UptimeDocumentWriter writer = new UptimeDocumentWriter();
+    ApplicationFactory.getDescriptorSource().readDescriptors();
+    writer.writeDocuments();
+    assertEquals("Should write exactly one document.", 1,
+        this.documentStore.getPerformedStoreOperations());
+    UptimeDocument document = this.documentStore.getDocument(
+        UptimeDocument.class, GABELMOO_FINGERPRINT);
+    this.assertOneMonthGraph(document, 2, "2014-03-16 10:00:00",
+        "2014-03-16 14:00:00", 2,
+        Arrays.asList(new Integer[] { 499, 249 }));
+  }
+}
+

    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

[tor-commits] [onionoo/master] Use Gson to format JSON uptime documents.