[tor-commits] [onionoo/master] Fix invalid first_seen timestamps in existing data.

[karsten at torproject.org]

commit 95b2cf941bdc78d273e1d17ca73a737b22e73fc8
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon Jan 23 12:29:41 2017 +0100

    Fix invalid first_seen timestamps in existing data.
    
    In a previous commit we fixed an issue with the first_seen timestamp
    of bridges when parsing new bridge descriptors.  We still need to fix
    invalid first_seen timestamps in existing data.
    
    There are two different cases of invalid first_seen timestamps in
    existing data:
    
    1) first_seen is 1970-01-01 00:00:00, last_seen is 2014-12-09 or
    later.  In these cases we parsed a server descriptor before first
    seeing the bridge in a status in a subsequent execution.  We need to
    special-case the previously known first_seen timestamp 1970-01-01
    00:00:00 (value 0) and not consider it earlier than an actual
    timestamp.
    
    2) Both first_seen and last_seen are 1970-01-01 00:00:00.  We have
    never seen these bridges in a status.  We can safely discard these
    entries while reading them from disk to memory.
    
    With this fix it will be possible to repair the status data of an
    existing Onionoo instance by re-importing archived bridge network
    statuses from 2014-12 and onwards.
    
    Fixes the remainder of #20994.
---
 src/main/java/org/torproject/onionoo/docs/NodeStatus.java     |  4 ++++
 .../torproject/onionoo/updater/NodeDetailsStatusUpdater.java  |  3 ++-
 src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java | 11 +++++++++++
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
index 7e4e438..6a71fb6 100644
--- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
@@ -463,6 +463,10 @@ public class NodeStatus extends Document {
         log.error("Parse exception while parsing node status "
             + "line '" + documentString + "'.  Skipping.");
         return null;
+      } else if (lastSeenMillis == 0L) {
+        log.debug("Skipping node status with fingerprint {} that has so far "
+            + "never been seen in a network status.", fingerprint);
+        return null;
       }
       nodeStatus.setLastSeenMillis(lastSeenMillis);
       int orPort = Integer.parseInt(parts[6]);
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
index b6dcdf7..802fb9d 100644
--- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -485,7 +485,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
               nodeStatus.getRecommendedVersion());
         }
         if (nodeStatus.getFirstSeenMillis()
-            < updatedNodeStatus.getFirstSeenMillis()) {
+            < updatedNodeStatus.getFirstSeenMillis()
+            && nodeStatus.getFirstSeenMillis() > 0L) {
           updatedNodeStatus.setFirstSeenMillis(
               nodeStatus.getFirstSeenMillis());
         }
diff --git a/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java
index 284edd9..0b4344d 100644
--- a/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java
@@ -4,6 +4,7 @@
 package org.torproject.onionoo.docs;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 import org.junit.Test;
 
@@ -112,4 +113,14 @@ public class NodeStatusTest {
     assertFamiliesCanBeDeSerialized(
         new String[] { NICK }, new String[] {}, new String[] {});
   }
+
+  private static final String KI_LAST_SEEN_1970_NODE_STATUS =
+      "b\tnull\t0016C76AC19753A65F8E122046079CD09D355BA1\tnull;;\t1970-01-01\t"
+      + "00:00:00\t0\t0\t\t-1\t??\t\t-1\tnull\tnull\t1970-01-01\t00:00:00\t"
+      + "null\tnull\tnull\t\tnull\t::";
+
+  @Test
+  public void testLastSeen19700101Skipped() {
+    assertNull(NodeStatus.fromString(KI_LAST_SEEN_1970_NODE_STATUS));
+  }
 }