commit 95b2cf941bdc78d273e1d17ca73a737b22e73fc8 Author: Karsten Loesing karsten.loesing@gmx.net Date: Mon Jan 23 12:29:41 2017 +0100

Fix invalid first_seen timestamps in existing data.

In a previous commit we fixed an issue with the first_seen timestamp of bridges when parsing new bridge descriptors. We still need to fix invalid first_seen timestamps in existing data.

There are two different cases of invalid first_seen timestamps in existing data:

1) first_seen is 1970-01-01 00:00:00, last_seen is 2014-12-09 or later. In these cases we parsed a server descriptor before first seeing the bridge in a status in a subsequent execution. We need to special-case the previously known first_seen timestamp 1970-01-01 00:00:00 (value 0) and not consider it earlier than an actual timestamp.

2) Both first_seen and last_seen are 1970-01-01 00:00:00. We have never seen these bridges in a status. We can safely discard these entries while reading them from disk to memory.

With this fix it will be possible to repair the status data of an existing Onionoo instance by re-importing archived bridge network statuses from 2014-12 and onwards.

Fixes the remainder of #20994. --- src/main/java/org/torproject/onionoo/docs/NodeStatus.java | 4 ++++ .../torproject/onionoo/updater/NodeDetailsStatusUpdater.java | 3 ++- src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java | 11 +++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 7e4e438..6a71fb6 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -463,6 +463,10 @@ public class NodeStatus extends Document { log.error("Parse exception while parsing node status " + "line '" + documentString + "'. Skipping."); return null; + } else if (lastSeenMillis == 0L) { + log.debug("Skipping node status with fingerprint {} that has so far " + + "never been seen in a network status.", fingerprint); + return null; } nodeStatus.setLastSeenMillis(lastSeenMillis); int orPort = Integer.parseInt(parts[6]); diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index b6dcdf7..802fb9d 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -485,7 +485,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.getRecommendedVersion()); } if (nodeStatus.getFirstSeenMillis() - < updatedNodeStatus.getFirstSeenMillis()) { + < updatedNodeStatus.getFirstSeenMillis() + && nodeStatus.getFirstSeenMillis() > 0L) { updatedNodeStatus.setFirstSeenMillis( nodeStatus.getFirstSeenMillis()); } diff --git a/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java index 284edd9..0b4344d 100644 --- a/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/NodeStatusTest.java @@ -4,6 +4,7 @@ package org.torproject.onionoo.docs;

import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull;

import org.junit.Test;

@@ -112,4 +113,14 @@ public class NodeStatusTest { assertFamiliesCanBeDeSerialized( new String[] { NICK }, new String[] {}, new String[] {}); } + + private static final String KI_LAST_SEEN_1970_NODE_STATUS = + "b\tnull\t0016C76AC19753A65F8E122046079CD09D355BA1\tnull;;\t1970-01-01\t" + + "00:00:00\t0\t0\t\t-1\t??\t\t-1\tnull\tnull\t1970-01-01\t00:00:00\t" + + "null\tnull\tnull\t\tnull\t::"; + + @Test + public void testLastSeen19700101Skipped() { + assertNull(NodeStatus.fromString(KI_LAST_SEEN_1970_NODE_STATUS)); + } }