[tor-commits] [onionoo/master] Add version parameter to filter for Tor version.

karsten at torproject.org karsten at torproject.org
Tue Aug 29 10:00:19 UTC 2017


commit 249c8549c0536e7c8e71b5f7f500f81b2f203e6d
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon Aug 7 23:06:36 2017 +0200

    Add version parameter to filter for Tor version.
    
    Implements #21427.
    
    Requires minor protocol update.
---
 .../org/torproject/onionoo/docs/DocumentStore.java |  3 +-
 .../org/torproject/onionoo/docs/NodeStatus.java    | 14 +++++
 .../torproject/onionoo/docs/SummaryDocument.java   | 13 +++-
 .../org/torproject/onionoo/server/NodeIndex.java   | 10 +++
 .../org/torproject/onionoo/server/NodeIndexer.java |  8 +++
 .../torproject/onionoo/server/RequestHandler.java  | 26 ++++++++
 .../torproject/onionoo/server/ResourceServlet.java | 24 +++++++-
 .../onionoo/updater/NodeDetailsStatusUpdater.java  |  2 +
 .../onionoo/writer/SummaryDocumentWriter.java      |  3 +-
 .../onionoo/docs/SummaryDocumentTest.java          |  2 +-
 .../onionoo/server/ResourceServletTest.java        | 72 +++++++++++++++++++---
 .../server/SummaryDocumentComparatorTest.java      |  2 +-
 12 files changed, 166 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
index 56ca406..7819d0f 100644
--- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java
@@ -435,13 +435,14 @@ public class DocumentStore {
     }
     SortedSet<String> relayFlags = new TreeSet<>();
     SortedSet<String> family = null;
+    String version = null;
     long lastSeenMillis = -1L;
     long consensusWeight = -1L;
     long firstSeenMillis = -1L;
     SummaryDocument summaryDocument = new SummaryDocument(isRelay,
         nickname, fingerprint, addresses, lastSeenMillis, running,
         relayFlags, consensusWeight, countryCode, firstSeenMillis,
-        asNumber, contact, family, family);
+        asNumber, contact, family, family, version);
     return summaryDocument;
   }
 
diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
index 6a71fb6..759d83e 100644
--- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java
@@ -321,6 +321,16 @@ public class NodeStatus extends Document {
     return this.recommendedVersion;
   }
 
+  private String version;
+
+  public void setVersion(String version) {
+    this.version = version.substring(version.lastIndexOf(" ") + 1);
+  }
+
+  public String getVersion() {
+    return this.version;
+  }
+
   /* From exit lists: */
 
   private SortedSet<String> exitAddresses;
@@ -547,6 +557,9 @@ public class NodeStatus extends Document {
         extendedFamily.addAll(indirectFamily);
         nodeStatus.setExtendedFamily(extendedFamily);
       }
+      if (parts.length >= 24 && !parts[23].isEmpty()) {
+        nodeStatus.setVersion(parts[23]);
+      }
       return nodeStatus;
     } catch (NumberFormatException e) {
       log.error("Number format exception while parsing node "
@@ -610,6 +623,7 @@ public class NodeStatus extends Document {
     sb.append("\t" + StringUtils.join(this.getAllegedFamily(), ";") + ":"
         + StringUtils.join(this.getEffectiveFamily(), ";") + ":"
         + StringUtils.join(this.getIndirectFamily(), ";"));
+    sb.append("\t" + (this.getVersion() != null ? this.getVersion() : ""));
     return sb.toString();
   }
 }
diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
index 3ca4960..527f91d 100644
--- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java
@@ -269,6 +269,16 @@ public class SummaryDocument extends Document {
     return this.stringArrayToSortedSet(this.ef);
   }
 
+  private String v;
+
+  public void setVersion(String version) {
+    this.v = version;
+  }
+
+  public String getVersion() {
+    return this.v;
+  }
+
   /* The familyFingerprints parameter can go away after September 8, 2015.
    * See above. */
   /** Instantiates a summary document with all given properties. */
@@ -277,7 +287,7 @@ public class SummaryDocument extends Document {
       boolean running, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, long firstSeenMillis, String asNumber,
       String contact, SortedSet<String> familyFingerprints,
-      SortedSet<String> effectiveFamily) {
+      SortedSet<String> effectiveFamily, String version) {
     this.setRelay(isRelay);
     this.setNickname(nickname);
     this.setFingerprint(fingerprint);
@@ -292,6 +302,7 @@ public class SummaryDocument extends Document {
     this.setContact(contact);
     this.setFamilyFingerprints(familyFingerprints);
     this.setEffectiveFamily(effectiveFamily);
+    this.setVersion(version);
   }
 }
 
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
index 439d302..dd4bd4d 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java
@@ -169,5 +169,15 @@ class NodeIndex {
   public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() {
     return bridgesByLastSeenDays;
   }
+
+  private Map<String, Set<String>> relaysByVersion;
+
+  public void setRelaysByVersion(Map<String, Set<String>> relaysByVersion) {
+    this.relaysByVersion = relaysByVersion;
+  }
+
+  public Map<String, Set<String>> getRelaysByVersion() {
+    return this.relaysByVersion;
+  }
 }
 
diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
index f6b9510..7ed9ea6 100644
--- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java
@@ -150,6 +150,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     Map<String, Set<String>> newBridgesByFlag = new HashMap<>();
     Map<String, Set<String>> newRelaysByContact = new HashMap<>();
     Map<String, Set<String>> newRelaysByFamily = new HashMap<>();
+    Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
     SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>();
     SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>();
     SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>();
@@ -246,6 +247,12 @@ public class NodeIndexer implements ServletContextListener, Runnable {
       }
       newRelaysByContact.get(contact).add(fingerprint);
       newRelaysByContact.get(contact).add(hashedFingerprint);
+      String version = entry.getVersion();
+      if (!newRelaysByVersion.containsKey(version)) {
+        newRelaysByVersion.put(version, new HashSet<String>());
+      }
+      newRelaysByVersion.get(version).add(fingerprint);
+      newRelaysByVersion.get(version).add(hashedFingerprint);
     }
     /* This loop can go away once all Onionoo services had their hourly
      * updater write effective families to summary documents at least
@@ -319,6 +326,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays);
     newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis);
     newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis);
+    newNodeIndex.setRelaysByVersion(newRelaysByVersion);
     synchronized (this) {
       this.lastIndexed = updateStatusMillis;
       this.latestNodeIndex = newNodeIndex;
diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
index 98ece58..1e08c24 100644
--- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java
+++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java
@@ -7,6 +7,9 @@ import org.torproject.onionoo.docs.DocumentStore;
 import org.torproject.onionoo.docs.DocumentStoreFactory;
 import org.torproject.onionoo.docs.SummaryDocument;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -91,6 +94,12 @@ public class RequestHandler {
     System.arraycopy(contact, 0, this.contact, 0, contact.length);
   }
 
+  private String version;
+
+  public void setVersion(String version) {
+    this.version = version;
+  }
+
   private String[] order;
 
   public void setOrder(String[] order) {
@@ -158,6 +167,7 @@ public class RequestHandler {
     this.filterNodesByLastSeenDays();
     this.filterByContact();
     this.filterByFamily();
+    this.filterByVersion();
     this.order();
     this.offset();
     this.limit();
@@ -514,6 +524,22 @@ public class RequestHandler {
     this.filteredBridges.clear();
   }
 
+  private void filterByVersion() {
+    if (null == this.version) {
+      /* Not filtering by version. */
+      return;
+    }
+    Set<String> keepRelays = new HashSet<>();
+    for (Map.Entry<String, Set<String>> e
+        : this.nodeIndex.getRelaysByVersion().entrySet()) {
+      if (e.getKey().startsWith(this.version)) {
+        keepRelays.addAll(e.getValue());
+      }
+    }
+    this.filteredRelays.keySet().retainAll(keepRelays);
+    this.filteredBridges.clear();
+  }
+
   private void order() {
     List<SummaryDocument> uniqueRelays = new ArrayList<>();
     List<SummaryDocument> uniqueBridges = new ArrayList<>();
diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
index 869574c..164b770 100644
--- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
+++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java
@@ -3,6 +3,8 @@
 
 package org.torproject.onionoo.server;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
@@ -66,7 +68,7 @@ public class ResourceServlet extends HttpServlet {
   private static Set<String> knownParameters = new HashSet<>(
       Arrays.asList(("type,running,search,lookup,fingerprint,country,as,"
           + "flag,first_seen_days,last_seen_days,contact,order,limit,"
-          + "offset,fields,family").split(",")));
+          + "offset,fields,family,version").split(",")));
 
   private static Set<String> illegalSearchQualifiers =
       new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -275,6 +277,15 @@ public class ResourceServlet extends HttpServlet {
       }
       rh.setContact(contactParts);
     }
+    if (parameterMap.containsKey("version")) {
+      String versionParameter = this.parseVersionParameter(
+          parameterMap.get("version"));
+      if (null == versionParameter) {
+        response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+        return;
+      }
+      rh.setVersion(versionParameter);
+    }
     if (parameterMap.containsKey("order")) {
       String[] order = this.parseOrderParameter(parameterMap.get("order"));
       if (order == null) {
@@ -532,5 +543,16 @@ public class ResourceServlet extends HttpServlet {
     }
     return parameter.toLowerCase().split(",");
   }
+
+  private static Pattern versionParameterPattern =
+      Pattern.compile("^[0-9a-zA-Z\\.-]+$");
+
+  private String parseVersionParameter(String parameter) {
+    if (!versionParameterPattern.matcher(parameter).matches()) {
+      /* Version contains illegal character(s). */
+      return null;
+    }
+    return parameter;
+  }
 }
 
diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
index cc50f4b..9fcff35 100644
--- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -283,6 +283,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
         nodeStatus.setRecommendedVersion((recommendedVersions == null
             || entry.getVersion() == null) ? null :
             recommendedVersions.contains(entry.getVersion()));
+        nodeStatus.setVersion(entry.getVersion());
       }
       if (entry.getUnmeasured()) {
         if (!this.lastSeenUnmeasured.containsKey(fingerprint)
@@ -484,6 +485,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
           updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber());
           updatedNodeStatus.setRecommendedVersion(
               nodeStatus.getRecommendedVersion());
+          updatedNodeStatus.setVersion(nodeStatus.getVersion());
         }
         if (nodeStatus.getFirstSeenMillis()
             < updatedNodeStatus.getFirstSeenMillis()
diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
index b2f678d..4364aad 100644
--- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -91,10 +91,11 @@ public class SummaryDocumentWriter implements DocumentWriter {
       SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily();
       SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
       String nickname = nodeStatus.getNickname();
+      String version = nodeStatus.getVersion();
       SummaryDocument summaryDocument = new SummaryDocument(isRelay,
           nickname, fingerprint, addresses, lastSeenMillis, running,
           relayFlags, consensusWeight, countryCode, firstSeenMillis,
-          asNumber, contact, declaredFamily, effectiveFamily);
+          asNumber, contact, declaredFamily, effectiveFamily, version);
       if (this.documentStore.store(summaryDocument, fingerprint)) {
         this.writtenDocuments++;
       }
diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
index 413ca83..9d7625a 100644
--- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java
@@ -26,7 +26,7 @@ public class SummaryDocumentTest {
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
-        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
+        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
   }
 
   @Test()
diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
index 04c5d7d..4ebf0d2 100644
--- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
+++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java
@@ -142,7 +142,8 @@ public class ResourceServletTest {
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
-        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
+        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
+        "0.2.3.25");
     org.torproject.onionoo.docs.SummaryDocument relayFerrari458 =
         new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458",
         "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList(
@@ -154,7 +155,7 @@ public class ResourceServletTest {
         new TreeSet<String>(Arrays.asList(new String[] {
             "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
         new TreeSet<>(Arrays.asList(new String[] {
-            "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })));
+            "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), "0.2.3.24-rc-dev");
     this.relays = new TreeMap<>();
     this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A",
         relayTorkaZ);
@@ -170,7 +171,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
         "1024D/51E2A1C7 steven j. murdoch "
         + "<tor+steven.murdoch at cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
-        new TreeSet<String>(), new TreeSet<String>());
+        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25");
     this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
         relayTimMayTribute);
     this.bridges = new TreeMap<>();
@@ -181,7 +182,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-21 18:07:03"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
-        null, null);
+        null, null, null);
     this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
         bridgeec2bridgercc7f31fe);
     org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -191,7 +192,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-20 17:37:04"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
-        null, null);
+        null, null, null);
     this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
         bridgeUnnamed);
     org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -202,7 +203,7 @@ public class ResourceServletTest {
         new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), -1L, null,
         DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
-        null);
+        null, null);
     this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
         bridgegummy);
   }
@@ -279,7 +280,8 @@ public class ResourceServletTest {
       int expectedRelaysNumber, String[] expectedRelaysNicknames,
       int expectedBridgesNumber, String[] expectedBridgesNicknames) {
     this.runTest(request);
-    assertNotNull(this.summaryDocument);
+    assertNotNull("Summary document is null, status code is "
+        + this.response.errorStatusCode, this.summaryDocument);
     assertEquals(expectedRelaysNumber,
         this.summaryDocument.relays.length);
     if (expectedRelaysNicknames != null) {
@@ -1531,5 +1533,61 @@ public class ResourceServletTest {
     this.assertErrorStatusCode(
         "/summary?family=00000000000000000000000000000000000000", 400);
   }
+
+  @Test
+  public void testVersion02325() {
+    this.assertSummaryDocument("/summary?version=0.2.3.25", 2,
+        new String[] { "TorkaZ", "TimMayTribute" }, 0, null);
+  }
+
+  @Test
+  public void testVersion02324() {
+    this.assertSummaryDocument("/summary?version=0.2.3.24-rc-dev", 1,
+        new String[] { "Ferrari458" }, 0, null);
+  }
+
+  @Test
+  public void testVersion12345() {
+    this.assertSummaryDocument("/summary?version=1.2.3.4.5", 0, null, 0, null);
+  }
+
+  @Test
+  public void testVersionBlaBlaBla() {
+    this.assertSummaryDocument("/summary?version=bla-bla-bla", 0, null, 0,
+        null);
+  }
+
+  @Test
+  public void testVersion0() {
+    this.assertSummaryDocument("/summary?version=0", 3, null, 0, null);
+  }
+
+  @Test
+  public void testVersion02() {
+    this.assertSummaryDocument("/summary?version=0.2", 3, null, 0, null);
+  }
+
+  @Test
+  public void testVersion023() {
+    this.assertSummaryDocument("/summary?version=0.2.3", 3, null, 0, null);
+  }
+
+  @Test
+  public void testVersion0232() {
+    /* This is correct when comparing strings. */
+    this.assertSummaryDocument("/summary?version=0.2.3.2", 3, null, 0, null);
+  }
+
+  @Test
+  public void testVersion023Dot() {
+    /* This is also correct when comparing strings. */
+    this.assertSummaryDocument("/summary?version=0.2.3.", 3, null, 0, null);
+  }
+
+  @Test
+  public void testVersionStart() {
+    /* This is also correct when comparing strings. */
+    this.assertErrorStatusCode("/summary?version=*", 400);
+  }
 }
 
diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
index c1d909f..fb3f5b3 100644
--- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
+++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java
@@ -40,7 +40,7 @@ public class SummaryDocumentComparatorTest {
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
-        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
+        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
   }
 
   /** Some values for running all comparison types. */





More information about the tor-commits mailing list