[tor-commits] [collector/master] Include microdescriptors when syncing.

[karsten at torproject.org]

commit a1b8ebb9545b148524c6e3db2a1601f228784905
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Wed Dec 9 22:01:21 2020 +0100

    Include microdescriptors when syncing.
---
 CHANGELOG.md                                       |  3 ++
 build.xml                                          |  2 +-
 .../persist/MicrodescriptorPersistence.java        | 36 ++++++++++++++++++++++
 .../collector/relaydescs/ArchiveWriter.java        | 11 +++++--
 .../metrics/collector/sync/SyncPersistence.java    | 10 ++++++
 5 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2928937..22d3517 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@
   - Clean up descriptors written to the `out/` directory by deleting
     files that are older than seven weeks.
   - Correctly index files that are moved away and back.
+  - Include microdescriptors when syncing from another CollecTor
+    instance.
+  - Update to metrics-lib 2.15.0.
 
 
 # Changes in version 1.16.1 - 2020-08-16
diff --git a/build.xml b/build.xml
index 5dcb29d..f2bda4e 100644
--- a/build.xml
+++ b/build.xml
@@ -12,7 +12,7 @@
   <property name="release.version" value="1.16.1-dev" />
   <property name="project-main-class" value="org.torproject.metrics.collector.Main" />
   <property name="name" value="collector"/>
-  <property name="metricslibversion" value="2.14.0" />
+  <property name="metricslibversion" value="2.15.0" />
   <property name="jarincludes" value="collector.properties logback.xml" />
 
   <patternset id="runtime" >
diff --git a/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
new file mode 100644
index 0000000..41929af
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
@@ -0,0 +1,36 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.persist;
+
+import org.torproject.descriptor.Microdescriptor;
+import org.torproject.metrics.collector.conf.Annotation;
+
+import java.nio.file.Paths;
+
+public class MicrodescriptorPersistence
+    extends DescriptorPersistence<Microdescriptor> {
+
+  private static final String RELAY_DESCRIPTORS = "relay-descriptors";
+
+  public MicrodescriptorPersistence(Microdescriptor descriptor, long received,
+      String year, String month) {
+    super(descriptor, Annotation.Microdescriptor.bytes());
+    calculatePaths(received, year, month);
+  }
+
+  private void calculatePaths(long received, String year, String month) {
+    String file = PersistenceUtils.dateTime(received);
+    this.recentPath = Paths.get(
+        RELAY_DESCRIPTORS, MICRODESCS, "micro",
+        file + "-micro-" + year + "-" + month).toString();
+    String digest = desc.getDigestSha256Hex();
+    this.storagePath = Paths.get(
+        RELAY_DESCRIPTORS,
+        MICRODESC, year, month, "micro",
+        digest.substring(0,1),
+        digest.substring(1,2),
+        digest).toString();
+  }
+}
+
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 28472f8..5c58f23 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile;
 import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParser;
 import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.Microdescriptor;
 import org.torproject.descriptor.RelayExtraInfoDescriptor;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -109,6 +110,8 @@ public class ArchiveWriter extends CollecTorMain {
     this.mapPathDescriptors.put(
         "recent/relay-descriptors/microdescs/consensus-microdesc",
         RelayNetworkStatusConsensus.class);
+    this.mapPathDescriptors.put("recent/relay-descriptors/microdescs/micro/",
+        Microdescriptor.class);
     this.mapPathDescriptors.put("recent/relay-descriptors/server-descriptors",
         RelayServerDescriptor.class);
     this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos",
@@ -801,15 +804,17 @@ public class ArchiveWriter extends CollecTorMain {
      * file written in the first call.  However, this method must be
      * called twice to store the same microdescriptor in two different
      * valid-after months. */
-    SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+    SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy-MM");
+    String[] yearMonth = descriptorFormat.format(validAfter).split("-");
     File tarballFile = Paths.get(this.outputDirectory, MICRODESC,
-        descriptorFormat.format(validAfter), MICRO,
+        yearMonth[0], yearMonth[1], MICRO,
         microdescriptorDigest.substring(0, 1),
         microdescriptorDigest.substring(1, 2),
         microdescriptorDigest).toFile();
     boolean tarballFileExistedBefore = tarballFile.exists();
     File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
-        MICRODESCS, MICRO, this.rsyncCatString + "-micro.tmp").toFile();
+        MICRODESCS, MICRO, this.rsyncCatString + "-micro-" + yearMonth[0] + "-"
+        + yearMonth[1] + ".tmp").toFile();
     File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
     boolean[] append = new boolean[] { false, true };
     if (this.store(Annotation.Microdescriptor.bytes(), data, outputFiles,
diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
index bbb61b7..e8f780e 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
@@ -11,6 +11,7 @@ import org.torproject.descriptor.BridgeServerDescriptor;
 import org.torproject.descriptor.BridgedbMetrics;
 import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.Microdescriptor;
 import org.torproject.descriptor.RelayExtraInfoDescriptor;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -32,6 +33,7 @@ import org.torproject.metrics.collector.persist.DescriptorPersistence;
 import org.torproject.metrics.collector.persist.ExitlistPersistence;
 import org.torproject.metrics.collector.persist.ExtraInfoPersistence;
 import org.torproject.metrics.collector.persist.MicroConsensusPersistence;
+import org.torproject.metrics.collector.persist.MicrodescriptorPersistence;
 import org.torproject.metrics.collector.persist.OnionPerfPersistence;
 import org.torproject.metrics.collector.persist.PersistenceUtils;
 import org.torproject.metrics.collector.persist.ServerDescriptorPersistence;
@@ -118,6 +120,14 @@ public class SyncPersistence {
           descPersist = new ExtraInfoPersistence(
               (RelayExtraInfoDescriptor) desc, received);
           break;
+        case "Microdescriptor":
+          if (filename.contains("-micro-")) {
+            String[] yearMonth = filename.substring(
+                filename.indexOf("-micro-") + 7).split("-");
+            descPersist = new MicrodescriptorPersistence((Microdescriptor) desc,
+                received, yearMonth[0], yearMonth[1]);
+          }
+          break;
         case "BridgeNetworkStatus": // need to infer authId from filename
           String[] filenameParts = filename.split(DASH);
           if (filenameParts.length < 3) {