commit a1b8ebb9545b148524c6e3db2a1601f228784905 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Dec 9 22:01:21 2020 +0100
Include microdescriptors when syncing. --- CHANGELOG.md | 3 ++ build.xml | 2 +- .../persist/MicrodescriptorPersistence.java | 36 ++++++++++++++++++++++ .../collector/relaydescs/ArchiveWriter.java | 11 +++++-- .../metrics/collector/sync/SyncPersistence.java | 10 ++++++ 5 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md index 2928937..22d3517 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ - Clean up descriptors written to the `out/` directory by deleting files that are older than seven weeks. - Correctly index files that are moved away and back. + - Include microdescriptors when syncing from another CollecTor + instance. + - Update to metrics-lib 2.15.0.
# Changes in version 1.16.1 - 2020-08-16 diff --git a/build.xml b/build.xml index 5dcb29d..f2bda4e 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="release.version" value="1.16.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.14.0" /> + <property name="metricslibversion" value="2.15.0" /> <property name="jarincludes" value="collector.properties logback.xml" />
<patternset id="runtime" > diff --git a/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java new file mode 100644 index 0000000..41929af --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java @@ -0,0 +1,36 @@ +/* Copyright 2020 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.persist; + +import org.torproject.descriptor.Microdescriptor; +import org.torproject.metrics.collector.conf.Annotation; + +import java.nio.file.Paths; + +public class MicrodescriptorPersistence + extends DescriptorPersistence<Microdescriptor> { + + private static final String RELAY_DESCRIPTORS = "relay-descriptors"; + + public MicrodescriptorPersistence(Microdescriptor descriptor, long received, + String year, String month) { + super(descriptor, Annotation.Microdescriptor.bytes()); + calculatePaths(received, year, month); + } + + private void calculatePaths(long received, String year, String month) { + String file = PersistenceUtils.dateTime(received); + this.recentPath = Paths.get( + RELAY_DESCRIPTORS, MICRODESCS, "micro", + file + "-micro-" + year + "-" + month).toString(); + String digest = desc.getDigestSha256Hex(); + this.storagePath = Paths.get( + RELAY_DESCRIPTORS, + MICRODESC, year, month, "micro", + digest.substring(0,1), + digest.substring(1,2), + digest).toString(); + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 28472f8..5c58f23 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.Microdescriptor; import org.torproject.descriptor.RelayExtraInfoDescriptor; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.RelayNetworkStatusVote; @@ -109,6 +110,8 @@ public class ArchiveWriter extends CollecTorMain { this.mapPathDescriptors.put( "recent/relay-descriptors/microdescs/consensus-microdesc", RelayNetworkStatusConsensus.class); + this.mapPathDescriptors.put("recent/relay-descriptors/microdescs/micro/", + Microdescriptor.class); this.mapPathDescriptors.put("recent/relay-descriptors/server-descriptors", RelayServerDescriptor.class); this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos", @@ -801,15 +804,17 @@ public class ArchiveWriter extends CollecTorMain { * file written in the first call. However, this method must be * called twice to store the same microdescriptor in two different * valid-after months. */ - SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/"); + SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy-MM"); + String[] yearMonth = descriptorFormat.format(validAfter).split("-"); File tarballFile = Paths.get(this.outputDirectory, MICRODESC, - descriptorFormat.format(validAfter), MICRO, + yearMonth[0], yearMonth[1], MICRO, microdescriptorDigest.substring(0, 1), microdescriptorDigest.substring(1, 2), microdescriptorDigest).toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, - MICRODESCS, MICRO, this.rsyncCatString + "-micro.tmp").toFile(); + MICRODESCS, MICRO, this.rsyncCatString + "-micro-" + yearMonth[0] + "-" + + yearMonth[1] + ".tmp").toFile(); File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; boolean[] append = new boolean[] { false, true }; if (this.store(Annotation.Microdescriptor.bytes(), data, outputFiles, diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java index bbb61b7..e8f780e 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java @@ -11,6 +11,7 @@ import org.torproject.descriptor.BridgeServerDescriptor; import org.torproject.descriptor.BridgedbMetrics; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.Microdescriptor; import org.torproject.descriptor.RelayExtraInfoDescriptor; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.RelayNetworkStatusVote; @@ -32,6 +33,7 @@ import org.torproject.metrics.collector.persist.DescriptorPersistence; import org.torproject.metrics.collector.persist.ExitlistPersistence; import org.torproject.metrics.collector.persist.ExtraInfoPersistence; import org.torproject.metrics.collector.persist.MicroConsensusPersistence; +import org.torproject.metrics.collector.persist.MicrodescriptorPersistence; import org.torproject.metrics.collector.persist.OnionPerfPersistence; import org.torproject.metrics.collector.persist.PersistenceUtils; import org.torproject.metrics.collector.persist.ServerDescriptorPersistence; @@ -118,6 +120,14 @@ public class SyncPersistence { descPersist = new ExtraInfoPersistence( (RelayExtraInfoDescriptor) desc, received); break; + case "Microdescriptor": + if (filename.contains("-micro-")) { + String[] yearMonth = filename.substring( + filename.indexOf("-micro-") + 7).split("-"); + descPersist = new MicrodescriptorPersistence((Microdescriptor) desc, + received, yearMonth[0], yearMonth[1]); + } + break; case "BridgeNetworkStatus": // need to infer authId from filename String[] filenameParts = filename.split(DASH); if (filenameParts.length < 3) {