commit 9834cec8c0e88c1e810434f543f5d8c4d5cb2de8 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Dec 14 09:35:35 2020 +0100 Include certs when syncing from another instance. --- CHANGELOG.md | 4 ++-- .../DirectoryKeyCertificatePersistence.java | 27 ++++++++++++++++++++++ .../collector/relaydescs/ArchiveWriter.java | 7 +++++- .../metrics/collector/sync/SyncPersistence.java | 6 +++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d3517..f56f74d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ - Clean up descriptors written to the `out/` directory by deleting files that are older than seven weeks. - Correctly index files that are moved away and back. - - Include microdescriptors when syncing from another CollecTor - instance. + - Include microdescriptors and certs when syncing from another + CollecTor instance. - Update to metrics-lib 2.15.0. diff --git a/src/main/java/org/torproject/metrics/collector/persist/DirectoryKeyCertificatePersistence.java b/src/main/java/org/torproject/metrics/collector/persist/DirectoryKeyCertificatePersistence.java new file mode 100644 index 0000000..39f88f3 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/persist/DirectoryKeyCertificatePersistence.java @@ -0,0 +1,27 @@ +/* Copyright 2020 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.persist; + +import org.torproject.descriptor.DirectoryKeyCertificate; +import org.torproject.metrics.collector.conf.Annotation; + +import java.nio.file.Paths; + +public class DirectoryKeyCertificatePersistence + extends DescriptorPersistence<DirectoryKeyCertificate> { + + public DirectoryKeyCertificatePersistence( + DirectoryKeyCertificate descriptor) { + super(descriptor, Annotation.Cert.bytes()); + this.calculatePaths(); + } + + private void calculatePaths() { + String fileName = this.desc.getFingerprint().toUpperCase() + "-" + + PersistenceUtils.dateTime(this.desc.getDirKeyPublishedMillis()); + this.recentPath = Paths.get(RELAYDESCS, "certs", fileName).toString(); + this.storagePath = this.recentPath; + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 5c58f23..616d7dd 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.DirectoryKeyCertificate; import org.torproject.descriptor.Microdescriptor; import org.torproject.descriptor.RelayExtraInfoDescriptor; import org.torproject.descriptor.RelayNetworkStatusConsensus; @@ -105,6 +106,8 @@ public class ArchiveWriter extends CollecTorMain { super(config); this.mapPathDescriptors.put("recent/relay-descriptors/votes", RelayNetworkStatusVote.class); + this.mapPathDescriptors.put("recent/relay-descriptors/certs", + DirectoryKeyCertificate.class); this.mapPathDescriptors.put("recent/relay-descriptors/consensuses", RelayNetworkStatusConsensus.class); this.mapPathDescriptors.put( @@ -738,7 +741,9 @@ public class ArchiveWriter extends CollecTorMain { "yyyy-MM-dd-HH-mm-ss"); File tarballFile = Paths.get(this.outputDirectory, "certs", fingerprint + "-" + printFormat.format(new Date(published))).toFile(); - File[] outputFiles = new File[] { tarballFile }; + File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "certs", + tarballFile.getName()).toFile(); + File[] outputFiles = new File[] { tarballFile, rsyncFile }; if (this.store(Annotation.Cert.bytes(), data, outputFiles, null)) { this.storedCertsCounter++; } diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java index e8f780e..ba06bd1 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java @@ -10,6 +10,7 @@ import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.BridgeServerDescriptor; import org.torproject.descriptor.BridgedbMetrics; import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DirectoryKeyCertificate; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.Microdescriptor; import org.torproject.descriptor.RelayExtraInfoDescriptor; @@ -30,6 +31,7 @@ import org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistenc import org.torproject.metrics.collector.persist.BridgedbMetricsPersistence; import org.torproject.metrics.collector.persist.ConsensusPersistence; import org.torproject.metrics.collector.persist.DescriptorPersistence; +import org.torproject.metrics.collector.persist.DirectoryKeyCertificatePersistence; import org.torproject.metrics.collector.persist.ExitlistPersistence; import org.torproject.metrics.collector.persist.ExtraInfoPersistence; import org.torproject.metrics.collector.persist.MicroConsensusPersistence; @@ -108,6 +110,10 @@ public class SyncPersistence { descPersist = new MicroConsensusPersistence(cons, received); } break; + case "DirectoryKeyCertificate": + descPersist = new DirectoryKeyCertificatePersistence( + (DirectoryKeyCertificate) desc); + break; case "RelayServerDescriptor": descPersist = new ServerDescriptorPersistence( (RelayServerDescriptor) desc, received);