commit 8ffdfd6c2cd4a1d6a6f0660681e9a788b88dc342 Author: iwakeh <iwakeh@torproject.org> Date: Wed Jul 12 18:47:45 2017 +0000 Enable OnionPerf 'Sync' runs. --- .../org/torproject/collector/conf/Annotation.java | 2 +- .../java/org/torproject/collector/conf/Key.java | 1 + .../collector/onionperf/OnionPerfDownloader.java | 1 + .../collector/persist/OnionPerfPersistence.java | 49 ++++++++++++++++++++++ .../torproject/collector/sync/SyncPersistence.java | 5 +++ src/main/resources/collector.properties | 5 ++- 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/collector/conf/Annotation.java b/src/main/java/org/torproject/collector/conf/Annotation.java index 0dab546..23f485c 100644 --- a/src/main/java/org/torproject/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/collector/conf/Annotation.java @@ -16,7 +16,7 @@ public enum Annotation { Microdescriptor("@type microdescriptor 1.0\n"), Server("@type server-descriptor 1.0\n"), Status("@type bridge-network-status 1.2\n"), - OnionPerf("@type torperf 1.0\n"), + OnionPerf("@type torperf 1.1\n"), Vote("@type network-status-vote-3 1.0\n"); private final String annotation; diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index 4341432..72af4bb 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -34,6 +34,7 @@ public enum Key { BridgeSyncOrigins(URL[].class), BridgeLocalOrigins(Path.class), ExitlistSyncOrigins(URL[].class), + OnionPerfSyncOrigins(URL[].class), BridgedescsActivated(Boolean.class), BridgedescsOffsetMinutes(Integer.class), BridgedescsPeriodMinutes(Integer.class), diff --git a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java index b2a27d4..36b006a 100644 --- a/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/collector/onionperf/OnionPerfDownloader.java @@ -51,6 +51,7 @@ public class OnionPerfDownloader extends CollecTorMain { /** Instantiate the OnionPerf module using the given configuration. */ public OnionPerfDownloader(Configuration config) { super(config); + this.mapPathDescriptors.put("recent/torperf", TorperfResult.class); } /** File containing the download history, which is necessary, because diff --git a/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java new file mode 100644 index 0000000..22093eb --- /dev/null +++ b/src/main/java/org/torproject/collector/persist/OnionPerfPersistence.java @@ -0,0 +1,49 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.persist; + +import org.torproject.collector.conf.Annotation; +import org.torproject.descriptor.TorperfResult; + +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; + +public class OnionPerfPersistence + extends DescriptorPersistence<TorperfResult> { + + private static final String ONIONPERF = "torperf"; + + public OnionPerfPersistence(TorperfResult desc) { + super(desc, Annotation.OnionPerf.bytes()); + calculatePaths(); + } + + private void calculatePaths() { + String[] parts = PersistenceUtils.dateTimeParts(desc.getStartMillis()); + String name = desc.getSource() + DASH + desc.getFileSize() + DASH + + parts[0] + DASH + parts[1] + DASH + parts[2] + ".tpf"; + this.recentPath = Paths.get(ONIONPERF, name).toString(); + this.storagePath = Paths.get( + ONIONPERF, + parts[0], // year + parts[1], // month + parts[2], // day + name).toString(); + } + + /** OnionPerf default storage appends. */ + @Override + public boolean storeOut(String outRoot) { + return super.storeOut(outRoot, StandardOpenOption.APPEND); + } + + /** OnionPerf default storage appends. */ + @Override + public boolean storeAll(String recentRoot, String outRoot) { + return super.storeAll(recentRoot, outRoot, StandardOpenOption.APPEND, + StandardOpenOption.APPEND); + } + +} + diff --git a/src/main/java/org/torproject/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/collector/sync/SyncPersistence.java index 5fd6dc6..baca83d 100644 --- a/src/main/java/org/torproject/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/collector/sync/SyncPersistence.java @@ -13,6 +13,7 @@ import org.torproject.collector.persist.DescriptorPersistence; import org.torproject.collector.persist.ExitlistPersistence; import org.torproject.collector.persist.ExtraInfoPersistence; import org.torproject.collector.persist.MicroConsensusPersistence; +import org.torproject.collector.persist.OnionPerfPersistence; import org.torproject.collector.persist.PersistenceUtils; import org.torproject.collector.persist.ServerDescriptorPersistence; import org.torproject.collector.persist.StatusPersistence; @@ -26,6 +27,7 @@ import org.torproject.descriptor.RelayExtraInfoDescriptor; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.RelayNetworkStatusVote; import org.torproject.descriptor.RelayServerDescriptor; +import org.torproject.descriptor.TorperfResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,6 +129,9 @@ public class SyncPersistence { case "ExitList": // downloaded is part of desc, which to use? descPersist = new ExitlistPersistence((ExitList) desc, received); break; + case "TorperfResult": + descPersist = new OnionPerfPersistence((TorperfResult) desc); + break; default: log.trace("Invalid descriptor type {} for sync-merge.", clazz.getName()); diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index d49bd58..0a9f932 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -142,8 +142,11 @@ ExitlistUrl = https://check.torproject.org/exit-addresses ######## OnionPerf downloader ######## # ## Define descriptor sources -# possible values: Remote +# possible values: Remote,Sync OnionPerfSources = Remote +# Retrieve files from the following CollecTor instances. +# List of URLs separated by comma. +OnionPerfSyncOrigins = https://collector.torproject.org # ## OnionPerf base URLs ## Hosts must be configured to use the first subdomain part of the given URL as