commit 3121a2692cf76c344c70a1bf2abcaed4966495eb Author: iwakeh <iwakeh@torproject.org> Date: Wed Oct 12 19:08:04 2016 +0200 Introduced *Sources and *Origins properties to simplify data source definition. Already provides properties for task-18910. --- CHANGELOG.md | 2 + .../bridgedescs/SanitizedBridgesWriter.java | 2 +- .../torproject/collector/conf/Configuration.java | 26 ++++++++++ .../java/org/torproject/collector/conf/Key.java | 16 +++--- .../org/torproject/collector/conf/SourceType.java | 11 +++++ .../collector/relaydescs/ArchiveWriter.java | 13 +++-- src/main/resources/collector.properties | 39 +++++++++------ .../bridgedescs/SanitizedBridgesWriterTest.java | 2 +- .../collector/conf/ConfigurationTest.java | 57 +++++++++++++++++----- 9 files changed, 129 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fffad5e..6e1b7eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ * Medium changes - Replace four properties for configuring where to write descriptors by a single 'OutPath' property. + - Introduced *Sources and *Origins properties to simplify + data source definition. # Changes in version 1.0.2 - 2016-10-07 diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 00aa54b..79cb13c 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -100,7 +100,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(), BRIDGE_DESCRIPTORS).toString(); File bridgeDirectoriesDirectory = - config.getPath(Key.BridgeSnapshotsDirectory).toFile(); + config.getPath(Key.BridgeLocalOrigins).toFile(); File sanitizedBridgesDirectory = new File(outputPathName); File statsDirectory = config.getPath(Key.StatsPath).toFile(); diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java index b0b76ed..b63a00c 100644 --- a/src/main/java/org/torproject/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/collector/conf/Configuration.java @@ -15,8 +15,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.FileTime; +import java.util.EnumSet; import java.util.Observable; import java.util.Properties; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -251,6 +253,30 @@ public class Configuration extends Observable implements Cloneable { } /** + * Returns a {@code SourceType} as List, e.g. + * {@code sourcetypeproperty = Remote, Sync}. + */ + public Set<SourceType> getSourceTypeSet(Key key) throws ConfigurationException { + Set<SourceType> res = null; + try { + checkClass(key, SourceType[].class); + String[] interim = props.getProperty(key.name()).split(FIELDSEP); + for (int i = 0; i < interim.length; i++) { + SourceType st = SourceType.valueOf(interim[i].trim()); + if (null == res) { + res = EnumSet.of(st); + } else { + res.add(st); + } + } + return res; + } catch (RuntimeException re) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + re.getMessage(), re); + } + } + + /** * Returns a {@code URL} property, e.g. * {@code urlProperty = https://my.url.here}. */ diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index 613dc7a..d06fe0c 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -21,6 +21,16 @@ public enum Key { OutputPath(Path.class), IndexPath(Path.class), StatsPath(Path.class), + SyncPath(Path.class), + RelaySources(SourceType[].class), + BridgeSources(SourceType[].class), + ExitlistSources(SourceType[].class), + RelayCacheOrigins(String[].class), + RelayLocalOrigins(Path.class), + RelaySyncOrigins(URL[].class), + BridgeSyncOrigins(URL[].class), + BridgeLocalOrigins(Path.class), + ExitlistSyncOrigins(URL[].class), BridgedescsActivated(Boolean.class), BridgedescsOffsetMinutes(Integer.class), BridgedescsPeriodMinutes(Integer.class), @@ -36,11 +46,7 @@ public enum Key { UpdateindexActivated(Boolean.class), UpdateindexOffsetMinutes(Integer.class), UpdateindexPeriodMinutes(Integer.class), - BridgeSnapshotsDirectory(Path.class), - CachedRelayDescriptorsDirectories(String[].class), CompressRelayDescriptorDownloads(Boolean.class), - DirectoryArchivesDirectory(Path.class), - DownloadRelayDescriptors(Boolean.class), DirectoryAuthoritiesAddresses(String[].class), DirectoryAuthoritiesFingerprintsForVotes(String[].class), DownloadCurrentConsensus(Boolean.class), @@ -51,8 +57,6 @@ public enum Key { DownloadMissingMicrodescriptors(Boolean.class), DownloadAllServerDescriptors(Boolean.class), DownloadAllExtraInfoDescriptors(Boolean.class), - ImportCachedRelayDescriptors(Boolean.class), - ImportDirectoryArchives(Boolean.class), KeepDirectoryArchiveImportHistory(Boolean.class), ReplaceIpAddressesWithHashes(Boolean.class), BridgeDescriptorMappingsLimit(Integer.class), diff --git a/src/main/java/org/torproject/collector/conf/SourceType.java b/src/main/java/org/torproject/collector/conf/SourceType.java new file mode 100644 index 0000000..b6eb7ce --- /dev/null +++ b/src/main/java/org/torproject/collector/conf/SourceType.java @@ -0,0 +1,11 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.conf; + +public enum SourceType { + Cache, + Local, + Remote, + Sync; +} diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index cac5d14..cdf6506 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -6,6 +6,7 @@ package org.torproject.collector.relaydescs; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; +import org.torproject.collector.conf.SourceType; import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; @@ -146,7 +147,9 @@ public class ArchiveWriter extends CollecTorMain { RelayDescriptorParser rdp = new RelayDescriptorParser(this); RelayDescriptorDownloader rdd = null; - if (config.getBool(Key.DownloadRelayDescriptors)) { + + Set<SourceType> sources = config.getSourceTypeSet(Key.RelaySources); + if (sources.contains(SourceType.Remote)) { String[] dirSources = config.getStringArray(Key.DirectoryAuthoritiesAddresses); rdd = new RelayDescriptorDownloader(rdp, dirSources, @@ -162,15 +165,15 @@ public class ArchiveWriter extends CollecTorMain { config.getBool(Key.CompressRelayDescriptorDownloads)); rdp.setRelayDescriptorDownloader(rdd); } - if (config.getBool(Key.ImportCachedRelayDescriptors)) { + if (sources.contains(SourceType.Cache)) { new CachedRelayDescriptorReader(rdp, - config.getStringArray(Key.CachedRelayDescriptorsDirectories), statsDirectory); + config.getStringArray(Key.RelayCacheOrigins), statsDirectory); this.intermediateStats("importing relay descriptors from local " + "Tor data directories"); } - if (config.getBool(Key.ImportDirectoryArchives)) { + if (sources.contains(SourceType.Local)) { new ArchiveReader(rdp, - config.getPath(Key.DirectoryArchivesDirectory).toFile(), + config.getPath(Key.RelayLocalOrigins).toFile(), statsDirectory, config.getBool(Key.KeepDirectoryArchiveImportHistory)); this.intermediateStats("importing relay descriptors from local " diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 1c02c92..4441e1d 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -64,20 +64,22 @@ RecentPath = recent OutputPath = out # Some statistics are stored here. StatsPath = stats +# Path for descriptors downloaded from other instances +SyncPath = sync ######## Relay descriptors ######## # -## Read cached-* files from a local Tor data directory -ImportCachedRelayDescriptors = false +## Define descriptor sources +# possible values: Sync, Cache, Remote, Local +RelaySources = Remote +# Retrieve files from the following CollecTor instances. +# List of URLs separated by comma. +RelaySyncOrigins = https://collector.torproject.org # -## Relative path to Tor data directory to read cached-* files from +## Path to Tor data directory to read cached-* files from ## the listed path(s). If there is more that one separated by comma. -CachedRelayDescriptorsDirectories = in/relay-descriptors/cacheddesc/ -# -## Import directory archives from disk, if available -ImportDirectoryArchives = false -# +RelayCacheOrigins = in/relay-descriptors/cacheddesc/ ## Relative path to directory to import directory archives from -DirectoryArchivesDirectory = in/relay-descriptors/archives/ +RelayLocalOrigins = in/relay-descriptors/archives/ # ## Keep a history of imported directory archive files to know which files ## have been imported before. This history can be useful when importing @@ -85,9 +87,6 @@ DirectoryArchivesDirectory = in/relay-descriptors/archives/ ## again, but it can be confusing to users who don't know about it. KeepDirectoryArchiveImportHistory = false # -## Download relay descriptors from directory authorities, if required -DownloadRelayDescriptors = false -# ## Comma separated list of directory authority addresses (IP[:port]) to ## download missing relay descriptors from DirectoryAuthoritiesAddresses = 128.31.0.39:9131,86.59.21.38,194.109.206.212,131.188.40.189,193.23.244.244,171.25.193.9:443,154.35.175.225,199.254.238.52 @@ -132,8 +131,14 @@ CompressRelayDescriptorDownloads = false # ######## Bridge descriptors ######## # +## Define descriptor sources +# possible values: Sync, Local +BridgeSources = Local +# Retrieve files from the following instances. +# List of URLs separated by comma. +BridgeSyncOrigins = https://collector.torproject.org ## Relative path to directory to import bridge descriptor snapshots from -BridgeSnapshotsDirectory = in/bridge-descriptors/ +BridgeLocalOrigins = in/bridge-descriptors/ # ## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z ## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we @@ -147,9 +152,15 @@ BridgeDescriptorMappingsLimit = inf # ######## Exit lists ######## # +## Define descriptor sources +# possible values: Sync, Remote +ExitlistSources = Remote +## Retrieve files from the following instances. +## List of URLs separated by comma. +ExitlistSyncOrigins = https://collector.torproject.org ## Where to download exit-lists from. ExitlistUrl = https://check.torproject.org/exit-addresses - +# ######## Torperf downloader ######## # ## Torperf source names and base URLs diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java index 299bd92..947f486 100644 --- a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -132,7 +132,7 @@ public class SanitizedBridgesWriterTest { this.configuration.setProperty(Key.RecentPath.name(), recentDirectory.getAbsolutePath()); this.configuration.setProperty(Key.StatsPath.name(), statsDirectory); - this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(), + this.configuration.setProperty(Key.BridgeLocalOrigins.name(), bridgeDirectoriesDir); this.configuration.setProperty(Key.OutputPath.name(), sanitizedBridgesDirectory.toFile().getParent().toString()); diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index 8e255c6..3f9c231 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.Observable; import java.util.Observer; import java.util.Random; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; public class ConfigurationTest { @@ -40,7 +41,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 46, Key.values().length); + 50, Key.values().length); } @Test() @@ -70,11 +71,11 @@ public class ConfigurationTest { Configuration conf = new Configuration(); for (String input : arrays) { conf.clear(); - conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), input); + conf.setProperty(Key.RelayCacheOrigins.name(), input); assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: " + Arrays.toString(conf - .getStringArray(Key.CachedRelayDescriptorsDirectories)), - array, conf.getStringArray(Key.CachedRelayDescriptorsDirectories)); + .getStringArray(Key.RelayCacheOrigins)), + array, conf.getStringArray(Key.RelayCacheOrigins)); } } @@ -82,10 +83,10 @@ public class ConfigurationTest { public void testBoolValues() throws Exception { Configuration conf = new Configuration(); conf.setProperty(Key.CompressRelayDescriptorDownloads.name(), "false"); - conf.setProperty(Key.ImportDirectoryArchives.name(), "trUe"); + conf.setProperty(Key.RunOnce.name(), "trUe"); conf.setProperty(Key.ReplaceIpAddressesWithHashes.name(), "false"); assertFalse(conf.getBool(Key.CompressRelayDescriptorDownloads)); - assertTrue(conf.getBool(Key.ImportDirectoryArchives)); + assertTrue(conf.getBool(Key.RunOnce)); assertFalse(conf.getBool(Key.ReplaceIpAddressesWithHashes)); } @@ -117,6 +118,19 @@ public class ConfigurationTest { } @Test() + public void testSourceTypeValues() throws Exception { + String[] types = new String[] { "Local", "Cache", "Remote", "Sync"}; + Configuration conf = new Configuration(); + for (String type : types) { + conf.clear(); + conf.setProperty(Key.BridgeSources.name(), type); + Set<SourceType> sts = conf.getSourceTypeSet(Key.BridgeSources); + assertEquals(1, sts.size()); + assertTrue(sts.contains(SourceType.valueOf(type))); + } + } + + @Test() public void testArrayArrayValues() throws Exception { String[][] sourceStrings = new String[][] { new String[]{"localsource", "http://127.0.0.1:12345"}, @@ -129,17 +143,36 @@ public class ConfigurationTest { conf.getStringArrayArray(Key.TorperfSources)); } + @Test() + public void testUrlArrayValues() throws Exception { + URL[] array = new URL[randomSource.nextInt(30) + 1]; + for (int i = 0; i < array.length; i++) { + array[i] = new URL("https://" + + Integer.toBinaryString(randomSource.nextInt(100)) + ".dummy.org"); + } + String input = + Arrays.toString(array).replace("[", "").replace("]", "") + .replaceAll(" ", ""); + Configuration conf = new Configuration(); + conf.clear(); + conf.setProperty(Key.RelaySyncOrigins.name(), input); + assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: " + + Arrays.toString(conf + .getUrlArray(Key.RelaySyncOrigins)), + array, conf.getUrlArray(Key.RelaySyncOrigins)); + } + @Test(expected = ConfigurationException.class) public void testArrayArrayValueException() throws Exception { Configuration conf = new Configuration(); - conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), ""); + conf.setProperty(Key.RelayCacheOrigins.name(), ""); conf.getStringArrayArray(Key.OutputPath); } @Test(expected = ConfigurationException.class) public void testArrayValueException() throws Exception { Configuration conf = new Configuration(); - conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), ""); + conf.setProperty(Key.RelayCacheOrigins.name(), ""); conf.getStringArray(Key.TorperfSources); } @@ -147,14 +180,14 @@ public class ConfigurationTest { public void testBoolValueException() throws Exception { Configuration conf = new Configuration(); conf.setProperty(Key.TorperfSources.name(), "http://x.y.z"); - conf.getBool(Key.CachedRelayDescriptorsDirectories); + conf.getBool(Key.RelayCacheOrigins); } @Test(expected = ConfigurationException.class) public void testPathValueException() throws Exception { Configuration conf = new Configuration(); - conf.setProperty(Key.DirectoryArchivesDirectory.name(), "\\\u0000:"); - conf.getPath(Key.DirectoryArchivesDirectory); + conf.setProperty(Key.RelayLocalOrigins.name(), "\\\u0000:"); + conf.getPath(Key.RelayLocalOrigins); } @Test(expected = ConfigurationException.class) @@ -214,7 +247,7 @@ public class ConfigurationTest { conf.setWatchableSourceAndLoad(confFile.toPath()); MainTest.waitSec(1); confFile.delete(); - conf.setProperty(Key.ImportDirectoryArchives.name(), "false"); + conf.setProperty(Key.RunOnce.name(), "false"); final Dummy dummy = new Dummy(conf); tmpf.newFolder("empty"); MainTest.waitSec(6);