[tor-commits] [collector/master] Introduced *Sources and *Origins properties to simplify data
karsten at torproject.org
karsten at torproject.org
Thu Oct 13 07:05:24 UTC 2016
commit 3121a2692cf76c344c70a1bf2abcaed4966495eb
Author: iwakeh <iwakeh at torproject.org>
Date: Wed Oct 12 19:08:04 2016 +0200
Introduced *Sources and *Origins properties to simplify data
source definition.
Already provides properties for task-18910.
---
CHANGELOG.md | 2 +
.../bridgedescs/SanitizedBridgesWriter.java | 2 +-
.../torproject/collector/conf/Configuration.java | 26 ++++++++++
.../java/org/torproject/collector/conf/Key.java | 16 +++---
.../org/torproject/collector/conf/SourceType.java | 11 +++++
.../collector/relaydescs/ArchiveWriter.java | 13 +++--
src/main/resources/collector.properties | 39 +++++++++------
.../bridgedescs/SanitizedBridgesWriterTest.java | 2 +-
.../collector/conf/ConfigurationTest.java | 57 +++++++++++++++++-----
9 files changed, 129 insertions(+), 39 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fffad5e..6e1b7eb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
* Medium changes
- Replace four properties for configuring where to write
descriptors by a single 'OutPath' property.
+ - Introduced *Sources and *Origins properties to simplify
+ data source definition.
# Changes in version 1.0.2 - 2016-10-07
diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 00aa54b..79cb13c 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -100,7 +100,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
BRIDGE_DESCRIPTORS).toString();
File bridgeDirectoriesDirectory =
- config.getPath(Key.BridgeSnapshotsDirectory).toFile();
+ config.getPath(Key.BridgeLocalOrigins).toFile();
File sanitizedBridgesDirectory = new File(outputPathName);
File statsDirectory = config.getPath(Key.StatsPath).toFile();
diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java
index b0b76ed..b63a00c 100644
--- a/src/main/java/org/torproject/collector/conf/Configuration.java
+++ b/src/main/java/org/torproject/collector/conf/Configuration.java
@@ -15,8 +15,10 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
+import java.util.EnumSet;
import java.util.Observable;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -251,6 +253,30 @@ public class Configuration extends Observable implements Cloneable {
}
/**
+ * Returns a {@code SourceType} as List, e.g.
+ * {@code sourcetypeproperty = Remote, Sync}.
+ */
+ public Set<SourceType> getSourceTypeSet(Key key) throws ConfigurationException {
+ Set<SourceType> res = null;
+ try {
+ checkClass(key, SourceType[].class);
+ String[] interim = props.getProperty(key.name()).split(FIELDSEP);
+ for (int i = 0; i < interim.length; i++) {
+ SourceType st = SourceType.valueOf(interim[i].trim());
+ if (null == res) {
+ res = EnumSet.of(st);
+ } else {
+ res.add(st);
+ }
+ }
+ return res;
+ } catch (RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
+ /**
* Returns a {@code URL} property, e.g.
* {@code urlProperty = https://my.url.here}.
*/
diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java
index 613dc7a..d06fe0c 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -21,6 +21,16 @@ public enum Key {
OutputPath(Path.class),
IndexPath(Path.class),
StatsPath(Path.class),
+ SyncPath(Path.class),
+ RelaySources(SourceType[].class),
+ BridgeSources(SourceType[].class),
+ ExitlistSources(SourceType[].class),
+ RelayCacheOrigins(String[].class),
+ RelayLocalOrigins(Path.class),
+ RelaySyncOrigins(URL[].class),
+ BridgeSyncOrigins(URL[].class),
+ BridgeLocalOrigins(Path.class),
+ ExitlistSyncOrigins(URL[].class),
BridgedescsActivated(Boolean.class),
BridgedescsOffsetMinutes(Integer.class),
BridgedescsPeriodMinutes(Integer.class),
@@ -36,11 +46,7 @@ public enum Key {
UpdateindexActivated(Boolean.class),
UpdateindexOffsetMinutes(Integer.class),
UpdateindexPeriodMinutes(Integer.class),
- BridgeSnapshotsDirectory(Path.class),
- CachedRelayDescriptorsDirectories(String[].class),
CompressRelayDescriptorDownloads(Boolean.class),
- DirectoryArchivesDirectory(Path.class),
- DownloadRelayDescriptors(Boolean.class),
DirectoryAuthoritiesAddresses(String[].class),
DirectoryAuthoritiesFingerprintsForVotes(String[].class),
DownloadCurrentConsensus(Boolean.class),
@@ -51,8 +57,6 @@ public enum Key {
DownloadMissingMicrodescriptors(Boolean.class),
DownloadAllServerDescriptors(Boolean.class),
DownloadAllExtraInfoDescriptors(Boolean.class),
- ImportCachedRelayDescriptors(Boolean.class),
- ImportDirectoryArchives(Boolean.class),
KeepDirectoryArchiveImportHistory(Boolean.class),
ReplaceIpAddressesWithHashes(Boolean.class),
BridgeDescriptorMappingsLimit(Integer.class),
diff --git a/src/main/java/org/torproject/collector/conf/SourceType.java b/src/main/java/org/torproject/collector/conf/SourceType.java
new file mode 100644
index 0000000..b6eb7ce
--- /dev/null
+++ b/src/main/java/org/torproject/collector/conf/SourceType.java
@@ -0,0 +1,11 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.conf;
+
+public enum SourceType {
+ Cache,
+ Local,
+ Remote,
+ Sync;
+}
diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
index cac5d14..cdf6506 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -6,6 +6,7 @@ package org.torproject.collector.relaydescs;
import org.torproject.collector.conf.Configuration;
import org.torproject.collector.conf.ConfigurationException;
import org.torproject.collector.conf.Key;
+import org.torproject.collector.conf.SourceType;
import org.torproject.collector.cron.CollecTorMain;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorParser;
@@ -146,7 +147,9 @@ public class ArchiveWriter extends CollecTorMain {
RelayDescriptorParser rdp = new RelayDescriptorParser(this);
RelayDescriptorDownloader rdd = null;
- if (config.getBool(Key.DownloadRelayDescriptors)) {
+
+ Set<SourceType> sources = config.getSourceTypeSet(Key.RelaySources);
+ if (sources.contains(SourceType.Remote)) {
String[] dirSources =
config.getStringArray(Key.DirectoryAuthoritiesAddresses);
rdd = new RelayDescriptorDownloader(rdp, dirSources,
@@ -162,15 +165,15 @@ public class ArchiveWriter extends CollecTorMain {
config.getBool(Key.CompressRelayDescriptorDownloads));
rdp.setRelayDescriptorDownloader(rdd);
}
- if (config.getBool(Key.ImportCachedRelayDescriptors)) {
+ if (sources.contains(SourceType.Cache)) {
new CachedRelayDescriptorReader(rdp,
- config.getStringArray(Key.CachedRelayDescriptorsDirectories), statsDirectory);
+ config.getStringArray(Key.RelayCacheOrigins), statsDirectory);
this.intermediateStats("importing relay descriptors from local "
+ "Tor data directories");
}
- if (config.getBool(Key.ImportDirectoryArchives)) {
+ if (sources.contains(SourceType.Local)) {
new ArchiveReader(rdp,
- config.getPath(Key.DirectoryArchivesDirectory).toFile(),
+ config.getPath(Key.RelayLocalOrigins).toFile(),
statsDirectory,
config.getBool(Key.KeepDirectoryArchiveImportHistory));
this.intermediateStats("importing relay descriptors from local "
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index 1c02c92..4441e1d 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -64,20 +64,22 @@ RecentPath = recent
OutputPath = out
# Some statistics are stored here.
StatsPath = stats
+# Path for descriptors downloaded from other instances
+SyncPath = sync
######## Relay descriptors ########
#
-## Read cached-* files from a local Tor data directory
-ImportCachedRelayDescriptors = false
+## Define descriptor sources
+# possible values: Sync, Cache, Remote, Local
+RelaySources = Remote
+# Retrieve files from the following CollecTor instances.
+# List of URLs separated by comma.
+RelaySyncOrigins = https://collector.torproject.org
#
-## Relative path to Tor data directory to read cached-* files from
+## Path to Tor data directory to read cached-* files from
## the listed path(s). If there is more that one separated by comma.
-CachedRelayDescriptorsDirectories = in/relay-descriptors/cacheddesc/
-#
-## Import directory archives from disk, if available
-ImportDirectoryArchives = false
-#
+RelayCacheOrigins = in/relay-descriptors/cacheddesc/
## Relative path to directory to import directory archives from
-DirectoryArchivesDirectory = in/relay-descriptors/archives/
+RelayLocalOrigins = in/relay-descriptors/archives/
#
## Keep a history of imported directory archive files to know which files
## have been imported before. This history can be useful when importing
@@ -85,9 +87,6 @@ DirectoryArchivesDirectory = in/relay-descriptors/archives/
## again, but it can be confusing to users who don't know about it.
KeepDirectoryArchiveImportHistory = false
#
-## Download relay descriptors from directory authorities, if required
-DownloadRelayDescriptors = false
-#
## Comma separated list of directory authority addresses (IP[:port]) to
## download missing relay descriptors from
DirectoryAuthoritiesAddresses = 128.31.0.39:9131,86.59.21.38,194.109.206.212,131.188.40.189,193.23.244.244,171.25.193.9:443,154.35.175.225,199.254.238.52
@@ -132,8 +131,14 @@ CompressRelayDescriptorDownloads = false
#
######## Bridge descriptors ########
#
+## Define descriptor sources
+# possible values: Sync, Local
+BridgeSources = Local
+# Retrieve files from the following instances.
+# List of URLs separated by comma.
+BridgeSyncOrigins = https://collector.torproject.org
## Relative path to directory to import bridge descriptor snapshots from
-BridgeSnapshotsDirectory = in/bridge-descriptors/
+BridgeLocalOrigins = in/bridge-descriptors/
#
## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z
## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we
@@ -147,9 +152,15 @@ BridgeDescriptorMappingsLimit = inf
#
######## Exit lists ########
#
+## Define descriptor sources
+# possible values: Sync, Remote
+ExitlistSources = Remote
+## Retrieve files from the following instances.
+## List of URLs separated by comma.
+ExitlistSyncOrigins = https://collector.torproject.org
## Where to download exit-lists from.
ExitlistUrl = https://check.torproject.org/exit-addresses
-
+#
######## Torperf downloader ########
#
## Torperf source names and base URLs
diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
index 299bd92..947f486 100644
--- a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
+++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
@@ -132,7 +132,7 @@ public class SanitizedBridgesWriterTest {
this.configuration.setProperty(Key.RecentPath.name(),
recentDirectory.getAbsolutePath());
this.configuration.setProperty(Key.StatsPath.name(), statsDirectory);
- this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(),
+ this.configuration.setProperty(Key.BridgeLocalOrigins.name(),
bridgeDirectoriesDir);
this.configuration.setProperty(Key.OutputPath.name(),
sanitizedBridgesDirectory.toFile().getParent().toString());
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
index 8e255c6..3f9c231 100644
--- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.Observable;
import java.util.Observer;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
public class ConfigurationTest {
@@ -40,7 +41,7 @@ public class ConfigurationTest {
public void testKeyCount() throws Exception {
assertEquals("The number of properties keys in enum Key changed."
+ "\n This test class should be adapted.",
- 46, Key.values().length);
+ 50, Key.values().length);
}
@Test()
@@ -70,11 +71,11 @@ public class ConfigurationTest {
Configuration conf = new Configuration();
for (String input : arrays) {
conf.clear();
- conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), input);
+ conf.setProperty(Key.RelayCacheOrigins.name(), input);
assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: "
+ Arrays.toString(conf
- .getStringArray(Key.CachedRelayDescriptorsDirectories)),
- array, conf.getStringArray(Key.CachedRelayDescriptorsDirectories));
+ .getStringArray(Key.RelayCacheOrigins)),
+ array, conf.getStringArray(Key.RelayCacheOrigins));
}
}
@@ -82,10 +83,10 @@ public class ConfigurationTest {
public void testBoolValues() throws Exception {
Configuration conf = new Configuration();
conf.setProperty(Key.CompressRelayDescriptorDownloads.name(), "false");
- conf.setProperty(Key.ImportDirectoryArchives.name(), "trUe");
+ conf.setProperty(Key.RunOnce.name(), "trUe");
conf.setProperty(Key.ReplaceIpAddressesWithHashes.name(), "false");
assertFalse(conf.getBool(Key.CompressRelayDescriptorDownloads));
- assertTrue(conf.getBool(Key.ImportDirectoryArchives));
+ assertTrue(conf.getBool(Key.RunOnce));
assertFalse(conf.getBool(Key.ReplaceIpAddressesWithHashes));
}
@@ -117,6 +118,19 @@ public class ConfigurationTest {
}
@Test()
+ public void testSourceTypeValues() throws Exception {
+ String[] types = new String[] { "Local", "Cache", "Remote", "Sync"};
+ Configuration conf = new Configuration();
+ for (String type : types) {
+ conf.clear();
+ conf.setProperty(Key.BridgeSources.name(), type);
+ Set<SourceType> sts = conf.getSourceTypeSet(Key.BridgeSources);
+ assertEquals(1, sts.size());
+ assertTrue(sts.contains(SourceType.valueOf(type)));
+ }
+ }
+
+ @Test()
public void testArrayArrayValues() throws Exception {
String[][] sourceStrings = new String[][] {
new String[]{"localsource", "http://127.0.0.1:12345"},
@@ -129,17 +143,36 @@ public class ConfigurationTest {
conf.getStringArrayArray(Key.TorperfSources));
}
+ @Test()
+ public void testUrlArrayValues() throws Exception {
+ URL[] array = new URL[randomSource.nextInt(30) + 1];
+ for (int i = 0; i < array.length; i++) {
+ array[i] = new URL("https://"
+ + Integer.toBinaryString(randomSource.nextInt(100)) + ".dummy.org");
+ }
+ String input =
+ Arrays.toString(array).replace("[", "").replace("]", "")
+ .replaceAll(" ", "");
+ Configuration conf = new Configuration();
+ conf.clear();
+ conf.setProperty(Key.RelaySyncOrigins.name(), input);
+ assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: "
+ + Arrays.toString(conf
+ .getUrlArray(Key.RelaySyncOrigins)),
+ array, conf.getUrlArray(Key.RelaySyncOrigins));
+ }
+
@Test(expected = ConfigurationException.class)
public void testArrayArrayValueException() throws Exception {
Configuration conf = new Configuration();
- conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), "");
+ conf.setProperty(Key.RelayCacheOrigins.name(), "");
conf.getStringArrayArray(Key.OutputPath);
}
@Test(expected = ConfigurationException.class)
public void testArrayValueException() throws Exception {
Configuration conf = new Configuration();
- conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), "");
+ conf.setProperty(Key.RelayCacheOrigins.name(), "");
conf.getStringArray(Key.TorperfSources);
}
@@ -147,14 +180,14 @@ public class ConfigurationTest {
public void testBoolValueException() throws Exception {
Configuration conf = new Configuration();
conf.setProperty(Key.TorperfSources.name(), "http://x.y.z");
- conf.getBool(Key.CachedRelayDescriptorsDirectories);
+ conf.getBool(Key.RelayCacheOrigins);
}
@Test(expected = ConfigurationException.class)
public void testPathValueException() throws Exception {
Configuration conf = new Configuration();
- conf.setProperty(Key.DirectoryArchivesDirectory.name(), "\\\u0000:");
- conf.getPath(Key.DirectoryArchivesDirectory);
+ conf.setProperty(Key.RelayLocalOrigins.name(), "\\\u0000:");
+ conf.getPath(Key.RelayLocalOrigins);
}
@Test(expected = ConfigurationException.class)
@@ -214,7 +247,7 @@ public class ConfigurationTest {
conf.setWatchableSourceAndLoad(confFile.toPath());
MainTest.waitSec(1);
confFile.delete();
- conf.setProperty(Key.ImportDirectoryArchives.name(), "false");
+ conf.setProperty(Key.RunOnce.name(), "false");
final Dummy dummy = new Dummy(conf);
tmpf.newFolder("empty");
MainTest.waitSec(6);
More information about the tor-commits
mailing list