[tor-commits] [collector/master] Implements task-19021 and task-19005.
karsten at torproject.org
karsten at torproject.org
Mon Jun 6 20:44:22 UTC 2016
commit 8767c73d0826dfa9aa21e70a2d857c8a2d77e524
Author: iwakeh <iwakeh at torproject.org>
Date: Mon May 30 15:14:49 2016 +0200
Implements task-19021 and task-19005.
Adds the very first tests to CollecTor.
Increases testability and prepares task-19018.
Avoid using literal path separators,use Paths.get instead.
---
.gitignore | 1 +
build.xml | 30 +-
config.template | 107 -------
src/main/java/org/torproject/collector/Main.java | 73 ++++-
.../bridgedescs/SanitizedBridgesWriter.java | 53 ++--
.../torproject/collector/conf/Configuration.java | 123 ++++++++
.../collector/conf/ConfigurationException.java | 18 ++
.../java/org/torproject/collector/conf/Key.java | 55 ++++
.../collector/exitlists/ExitListDownloader.java | 32 ++-
.../collector/index/CreateIndexJson.java | 18 +-
.../torproject/collector/main/Configuration.java | 318 ---------------------
.../org/torproject/collector/main/LockFile.java | 20 +-
.../collector/relaydescs/ArchiveWriter.java | 188 ++++++------
.../relaydescs/CachedRelayDescriptorReader.java | 4 +-
.../relaydescs/RelayDescriptorDownloader.java | 9 +-
.../collector/torperf/TorperfDownloader.java | 59 ++--
src/main/resources/collector.properties | 115 ++++++++
.../java/org/torproject/collector/MainTest.java | 72 +++++
.../collector/conf/ConfigurationTest.java | 143 +++++++++
src/test/resources/junittest.policy | 10 +
20 files changed, 822 insertions(+), 626 deletions(-)
diff --git a/.gitignore b/.gitignore
index afab74f..0ca0b1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,4 +12,5 @@
/generated
/lib
cobertura.ser
+*~
diff --git a/build.xml b/build.xml
index 10c5edd..8e46584 100644
--- a/build.xml
+++ b/build.xml
@@ -53,10 +53,22 @@
<include name="logback-classic-1.1.2.jar" />
</fileset>
</path>
+ <path id="cobertura.test.classpath">
+ <path location="${instrument}" />
+ <path refid="test.classpath" />
+ <path refid="cobertura.classpath" />
+ </path>
<path id="test.classpath">
+ <pathelement path="${classes}"/>
<pathelement path="${testclasses}"/>
+ <pathelement path="${resources}"/>
+ <pathelement path="${testresources}"/>
+ <fileset dir="${libs}">
+ <patternset refid="runtime" />
+ </fileset>
<fileset dir="${libs}">
<include name="junit4-4.11.jar"/>
+ <include name="hamcrest-all-1.3.jar"/>
</fileset>
</path>
<target name="init">
@@ -65,7 +77,6 @@
<mkdir dir="${docs}"/>
<mkdir dir="${testresult}"/>
<mkdir dir="${instrument}"/>
- <copy file="config.template" tofile="config"/>
</target>
<target name="clean">
<delete includeEmptyDirs="true" quiet="true">
@@ -123,6 +134,7 @@
<jar destfile="${jarfile}"
basedir="${classes}">
<fileset dir="${classes}"/>
+ <fileset dir="${resources}" includes="collector.properties"/>
<zipgroupfileset dir="${libs}" >
<patternset refid="runtime" />
</zipgroupfileset>
@@ -185,10 +197,10 @@
<junit fork="true" haltonfailure="false" printsummary="on">
<sysproperty key="net.sourceforge.cobertura.datafile"
file="${cobertura.ser.file}" />
- <classpath location="${instrument}" />
- <classpath refid="classpath" />
- <classpath refid="test.classpath" />
- <classpath refid="cobertura.classpath" />
+ <!-- The following jvmargs prevent test access to the network. -->
+ <jvmarg value="-Djava.security.policy=${testresources}/junittest.policy"/>
+ <jvmarg value="-Djava.security.manager"/>
+ <classpath refid="cobertura.test.classpath" />
<formatter type="xml" />
<batchtest toDir="${testresult}" >
<fileset dir="${testclasses}" />
@@ -199,11 +211,15 @@
<include name="**/*.java" />
</fileset>
</cobertura-report>
- <cobertura-check branchrate="0" totallinerate="0" />
+ <cobertura-check branchrate="0" totallinerate="15" totalbranchrate="5" >
+ <regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/>
+ </cobertura-check>
</target>
<target name="test" depends="compile,compile-tests">
<junit fork="true" haltonfailure="true" printsummary="off">
- <classpath refid="classpath"/>
+ <!-- The following jvmargs prevent test access to the network. -->
+ <jvmarg value="-Djava.security.policy=${testresources}/junittest.policy"/>
+ <jvmarg value="-Djava.security.manager"/>
<classpath refid="test.classpath"/>
<formatter type="plain" usefile="false"/>
<batchtest>
diff --git a/config.template b/config.template
deleted file mode 100644
index 88407a2..0000000
--- a/config.template
+++ /dev/null
@@ -1,107 +0,0 @@
-######## Relay descriptors ########
-#
-## Read cached-* files from a local Tor data directory
-#ImportCachedRelayDescriptors 0
-#
-## Relative path to Tor data directory to read cached-* files from (can be
-## specified multiple times)
-#CachedRelayDescriptorsDirectory in/relay-descriptors/cacheddesc/
-#
-## Import directory archives from disk, if available
-#ImportDirectoryArchives 0
-#
-## Relative path to directory to import directory archives from
-#DirectoryArchivesDirectory in/relay-descriptors/archives/
-#
-## Keep a history of imported directory archive files to know which files
-## have been imported before. This history can be useful when importing
-## from a changing source to avoid importing descriptors over and over
-## again, but it can be confusing to users who don't know about it.
-#KeepDirectoryArchiveImportHistory 0
-#
-## Download relay descriptors from directory authorities, if required
-#DownloadRelayDescriptors 0
-#
-## Comma separated list of directory authority addresses (IP[:port]) to
-## download missing relay descriptors from
-#DownloadFromDirectoryAuthorities 86.59.21.38,76.73.17.194:9030,171.25.193.9:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170,154.35.32.5
-#
-## Comma separated list of directory authority fingerprints to download
-## votes
-#DownloadVotesByFingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,27B6B5996C426270A5C95488AA5BCEB6BCC86956,49015F787433103580E3B66A1707A00E60F2D15B,585769C78764D58426B8B52B6651A5A71137189A,80550987E1D626E3EBA5E5E75A458DE0626D088C,D586D18309DED4CD6D57C18FDB97EFA96D330566,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,ED03BB616EB2F60BEC80151114BB25CEF515B226,EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97
-#
-## Download the current consensus (only if DownloadRelayDescriptors is 1)
-#DownloadCurrentConsensus 1
-#
-## Download the current microdesc consensus (only if
-## DownloadRelayDescriptors is 1)
-#DownloadCurrentMicrodescConsensus 1
-#
-## Download current votes (only if DownloadRelayDescriptors is 1)
-#DownloadCurrentVotes 1
-#
-## Download missing server descriptors (only if DownloadRelayDescriptors
-## is 1)
-#DownloadMissingServerDescriptors 1
-#
-## Download missing extra-info descriptors (only if
-## DownloadRelayDescriptors is 1)
-#DownloadMissingExtraInfoDescriptors 1
-#
-## Download missing microdescriptors (only if
-## DownloadRelayDescriptors is 1)
-#DownloadMissingMicrodescriptors 1
-#
-## Download all server descriptors from the directory authorities at most
-## once a day (only if DownloadRelayDescriptors is 1)
-#DownloadAllServerDescriptors 0
-#
-## Download all extra-info descriptors from the directory authorities at
-## most once a day (only if DownloadRelayDescriptors is 1)
-#DownloadAllExtraInfoDescriptors 0
-#
-## Compress relay descriptors downloads by adding .z to the URLs
-#CompressRelayDescriptorDownloads 0
-#
-## Relative path to directory to write directory archives to
-#DirectoryArchivesOutputDirectory out/relay-descriptors/
-#
-#
-######## Bridge descriptors ########
-#
-## Relative path to directory to import bridge descriptor snapshots from
-#BridgeSnapshotsDirectory in/bridge-descriptors/
-#
-## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z
-## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we
-## can learn about IP address changes.
-#ReplaceIPAddressesWithHashes 0
-#
-## Limit internal bridge descriptor mapping state to the following number
-## of days, or -1 for unlimited.
-#LimitBridgeDescriptorMappings -1
-#
-## Relative path to directory to write sanitized bridges to
-#SanitizedBridgesWriteDirectory out/bridge-descriptors/
-#
-#
-######## Exit lists ########
-#
-## (No options available)
-#
-#
-######## Torperf downloader ########
-#
-## Relative path to the directory to store Torperf files in
-#TorperfOutputDirectory out/torperf/
-#
-## Torperf source names and base URLs (option can be contained multiple
-## times)
-#TorperfSource torperf http://torperf.torproject.org/
-#
-## Torperf measurement file size in bytes, .data file, and .extradata file
-## available on a given source (option can be contained multiple times)
-#TorperfFiles torperf 51200 50kb.data 50kb.extradata
-#TorperfFiles torperf 1048576 1mb.data 1mb.extradata
-#TorperfFiles torperf 5242880 5mb.data 5mb.extradata
-
diff --git a/src/main/java/org/torproject/collector/Main.java b/src/main/java/org/torproject/collector/Main.java
index 9c64696..d21cfb6 100644
--- a/src/main/java/org/torproject/collector/Main.java
+++ b/src/main/java/org/torproject/collector/Main.java
@@ -4,12 +4,19 @@
package org.torproject.collector;
import org.torproject.collector.bridgedescs.SanitizedBridgesWriter;
+import org.torproject.collector.conf.Configuration;
import org.torproject.collector.exitlists.ExitListDownloader;
import org.torproject.collector.index.CreateIndexJson;
import org.torproject.collector.relaydescs.ArchiveWriter;
import org.torproject.collector.torperf.TorperfDownloader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;
@@ -24,11 +31,12 @@ import java.util.logging.Logger;
public class Main {
private static Logger log = Logger.getLogger(Main.class.getName());
+ public static final String CONF_FILE = "collector.properties";
/** All possible main classes.
* If a new CollecTorMain class is available, just add it to this map.
*/
- private static final Map<String, Class> collecTorMains = new HashMap<>();
+ static final Map<String, Class> collecTorMains = new HashMap<>();
static { // add a new main class here
collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class);
@@ -41,38 +49,73 @@ public class Main {
private static final String modules = collecTorMains.keySet().toString()
.replace("[", "").replace("]", "").replaceAll(", ", "|");
+ private static Configuration conf = new Configuration();
+
/**
* One argument is necessary.
* See class description {@link Main}.
*/
- public static void main(String[] args) {
- if (null == args || args.length != 1) {
- printUsageAndExit("CollecTor needs exactly one argument.");
+ public static void main(String[] args) throws Exception {
+ File confFile = null;
+ if (null == args || args.length < 1 || args.length > 2) {
+ printUsage("CollecTor needs one or two arguments.");
+ return;
+ } else if (args.length == 1) {
+ confFile = new File(CONF_FILE);
+ } else if (args.length == 2) {
+ confFile = new File(args[1]);
+ }
+ if (!confFile.exists() || confFile.length() < 1L) {
+ writeDefaultConfig(confFile);
+ return;
} else {
- invokeGivenMainAndExit(args[0]);
+ readConfigurationFrom(confFile);
}
+ invokeGivenMain(args[0]);
}
- private static void printUsageAndExit(String msg) {
+ private static void printUsage(String msg) {
final String usage = "Usage:\njava -jar collector.jar "
- + "<" + modules + ">";
+ + "<" + modules + "> [path/to/configFile]";
System.out.println(msg + "\n" + usage);
- System.exit(0);
}
- private static void invokeGivenMainAndExit(String mainId) {
+ private static void writeDefaultConfig(File confFile) {
+ try {
+ Files.copy(Main.class.getClassLoader().getResource(CONF_FILE).openStream(),
+ confFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ printUsage("Could not find config file. In the default "
+ + "configuration, we are not configured to read data from any "
+ + "data source or write data to any data sink. You need to "
+ + "change the configuration (" + CONF_FILE
+ + ") and provide at least one data source and one data sink. "
+ + "Refer to the manual for more information.");
+ } catch (IOException e) {
+ log.severe("Cannot write default configuration. Reason: " + e);
+ }
+ }
+
+ private static void readConfigurationFrom(File confFile) throws Exception {
+ try (FileInputStream fis = new FileInputStream(confFile)) {
+ conf.load(fis);
+ } catch (Exception e) { // catch all possible problems
+ log.severe("Cannot read configuration. Reason: " + e);
+ throw e;
+ }
+ }
+
+ private static void invokeGivenMain(String mainId) {
Class clazz = collecTorMains.get(mainId);
if (null == clazz) {
- printUsageAndExit("Unknown argument: " + mainId);
+ printUsage("Unknown argument: " + mainId);
}
- invokeMainOnClassAndExit(clazz);
+ invokeMainOnClass(clazz);
}
- private static void invokeMainOnClassAndExit(Class clazz) {
+ private static void invokeMainOnClass(Class clazz) {
try {
- clazz.getMethod("main", new Class[] { String[].class })
- .invoke(null, (Object) new String[]{});
- System.exit(0);
+ clazz.getMethod("main", new Class[] { Configuration.class })
+ .invoke(null, (Object) conf);
} catch (NoSuchMethodException | IllegalAccessException
| InvocationTargetException e) {
log.severe("Cannot invoke 'main' method on "
diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 3214715..fa24a3d 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -3,7 +3,9 @@
package org.torproject.collector.bridgedescs;
-import org.torproject.collector.main.Configuration;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
import org.torproject.collector.main.LockFile;
import org.apache.commons.codec.DecoderException;
@@ -35,36 +37,30 @@ import java.util.logging.Level;
import java.util.logging.Logger;
/**
- * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
+ * <p>Sanitizes bridge descriptors, i.e., removes all possibly sensitive
* information from them, and writes them to a local directory structure.
* During the sanitizing process, all information about the bridge
* identity or IP address are removed or replaced. The goal is to keep the
* sanitized bridge descriptors useful for statistical analysis while not
- * making it easier for an adversary to enumerate bridges.
+ * making it easier for an adversary to enumerate bridges.</p>
*
- * There are three types of bridge descriptors: bridge network statuses
+ * <p>There are three types of bridge descriptors: bridge network statuses
* (lists of all bridges at a given time), server descriptors (published
* by the bridge to advertise their capabilities), and extra-info
- * descriptors (published by the bridge, mainly for statistical analysis).
+ * descriptors (published by the bridge, mainly for statistical analysis).</p>
*/
public class SanitizedBridgesWriter extends Thread {
- public static void main(String[] args) {
+ private static Logger logger;
- Logger logger = Logger.getLogger(
- SanitizedBridgesWriter.class.getName());
- logger.info("Starting bridge-descriptors module of CollecTor.");
+ public static void main(Configuration config) throws ConfigurationException {
- // Initialize configuration
- Configuration config = new Configuration();
+ logger = Logger.getLogger(SanitizedBridgesWriter.class.getName());
+ logger.info("Starting bridge-descriptors module of CollecTor.");
// Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("bridge-descriptors");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
+ LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "bridge-descriptors");
+ lf.acquireLock();
// Sanitize bridge descriptors
new SanitizedBridgesWriter(config).run();
@@ -84,11 +80,6 @@ public class SanitizedBridgesWriter extends Thread {
this.config = config;
}
- /**
- * Logger for this class.
- */
- private Logger logger;
-
private String rsyncCatString;
private File bridgeDirectoriesDirectory;
@@ -112,16 +103,26 @@ public class SanitizedBridgesWriter extends Thread {
private SecureRandom secureRandom;
+ @Override
public void run() {
+ try {
+ startProcessing();
+ } catch (ConfigurationException ce) {
+ logger.severe("Configuration failed: " + ce);
+ throw new RuntimeException(ce);
+ }
+ }
+
+ private void startProcessing() throws ConfigurationException {
File bridgeDirectoriesDirectory =
- new File(config.getBridgeSnapshotsDirectory());
+ config.getPath(Key.BridgeSnapshotsDirectory).toFile();
File sanitizedBridgesDirectory =
- new File(config.getSanitizedBridgesWriteDirectory());
+ config.getPath(Key.SanitizedBridgesWriteDirectory).toFile();
boolean replaceIPAddressesWithHashes =
- config.getReplaceIPAddressesWithHashes();
+ config.getBool(Key.ReplaceIPAddressesWithHashes);
long limitBridgeSanitizingInterval =
- config.getLimitBridgeDescriptorMappings();
+ config.getInt(Key.BridgeDescriptorMappingsLimit);
File statsDirectory = new File("stats");
if (bridgeDirectoriesDirectory == null
diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java
new file mode 100644
index 0000000..8b8cc12
--- /dev/null
+++ b/src/main/java/org/torproject/collector/conf/Configuration.java
@@ -0,0 +1,123 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.conf;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Initialize configuration with defaults from collector.properties,
+ * unless a configuration properties file is available.
+ */
+public class Configuration extends Properties {
+
+ public static final String FIELDSEP = ",";
+ public static final String ARRAYSEP = ";";
+
+ /**
+ * Returns {@code String[][]} from a property. Commas seperate array elements
+ * and semicolons separate arrays, e.g.,
+ * {@code propertyname = a1, a2, a3; b1, b2, b3}
+ */
+ public String[][] getStringArrayArray(Key key) throws ConfigurationException {
+ try {
+ checkClass(key, String[][].class);
+ String[] interim = getProperty(key.name()).split(ARRAYSEP);
+ String[][] res = new String[interim.length][];
+ for (int i = 0; i < interim.length; i++) {
+ res[i] = interim[i].trim().split(FIELDSEP);
+ for (int j = 0; j < res[i].length; j++) {
+ res[i][j] = res[i][j].trim();
+ }
+ }
+ return res;
+ } catch (RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
+ /**
+ * Returns {@code String[]} from a property. Commas seperate array elements,
+ * e.g.,
+ * {@code propertyname = a1, a2, a3}
+ */
+ public String[] getStringArray(Key key) throws ConfigurationException {
+ try {
+ checkClass(key, String[].class);
+ String[] res = getProperty(key.name()).split(FIELDSEP);
+ for (int i = 0; i < res.length; i++) {
+ res[i] = res[i].trim();
+ }
+ return res;
+ } catch (RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
+ private void checkClass(Key key, Class clazz) {
+ if (!key.keyClass().getSimpleName().equals(clazz.getSimpleName())) {
+ throw new RuntimeException("Wrong type wanted! My class is "
+ + key.keyClass().getSimpleName());
+ }
+ }
+
+ /**
+ * Returns a {@code boolean} property (case insensitiv), e.g.
+ * {@code propertyOne = True}.
+ */
+ public boolean getBool(Key key) throws ConfigurationException {
+ try {
+ checkClass(key, Boolean.class);
+ return Boolean.parseBoolean(getProperty(key.name()));
+ } catch (RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
+ /**
+ * Parse an integer property and translate the String
+ * <code>"inf"</code> into Integer.MAX_VALUE.
+ * Verifies that this enum is a Key for an integer value.
+ */
+ public int getInt(Key key) throws ConfigurationException {
+ try {
+ checkClass(key, Integer.class);
+ String prop = getProperty(key.name());
+ if ("inf".equals(prop)) {
+ return Integer.MAX_VALUE;
+ } else {
+ return Integer.parseInt(prop);
+ }
+ } catch (RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
+ /**
+ * Returns a {@code Path} property, e.g.
+ * {@code pathProperty = /my/path/file}.
+ */
+ public Path getPath(Key key) throws ConfigurationException {
+ try {
+ checkClass(key, Path.class);
+ return Paths.get(getProperty(key.name()));
+ } catch (RuntimeException re) {
+ throw new ConfigurationException("Corrupt property: " + key
+ + " reason: " + re.getMessage(), re);
+ }
+ }
+
+}
diff --git a/src/main/java/org/torproject/collector/conf/ConfigurationException.java b/src/main/java/org/torproject/collector/conf/ConfigurationException.java
new file mode 100644
index 0000000..730b1b3
--- /dev/null
+++ b/src/main/java/org/torproject/collector/conf/ConfigurationException.java
@@ -0,0 +1,18 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.conf;
+
+public class ConfigurationException extends Exception {
+
+ public ConfigurationException() {}
+
+ public ConfigurationException(String msg) {
+ super(msg);
+ }
+
+ public ConfigurationException(String msg, Exception ex) {
+ super(msg, ex);
+ }
+
+}
diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java
new file mode 100644
index 0000000..67f91c5
--- /dev/null
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -0,0 +1,55 @@
+package org.torproject.collector.conf;
+
+import java.nio.file.Path;
+
+/**
+ * Enum containing all the properties keys of the configuration.
+ * Specifies the key type.
+ */
+public enum Key {
+
+ LockFilePath(Path.class),
+ ArchivePath(Path.class),
+ RecentPath(Path.class),
+ IndexPath(Path.class),
+ StatsPath(Path.class),
+ BridgeSnapshotsDirectory(Path.class),
+ CachedRelayDescriptorsDirectories(String[].class),
+ CompressRelayDescriptorDownloads(Boolean.class),
+ DirectoryArchivesDirectory(Path.class),
+ DirectoryArchivesOutputDirectory(Path.class),
+ DownloadRelayDescriptors(Boolean.class),
+ DirectoryAuthoritiesAddresses(String[].class),
+ DirectoryAuthoritiesFingerprintsForVotes(String[].class),
+ DownloadCurrentConsensus(Boolean.class),
+ DownloadCurrentMicrodescConsensus(Boolean.class),
+ DownloadCurrentVotes(Boolean.class),
+ DownloadMissingServerDescriptors(Boolean.class),
+ DownloadMissingExtraInfoDescriptors(Boolean.class),
+ DownloadMissingMicrodescriptors(Boolean.class),
+ DownloadAllServerDescriptors(Boolean.class),
+ DownloadAllExtraInfoDescriptors(Boolean.class),
+ ImportCachedRelayDescriptors(Boolean.class),
+ ImportDirectoryArchives(Boolean.class),
+ KeepDirectoryArchiveImportHistory(Boolean.class),
+ ReplaceIPAddressesWithHashes(Boolean.class),
+ BridgeDescriptorMappingsLimit(Integer.class),
+ SanitizedBridgesWriteDirectory(Path.class),
+ TorperfOutputDirectory(Path.class),
+ TorperfFilesLines(String[].class),
+ TorperfSources(String[][].class);
+
+ private Class clazz;
+
+ /**
+ * @param Class of key value.
+ */
+ Key(Class clazz) {
+ this.clazz = clazz;
+ }
+
+ public Class keyClass() {
+ return clazz;
+ }
+
+}
diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
index 54fd50f..53fc300 100644
--- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -3,7 +3,9 @@
package org.torproject.collector.exitlists;
-import org.torproject.collector.main.Configuration;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
import org.torproject.collector.main.LockFile;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
@@ -31,21 +33,15 @@ import java.util.logging.Logger;
public class ExitListDownloader extends Thread {
- public static void main(String[] args) {
+ private static Logger logger =
+ Logger.getLogger(ExitListDownloader.class.getName());
- Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
+ public static void main(Configuration config) throws ConfigurationException {
logger.info("Starting exit-lists module of CollecTor.");
- // Initialize configuration
- Configuration config = new Configuration();
-
// Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("exit-lists");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
+ LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "exit-lists");
+ lf.acquireLock();
// Download exit list and store it to disk
new ExitListDownloader(config).run();
@@ -56,12 +52,18 @@ public class ExitListDownloader extends Thread {
logger.info("Terminating exit-lists module of CollecTor.");
}
- public ExitListDownloader(Configuration config) {
- }
+ public ExitListDownloader(Configuration config) {}
public void run() {
+ try {
+ startProcessing();
+ } catch (ConfigurationException ce) {
+ logger.severe("Configuration failed: " + ce);
+ throw new RuntimeException(ce);
+ }
+ }
- Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
+ private void startProcessing() throws ConfigurationException {
SimpleDateFormat dateTimeFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
index ac5adf5..de69488 100644
--- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java
+++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
@@ -3,6 +3,10 @@
package org.torproject.collector.index;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
+
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -33,12 +37,11 @@ import java.util.zip.GZIPOutputStream;
* we'll likely have to do that. */
public class CreateIndexJson {
- static final File indexJsonFile = new File("index.json");
+ private static File indexJsonFile;
- static final String basePath = "https://collector.torproject.org";
+ private static String basePath = "https://collector.torproject.org";
- static final File[] indexedDirectories = new File[] {
- new File("archive"), new File("recent") };
+ private static File[] indexedDirectories;
static final String dateTimePattern = "yyyy-MM-dd HH:mm";
@@ -46,7 +49,12 @@ public class CreateIndexJson {
static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");
- public static void main(String[] args) throws IOException {
+ public static void main(Configuration config)
+ throws ConfigurationException, IOException {
+ indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json");
+ indexedDirectories = new File[] {
+ new File(config.getPath(Key.ArchivePath).toFile(), "archive"),
+ new File(config.getPath(Key.RecentPath).toFile(), "recent") };
writeIndex(indexDirectories());
}
diff --git a/src/main/java/org/torproject/collector/main/Configuration.java b/src/main/java/org/torproject/collector/main/Configuration.java
deleted file mode 100644
index aee1d02..0000000
--- a/src/main/java/org/torproject/collector/main/Configuration.java
+++ /dev/null
@@ -1,318 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.main;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Initialize configuration with hard-coded defaults, overwrite with
- * configuration in config file, if exists, and answer Main.java about our
- * configuration.
- */
-public class Configuration {
- private String directoryArchivesOutputDirectory =
- "out/relay-descriptors/";
- private boolean importCachedRelayDescriptors = false;
- private List<String> cachedRelayDescriptorsDirectory =
- new ArrayList<String>(Arrays.asList(
- "in/relay-descriptors/cacheddesc/".split(",")));
- private boolean importDirectoryArchives = false;
- private String directoryArchivesDirectory =
- "in/relay-descriptors/archives/";
- private boolean keepDirectoryArchiveImportHistory = false;
- private boolean replaceIPAddressesWithHashes = false;
- private long limitBridgeDescriptorMappings = -1L;
- private String sanitizedBridgesWriteDirectory =
- "out/bridge-descriptors/";
- private String bridgeSnapshotsDirectory = "in/bridge-descriptors/";
- private boolean downloadRelayDescriptors = false;
- private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
- "86.59.21.38,76.73.17.194:9030,171.25.193.9:443,"
- + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
- + "194.109.206.212,212.112.245.170,154.35.32.5").split(","));
- private List<String> downloadVotesByFingerprint = Arrays.asList((
- "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
- + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
- + "49015F787433103580E3B66A1707A00E60F2D15B,"
- + "585769C78764D58426B8B52B6651A5A71137189A,"
- + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
- + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
- + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
- + "ED03BB616EB2F60BEC80151114BB25CEF515B226,"
- + "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97").split(","));
- private boolean downloadCurrentConsensus = true;
- private boolean downloadCurrentMicrodescConsensus = true;
- private boolean downloadCurrentVotes = true;
- private boolean downloadMissingServerDescriptors = true;
- private boolean downloadMissingExtraInfoDescriptors = true;
- private boolean downloadMissingMicrodescriptors = true;
- private boolean downloadAllServerDescriptors = false;
- private boolean downloadAllExtraInfoDescriptors = false;
- private boolean compressRelayDescriptorDownloads;
- private String torperfOutputDirectory = "out/torperf/";
- private SortedMap<String, String> torperfSources = null;
- private List<String> torperfFiles = null;
-
- public Configuration() {
-
- /* Initialize logger. */
- Logger logger = Logger.getLogger(Configuration.class.getName());
-
- /* Read config file, if present. */
- File configFile = new File("config");
- if (!configFile.exists()) {
- logger.warning("Could not find config file. In the default "
- + "configuration, we are not configured to read data from any "
- + "data source or write data to any data sink. You need to "
- + "create a config file (" + configFile.getAbsolutePath()
- + ") and provide at least one data source and one data sink. "
- + "Refer to the manual for more information.");
- return;
- }
- String line = null;
- boolean containsCachedRelayDescriptorsDirectory = false;
- try {
- BufferedReader br = new BufferedReader(new FileReader(configFile));
- while ((line = br.readLine()) != null) {
- if (line.startsWith("#") || line.length() < 1) {
- continue;
- } else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
- this.directoryArchivesOutputDirectory = line.split(" ")[1];
- } else if (line.startsWith("ImportCachedRelayDescriptors")) {
- this.importCachedRelayDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
- if (!containsCachedRelayDescriptorsDirectory) {
- this.cachedRelayDescriptorsDirectory.clear();
- containsCachedRelayDescriptorsDirectory = true;
- }
- this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
- } else if (line.startsWith("ImportDirectoryArchives")) {
- this.importDirectoryArchives = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DirectoryArchivesDirectory")) {
- this.directoryArchivesDirectory = line.split(" ")[1];
- } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
- this.keepDirectoryArchiveImportHistory = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
- this.replaceIPAddressesWithHashes = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
- this.limitBridgeDescriptorMappings = Long.parseLong(
- line.split(" ")[1]);
- } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
- this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
- } else if (line.startsWith("BridgeSnapshotsDirectory")) {
- this.bridgeSnapshotsDirectory = line.split(" ")[1];
- } else if (line.startsWith("DownloadRelayDescriptors")) {
- this.downloadRelayDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadFromDirectoryAuthorities")) {
- this.downloadFromDirectoryAuthorities = new ArrayList<String>();
- for (String dir : line.split(" ")[1].split(",")) {
- // test if IP:port pair has correct format
- if (dir.length() < 1) {
- logger.severe("Configuration file contains directory "
- + "authority IP:port of length 0 in line '" + line
- + "'! Exiting!");
- System.exit(1);
- }
- new URL("http://" + dir + "/");
- this.downloadFromDirectoryAuthorities.add(dir);
- }
- } else if (line.startsWith("DownloadVotesByFingerprint")) {
- this.downloadVotesByFingerprint = new ArrayList<String>();
- for (String fingerprint : line.split(" ")[1].split(",")) {
- this.downloadVotesByFingerprint.add(fingerprint);
- }
- } else if (line.startsWith("DownloadCurrentConsensus")) {
- this.downloadCurrentConsensus = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadCurrentMicrodescConsensus")) {
- this.downloadCurrentMicrodescConsensus = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadCurrentVotes")) {
- this.downloadCurrentVotes = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadMissingServerDescriptors")) {
- this.downloadMissingServerDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith(
- "DownloadMissingExtraInfoDescriptors")) {
- this.downloadMissingExtraInfoDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadMissingMicrodescriptors")) {
- this.downloadMissingMicrodescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadAllServerDescriptors")) {
- this.downloadAllServerDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) {
- this.downloadAllExtraInfoDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("CompressRelayDescriptorDownloads")) {
- this.compressRelayDescriptorDownloads = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("TorperfOutputDirectory")) {
- this.torperfOutputDirectory = line.split(" ")[1];
- } else if (line.startsWith("TorperfSource")) {
- if (this.torperfSources == null) {
- this.torperfSources = new TreeMap<String, String>();
- }
- String[] parts = line.split(" ");
- String sourceName = parts[1];
- String baseUrl = parts[2];
- this.torperfSources.put(sourceName, baseUrl);
- } else if (line.startsWith("TorperfFiles")) {
- if (this.torperfFiles == null) {
- this.torperfFiles = new ArrayList<String>();
- }
- String[] parts = line.split(" ");
- if (parts.length != 5) {
- logger.severe("Configuration file contains TorperfFiles "
- + "option with wrong number of values in line '" + line
- + "'! Exiting!");
- System.exit(1);
- }
- this.torperfFiles.add(line);
- } else {
- logger.severe("Configuration file contains unrecognized "
- + "configuration key in line '" + line + "'! Exiting!");
- System.exit(1);
- }
- }
- br.close();
- } catch (ArrayIndexOutOfBoundsException e) {
- logger.severe("Configuration file contains configuration key "
- + "without value in line '" + line + "'. Exiting!");
- System.exit(1);
- } catch (MalformedURLException e) {
- logger.severe("Configuration file contains illegal URL or IP:port "
- + "pair in line '" + line + "'. Exiting!");
- System.exit(1);
- } catch (NumberFormatException e) {
- logger.severe("Configuration file contains illegal value in line '"
- + line + "' with legal values being 0 or 1. Exiting!");
- System.exit(1);
- } catch (IOException e) {
- logger.log(Level.SEVERE, "Unknown problem while reading config "
- + "file! Exiting!", e);
- System.exit(1);
- }
- }
-
- public String getDirectoryArchivesOutputDirectory() {
- return this.directoryArchivesOutputDirectory;
- }
-
- public boolean getImportCachedRelayDescriptors() {
- return this.importCachedRelayDescriptors;
- }
-
- public List<String> getCachedRelayDescriptorDirectory() {
- return this.cachedRelayDescriptorsDirectory;
- }
-
- public boolean getImportDirectoryArchives() {
- return this.importDirectoryArchives;
- }
-
- public String getDirectoryArchivesDirectory() {
- return this.directoryArchivesDirectory;
- }
-
- public boolean getKeepDirectoryArchiveImportHistory() {
- return this.keepDirectoryArchiveImportHistory;
- }
-
- public boolean getReplaceIPAddressesWithHashes() {
- return this.replaceIPAddressesWithHashes;
- }
-
- public long getLimitBridgeDescriptorMappings() {
- return this.limitBridgeDescriptorMappings;
- }
-
- public String getSanitizedBridgesWriteDirectory() {
- return this.sanitizedBridgesWriteDirectory;
- }
-
- public String getBridgeSnapshotsDirectory() {
- return this.bridgeSnapshotsDirectory;
- }
-
- public boolean getDownloadRelayDescriptors() {
- return this.downloadRelayDescriptors;
- }
-
- public List<String> getDownloadFromDirectoryAuthorities() {
- return this.downloadFromDirectoryAuthorities;
- }
-
- public List<String> getDownloadVotesByFingerprint() {
- return this.downloadVotesByFingerprint;
- }
-
- public boolean getDownloadCurrentConsensus() {
- return this.downloadCurrentConsensus;
- }
-
- public boolean getDownloadCurrentMicrodescConsensus() {
- return this.downloadCurrentMicrodescConsensus;
- }
-
- public boolean getDownloadCurrentVotes() {
- return this.downloadCurrentVotes;
- }
-
- public boolean getDownloadMissingServerDescriptors() {
- return this.downloadMissingServerDescriptors;
- }
-
- public boolean getDownloadMissingExtraInfoDescriptors() {
- return this.downloadMissingExtraInfoDescriptors;
- }
-
- public boolean getDownloadMissingMicrodescriptors() {
- return this.downloadMissingMicrodescriptors;
- }
-
- public boolean getDownloadAllServerDescriptors() {
- return this.downloadAllServerDescriptors;
- }
-
- public boolean getDownloadAllExtraInfoDescriptors() {
- return this.downloadAllExtraInfoDescriptors;
- }
-
- public boolean getCompressRelayDescriptorDownloads() {
- return this.compressRelayDescriptorDownloads;
- }
-
- public String getTorperfOutputDirectory() {
- return this.torperfOutputDirectory;
- }
-
- public SortedMap<String, String> getTorperfSources() {
- return this.torperfSources;
- }
-
- public List<String> getTorperfFiles() {
- return this.torperfFiles;
- }
-}
-
diff --git a/src/main/java/org/torproject/collector/main/LockFile.java b/src/main/java/org/torproject/collector/main/LockFile.java
index b07d4b1..f168bc3 100644
--- a/src/main/java/org/torproject/collector/main/LockFile.java
+++ b/src/main/java/org/torproject/collector/main/LockFile.java
@@ -13,12 +13,17 @@ import java.util.logging.Logger;
public class LockFile {
- private File lockFile;
- private Logger logger;
+ private final File lockFile;
+ private final String moduleName;
+ private final Logger logger = Logger.getLogger(LockFile.class.getName());
public LockFile(String moduleName) {
- this.lockFile = new File("lock/" + moduleName);
- this.logger = Logger.getLogger(LockFile.class.getName());
+ this("lock", moduleName);
+ }
+
+ public LockFile(String lockFilePath, String moduleName) {
+ this.lockFile = new File(lockFilePath, moduleName);
+ this.moduleName = moduleName;
}
public boolean acquireLock() {
@@ -30,7 +35,7 @@ public class LockFile {
long runStarted = Long.parseLong(br.readLine());
br.close();
if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) {
- return false;
+ throw new RuntimeException("Cannot acquire lock for " + moduleName);
}
}
this.lockFile.getParentFile().mkdirs();
@@ -41,9 +46,8 @@ public class LockFile {
this.logger.fine("Acquired lock.");
return true;
} catch (IOException e) {
- this.logger.warning("Caught exception while trying to acquire "
- + "lock!");
- return false;
+ throw new RuntimeException("Caught exception while trying to acquire "
+ + "lock for " + moduleName);
}
}
diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
index cf603d1..43c7975 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -3,7 +3,9 @@
package org.torproject.collector.relaydescs;
-import org.torproject.collector.main.Configuration;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
import org.torproject.collector.main.LockFile;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorParser;
@@ -17,6 +19,8 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
@@ -36,11 +40,12 @@ import java.util.logging.Logger;
public class ArchiveWriter extends Thread {
+ private static Logger logger = Logger.getLogger(ArchiveWriter.class.getName());
+
private Configuration config;
private long now = System.currentTimeMillis();
- private Logger logger;
- private File outputDirectory;
+ private String outputDirectory;
private String rsyncCatString;
private DescriptorParser descriptorParser;
private int storedConsensusesCounter = 0;
@@ -67,12 +72,9 @@ public class ArchiveWriter extends Thread {
private SortedMap<Long, Set<String>> storedMicrodescriptors =
new TreeMap<Long, Set<String>>();
- private File storedServerDescriptorsFile = new File(
- "stats/stored-server-descriptors");
- private File storedExtraInfoDescriptorsFile = new File(
- "stats/stored-extra-info-descriptors");
- private File storedMicrodescriptorsFile = new File(
- "stats/stored-microdescriptors");
+ private File storedServerDescriptorsFile;
+ private File storedExtraInfoDescriptorsFile;
+ private File storedMicrodescriptorsFile;
private static final byte[] CONSENSUS_ANNOTATION =
"@type network-status-consensus-3 1.0\n".getBytes();
@@ -97,28 +99,31 @@ public class ArchiveWriter extends Thread {
private StringBuilder intermediateStats = new StringBuilder();
- public static void main(String[] args) {
+ private static Path recentPath;
+ private static String recentPathName;
+ private static final String RELAY_DESCRIPTORS = "relay-descriptors";
+ private static final String MICRO = "micro";
+ private static final String CONSENSUS_MICRODESC = "consensus-microdesc";
+ private static final String MICRODESC = "microdesc";
+ private static final String MICRODESCS = "microdescs";
+ public static void main(Configuration config) throws ConfigurationException {
- Logger logger = Logger.getLogger(ArchiveWriter.class.getName());
logger.info("Starting relay-descriptors module of CollecTor.");
- // Initialize configuration
- Configuration config = new Configuration();
-
// Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("relay-descriptors");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
+ LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), RELAY_DESCRIPTORS);
+ lf.acquireLock();
+
+ recentPath = config.getPath(Key.RecentPath);
+ recentPathName = recentPath.toString();
// Import/download relay descriptors from the various sources
new ArchiveWriter(config).run();
- new ReferenceChecker(new File("recent/relay-descriptors"),
- new File("stats/references"),
- new File("stats/references-history")).check();
+ new ReferenceChecker(
+ recentPath.toFile(),
+ new File(config.getPath(Key.StatsPath).toFile(), "references"),
+ new File(config.getPath(Key.StatsPath).toFile(), "references-history")).check();
// Remove lock file
lf.releaseLock();
@@ -126,18 +131,29 @@ public class ArchiveWriter extends Thread {
logger.info("Terminating relay-descriptors module of CollecTor.");
}
- public ArchiveWriter(Configuration config) {
+ public ArchiveWriter(Configuration config) throws ConfigurationException {
this.config = config;
+ storedServerDescriptorsFile =
+ new File(config.getPath(Key.StatsPath).toFile(), "stored-server-descriptors");
+ storedExtraInfoDescriptorsFile =
+ new File(config.getPath(Key.StatsPath).toFile(), "stored-extra-info-descriptors");
+ storedMicrodescriptorsFile =
+ new File(config.getPath(Key.StatsPath).toFile(), "stored-microdescriptors");
}
public void run() {
+ try {
+ startProcessing();
+ } catch (ConfigurationException ce) {
+ logger.severe("Configuration failed: " + ce);
+ throw new RuntimeException(ce);
+ }
+ }
- File outputDirectory =
- new File(config.getDirectoryArchivesOutputDirectory());
- File statsDirectory = new File("stats");
+ private void startProcessing() throws ConfigurationException {
- this.logger = Logger.getLogger(ArchiveWriter.class.getName());
- this.outputDirectory = outputDirectory;
+ File statsDirectory = new File("stats");
+ this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString();
SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
"yyyy-MM-dd-HH-mm-ss");
rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -152,33 +168,33 @@ public class ArchiveWriter extends Thread {
RelayDescriptorParser rdp = new RelayDescriptorParser(this);
RelayDescriptorDownloader rdd = null;
- if (config.getDownloadRelayDescriptors()) {
- List<String> dirSources =
- config.getDownloadFromDirectoryAuthorities();
+ if (config.getBool(Key.DownloadRelayDescriptors)) {
+ String[] dirSources =
+ config.getStringArray(Key.DirectoryAuthoritiesAddresses);
rdd = new RelayDescriptorDownloader(rdp, dirSources,
- config.getDownloadVotesByFingerprint(),
- config.getDownloadCurrentConsensus(),
- config.getDownloadCurrentMicrodescConsensus(),
- config.getDownloadCurrentVotes(),
- config.getDownloadMissingServerDescriptors(),
- config.getDownloadMissingExtraInfoDescriptors(),
- config.getDownloadMissingMicrodescriptors(),
- config.getDownloadAllServerDescriptors(),
- config.getDownloadAllExtraInfoDescriptors(),
- config.getCompressRelayDescriptorDownloads());
+ config.getStringArray(Key.DirectoryAuthoritiesFingerprintsForVotes),
+ config.getBool(Key.DownloadCurrentConsensus),
+ config.getBool(Key.DownloadCurrentMicrodescConsensus),
+ config.getBool(Key.DownloadCurrentVotes),
+ config.getBool(Key.DownloadMissingServerDescriptors),
+ config.getBool(Key.DownloadMissingExtraInfoDescriptors),
+ config.getBool(Key.DownloadMissingMicrodescriptors),
+ config.getBool(Key.DownloadAllServerDescriptors),
+ config.getBool(Key.DownloadAllExtraInfoDescriptors),
+ config.getBool(Key.CompressRelayDescriptorDownloads));
rdp.setRelayDescriptorDownloader(rdd);
}
- if (config.getImportCachedRelayDescriptors()) {
+ if (config.getBool(Key.ImportCachedRelayDescriptors)) {
new CachedRelayDescriptorReader(rdp,
- config.getCachedRelayDescriptorDirectory(), statsDirectory);
+ config.getStringArray(Key.CachedRelayDescriptorsDirectories), statsDirectory);
this.intermediateStats("importing relay descriptors from local "
+ "Tor data directories");
}
- if (config.getImportDirectoryArchives()) {
+ if (config.getBool(Key.ImportDirectoryArchives)) {
new ArchiveReader(rdp,
- new File(config.getDirectoryArchivesDirectory()),
+ config.getPath(Key.DirectoryArchivesDirectory).toFile(),
statsDirectory,
- config.getKeepDirectoryArchiveImportHistory());
+ config.getBool(Key.KeepDirectoryArchiveImportHistory));
this.intermediateStats("importing relay descriptors from local "
+ "directory");
}
@@ -557,7 +573,7 @@ public class ArchiveWriter extends Thread {
- 3L * 24L * 60L * 60L * 1000L;
long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/relay-descriptors"));
+ allFiles.add(new File(recentPathName, RELAY_DESCRIPTORS));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
@@ -633,11 +649,11 @@ public class ArchiveWriter extends Thread {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus");
+ File tarballFile = Paths.get(this.outputDirectory, "consensus",
+ printFormat.format(new Date(validAfter)) + "-consensus").toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncFile = new File("recent/relay-descriptors/consensuses/"
- + tarballFile.getName());
+ File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
+ "consensuses", tarballFile.getName()).toFile();
File[] outputFiles = new File[] { tarballFile, rsyncFile };
if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) {
this.storedConsensusesCounter++;
@@ -657,14 +673,12 @@ public class ArchiveWriter extends Thread {
SimpleDateFormat dayDirectoryFileFormat = new SimpleDateFormat(
"dd/yyyy-MM-dd-HH-mm-ss");
dayDirectoryFileFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory
- + "/microdesc/" + yearMonthDirectoryFormat.format(validAfter)
- + "/consensus-microdesc/"
- + dayDirectoryFileFormat.format(validAfter)
- + "-consensus-microdesc");
+ File tarballFile = Paths.get(this.outputDirectory, MICRODESC,
+ yearMonthDirectoryFormat.format(validAfter), CONSENSUS_MICRODESC,
+ dayDirectoryFileFormat.format(validAfter) + "-consensus-microdesc").toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncFile = new File("recent/relay-descriptors/microdescs/"
- + "consensus-microdesc/" + tarballFile.getName());
+ File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, MICRODESCS,
+ CONSENSUS_MICRODESC, tarballFile.getName()).toFile();
File[] outputFiles = new File[] { tarballFile, rsyncFile };
if (this.store(MICRODESCCONSENSUS_ANNOTATION, data, outputFiles,
null)) {
@@ -683,12 +697,12 @@ public class ArchiveWriter extends Thread {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest);
+ File tarballFile = Paths.get(this.outputDirectory, "vote",
+ printFormat.format(new Date(validAfter)) + "-vote-"
+ + fingerprint + "-" + digest).toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncFile = new File("recent/relay-descriptors/votes/"
- + tarballFile.getName());
+ File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "votes",
+ tarballFile.getName()).toFile();
File[] outputFiles = new File[] { tarballFile, rsyncFile };
if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) {
this.storedVotesCounter++;
@@ -709,8 +723,8 @@ public class ArchiveWriter extends Thread {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/certs/"
- + fingerprint + "-" + printFormat.format(new Date(published)));
+ File tarballFile = Paths.get(this.outputDirectory, "certs",
+ fingerprint + "-" + printFormat.format(new Date(published))).toFile();
File[] outputFiles = new File[] { tarballFile };
if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles, null)) {
this.storedCertsCounter++;
@@ -721,14 +735,13 @@ public class ArchiveWriter extends Thread {
long published, String extraInfoDigest) {
SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory
- + "/server-descriptor/" + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
- + digest);
+ File tarballFile = Paths.get(this.outputDirectory,
+ "server-descriptor", printFormat.format(new Date(published)),
+ digest.substring(0, 1), digest.substring(1, 2), digest).toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncCatFile = new File("recent/relay-descriptors/"
- + "server-descriptors/" + this.rsyncCatString
- + "-server-descriptors.tmp");
+ File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
+ "server-descriptors",
+ this.rsyncCatString + "-server-descriptors.tmp").toFile();
File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
boolean[] append = new boolean[] { false, true };
if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles,
@@ -750,14 +763,14 @@ public class ArchiveWriter extends Thread {
String extraInfoDigest, long published) {
SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest);
+ File tarballFile = Paths.get(this.outputDirectory, "extra-info",
+ descriptorFormat.format(new Date(published)),
+ extraInfoDigest.substring(0, 1),
+ extraInfoDigest.substring(1, 2),
+ extraInfoDigest).toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncCatFile = new File("recent/relay-descriptors/"
- + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
+ File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
+ "extra-infos", this.rsyncCatString + "-extra-infos.tmp").toFile();
File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
boolean[] append = new boolean[] { false, true };
if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) {
@@ -784,15 +797,14 @@ public class ArchiveWriter extends Thread {
* valid-after months. */
SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/microdesc/"
- + descriptorFormat.format(validAfter) + "micro/"
- + microdescriptorDigest.substring(0, 1) + "/"
- + microdescriptorDigest.substring(1, 2) + "/"
- + microdescriptorDigest);
+ File tarballFile = Paths.get(this.outputDirectory, MICRODESC,
+ descriptorFormat.format(validAfter), MICRO,
+ microdescriptorDigest.substring(0, 1),
+ microdescriptorDigest.substring(1, 2),
+ microdescriptorDigest).toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncCatFile = new File("recent/relay-descriptors/"
- + "microdescs/micro/" + this.rsyncCatString
- + "-micro.tmp");
+ File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
+ MICRODESCS, MICRO, this.rsyncCatString + "-micro.tmp").toFile();
File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
boolean[] append = new boolean[] { false, true };
if (this.store(MICRODESCRIPTOR_ANNOTATION, data, outputFiles,
diff --git a/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
index b9001dd..00eeab1 100644
--- a/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
+++ b/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -35,10 +35,10 @@ import java.util.logging.Logger;
*/
public class CachedRelayDescriptorReader {
public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
- List<String> inputDirectories, File statsDirectory) {
+ String[] inputDirectories, File statsDirectory) {
if (rdp == null || inputDirectories == null
- || inputDirectories.isEmpty() || statsDirectory == null) {
+ || inputDirectories.length == 0 || statsDirectory == null) {
throw new IllegalArgumentException();
}
diff --git a/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
index 458332a..bd0a482 100644
--- a/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
+++ b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
@@ -19,7 +19,7 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -288,7 +288,7 @@ public class RelayDescriptorDownloader {
* <code>stats/last-downloaded-all-descriptors</code>.
*/
public RelayDescriptorDownloader(RelayDescriptorParser rdp,
- List<String> authorities, List<String> authorityFingerprints,
+ String[] authorities, String[] authorityFingerprints,
boolean downloadCurrentConsensus,
boolean downloadCurrentMicrodescConsensus,
boolean downloadCurrentVotes,
@@ -300,9 +300,8 @@ public class RelayDescriptorDownloader {
/* Memorize argument values. */
this.rdp = rdp;
- this.authorities = new ArrayList<String>(authorities);
- this.authorityFingerprints = new ArrayList<String>(
- authorityFingerprints);
+ this.authorities = Arrays.asList(authorities);
+ this.authorityFingerprints = Arrays.asList(authorityFingerprints);
this.downloadCurrentConsensus = downloadCurrentConsensus;
this.downloadCurrentMicrodescConsensus =
downloadCurrentMicrodescConsensus;
diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
index 7bcfbf3..c80f99e 100644
--- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
+++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
@@ -3,7 +3,9 @@
package org.torproject.collector.torperf;
-import org.torproject.collector.main.Configuration;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
import org.torproject.collector.main.LockFile;
import java.io.BufferedReader;
@@ -17,6 +19,7 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
@@ -30,22 +33,14 @@ import java.util.logging.Logger;
* configured sources, append them to the files we already have, and merge
* the two files into the .tpf format. */
public class TorperfDownloader extends Thread {
+ private static Logger logger = Logger.getLogger(TorperfDownloader.class.getName());
- public static void main(String[] args) {
-
- Logger logger = Logger.getLogger(TorperfDownloader.class.getName());
+ public static void main(Configuration config) throws ConfigurationException {
logger.info("Starting torperf module of CollecTor.");
- // Initialize configuration
- Configuration config = new Configuration();
-
// Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("torperf");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
+ LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "torperf");
+ lf.acquireLock();
// Process Torperf files
new TorperfDownloader(config).run();
@@ -63,30 +58,34 @@ public class TorperfDownloader extends Thread {
}
private File torperfOutputDirectory = null;
- private SortedMap<String, String> torperfSources = null;
- private List<String> torperfFilesLines = null;
- private Logger logger = null;
+ private Map<String, String> torperfSources = new HashMap<>();
+ private String[] torperfFilesLines = null;
private SimpleDateFormat dateFormat;
public void run() {
+ try {
+ startProcessing();
+ } catch (ConfigurationException ce) {
+ logger.severe("Configuration failed: " + ce);
+ throw new RuntimeException(ce);
+ }
+ }
- File torperfOutputDirectory =
- new File(config.getTorperfOutputDirectory());
- SortedMap<String, String> torperfSources = config.getTorperfSources();
- List<String> torperfFilesLines = config.getTorperfFiles();
-
- this.torperfOutputDirectory = torperfOutputDirectory;
- this.torperfSources = torperfSources;
- this.torperfFilesLines = torperfFilesLines;
+ private void startProcessing() throws ConfigurationException {
+ this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines);
+ this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory)
+ .toFile();
if (!this.torperfOutputDirectory.exists()) {
this.torperfOutputDirectory.mkdirs();
}
- this.logger = Logger.getLogger(TorperfDownloader.class.getName());
this.dateFormat = new SimpleDateFormat("yyyy-MM-dd");
this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
this.readLastMergedTimestamps();
+ for (String[] source : config.getStringArrayArray(Key.TorperfSources)) {
+ torperfSources.put(source[0], source[1]);
+ }
for (String torperfFilesLine : this.torperfFilesLines) {
- this.downloadAndMergeFiles(torperfFilesLine);
+ this.downloadAndMergeFiles(torperfFilesLine);
}
this.writeLastMergedTimestamps();
@@ -161,10 +160,10 @@ public class TorperfDownloader extends Thread {
private void downloadAndMergeFiles(String torperfFilesLine) {
String[] parts = torperfFilesLine.split(" ");
- String sourceName = parts[1];
+ String sourceName = parts[0];
int fileSize = -1;
try {
- fileSize = Integer.parseInt(parts[2]);
+ fileSize = Integer.parseInt(parts[1]);
} catch (NumberFormatException e) {
this.logger.log(Level.WARNING, "Could not parse file size in "
+ "TorperfFiles configuration line '" + torperfFilesLine
@@ -173,7 +172,7 @@ public class TorperfDownloader extends Thread {
}
/* Download and append the .data file. */
- String dataFileName = parts[3];
+ String dataFileName = parts[2];
String sourceBaseUrl = torperfSources.get(sourceName);
String dataUrl = sourceBaseUrl + dataFileName;
String dataOutputFileName = sourceName + "-" + dataFileName;
@@ -183,7 +182,7 @@ public class TorperfDownloader extends Thread {
dataOutputFile, true);
/* Download and append the .extradata file. */
- String extradataFileName = parts[4];
+ String extradataFileName = parts[3];
String extradataUrl = sourceBaseUrl + extradataFileName;
String extradataOutputFileName = sourceName + "-" + extradataFileName;
File extradataOutputFile = new File(torperfOutputDirectory,
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
new file mode 100644
index 0000000..2645d01
--- /dev/null
+++ b/src/main/resources/collector.properties
@@ -0,0 +1,115 @@
+######## Collector Properties
+#
+######## General Properties ########
+LockFilePath = lock
+IndexPath = out/index
+ArchivePath = out/archive
+RecentPath = out/recent
+StatsPath = out/stats
+
+######## Relay descriptors ########
+#
+## Read cached-* files from a local Tor data directory
+ImportCachedRelayDescriptors = false
+#
+## Relative path to Tor data directory to read cached-* files from
+## the listed path(s). If there is more that one separated by comma.
+CachedRelayDescriptorsDirectories = in/relay-descriptors/cacheddesc/
+#
+## Import directory archives from disk, if available
+ImportDirectoryArchives = false
+#
+## Relative path to directory to import directory archives from
+DirectoryArchivesDirectory = in/relay-descriptors/archives/
+#
+## Keep a history of imported directory archive files to know which files
+## have been imported before. This history can be useful when importing
+## from a changing source to avoid importing descriptors over and over
+## again, but it can be confusing to users who don't know about it.
+KeepDirectoryArchiveImportHistory = false
+#
+## Download relay descriptors from directory authorities, if required
+DownloadRelayDescriptors = false
+#
+## Comma separated list of directory authority addresses (IP[:port]) to
+## download missing relay descriptors from
+DirectoryAuthoritiesAddresses = 86.59.21.38,76.73.17.194:9030,171.25.193.9:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170,154.35.32.5
+#
+## Comma separated list of directory authority fingerprints to download
+## votes
+DirectoryAuthoritiesFingerprintsForVotes = 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,27B6B5996C426270A5C95488AA5BCEB6BCC86956,49015F787433103580E3B66A1707A00E60F2D15B,585769C78764D58426B8B52B6651A5A71137189A,80550987E1D626E3EBA5E5E75A458DE0626D088C,D586D18309DED4CD6D57C18FDB97EFA96D330566,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,ED03BB616EB2F60BEC80151114BB25CEF515B226,EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97
+#
+## Download the current consensus (only if DownloadRelayDescriptors is 1)
+DownloadCurrentConsensus = true
+#
+## Download the current microdesc consensus (only if
+## DownloadRelayDescriptors is true)
+DownloadCurrentMicrodescConsensus = true
+#
+## Download current votes (only if DownloadRelayDescriptors is true)
+DownloadCurrentVotes = true
+#
+## Download missing server descriptors (only if DownloadRelayDescriptors
+## is true)
+DownloadMissingServerDescriptors = true
+#
+## Download missing extra-info descriptors (only if
+## DownloadRelayDescriptors is true)
+DownloadMissingExtraInfoDescriptors = true
+#
+## Download missing microdescriptors (only if
+## DownloadRelayDescriptors is true)
+DownloadMissingMicrodescriptors = true
+#
+## Download all server descriptors from the directory authorities at most
+## once a day (only if DownloadRelayDescriptors is true)
+DownloadAllServerDescriptors false
+#
+## Download all extra-info descriptors from the directory authorities at
+## most once a day (only if DownloadRelayDescriptors is true)
+DownloadAllExtraInfoDescriptors = false
+#
+## Compress relay descriptors downloads by adding .z to the URLs
+CompressRelayDescriptorDownloads = false
+#
+## Relative path to directory to write directory archives to
+DirectoryArchivesOutputDirectory = out/relay-descriptors/
+#
+#
+######## Bridge descriptors ########
+#
+## Relative path to directory to import bridge descriptor snapshots from
+BridgeSnapshotsDirectory = in/bridge-descriptors/
+#
+## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z
+## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we
+## can learn about IP address changes.
+ReplaceIPAddressesWithHashes = false
+#
+## Limit internal bridge descriptor mapping state to the following number
+## of days, or inf for unlimited.
+BridgeDescriptorMappingsLimit = inf
+#
+## Relative path to directory to write sanitized bridges to
+SanitizedBridgesWriteDirectory = out/bridge-descriptors/
+
+######## Exit lists ########
+#
+## (No options available)
+#
+#
+######## Torperf downloader ########
+#
+## Path to the directory to store Torperf files in.
+## A relative path starts with ./
+TorperfOutputDirectory = out/torperf/
+
+## Torperf source names and base URLs
+## multiple pairs can be specified separated by semi-colon, e.g.
+## TorperfSourceName = torperf_A, http://some.torproject.org/; another, http://another.torproject.org/
+TorperfSources = torperf, http://torperf.torproject.org/
+
+## Torperf measurement file size in bytes, .data file, and .extradata file
+## available on a given source (multiple times lists can be given
+## TorperfFiles = torperf 51200 50kb.data 50kb.extradata, torperf 1048576 1mb.data 1mb.extradata
+TorperfFilesLines = torperf 51200 50kb.data 50kb.extradata, torperf 1048576 1mb.data 1mb.extradata, torperf 5242880 5mb.data 5mb.extradata
diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java
new file mode 100644
index 0000000..9a19285
--- /dev/null
+++ b/src/test/java/org/torproject/collector/MainTest.java
@@ -0,0 +1,72 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.collector;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import org.torproject.collector.conf.Key;
+import org.torproject.collector.conf.ConfigurationException;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.net.URL;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.AccessControlException;
+import java.security.Policy;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.rules.TemporaryFolder;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class MainTest {
+
+ private Random randomSource = new Random();
+
+ @Rule
+ public TemporaryFolder tmpf = new TemporaryFolder();
+
+ @Test()
+ public void testSmoke() throws Exception {
+ System.out.println("\n!!!! Three SEVERE log messages are expected."
+ + "\nOne each from: ExitListDownloader, "
+ + "TorperfDownloader, and CreateIndexJson.\n");
+ File conf = tmpf.newFile("test.conf");
+ File lockPath = tmpf.newFolder("test.lock");
+ assertEquals(0L, conf.length());
+ Main.main(new String[]{"relaydescs", conf.toString()});
+ assertTrue(4_000L <= conf.length());
+ changeLockFilePath(conf, lockPath);
+ for ( String key : Main.collecTorMains.keySet()) {
+ Main.main(new String[]{key, conf.toString()});
+ }
+ }
+
+ private void changeLockFilePath(File f, File l) throws Exception {
+ List<String> lines = Files.readAllLines(f.toPath());
+ BufferedWriter bw = Files.newBufferedWriter(f.toPath());
+ File out = tmpf.newFolder();
+ for(String line : lines) {
+ if (line.contains(Key.LockFilePath.name())) {
+ line = Key.LockFilePath.name() + " = " + l.toString();
+ } else if (line.contains("out")) {
+ line = line.replace("out", out.toString() + "out");
+ }
+ bw.write(line);
+ bw.newLine();
+ }
+ bw.flush();
+ bw.close();
+ }
+
+}
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
new file mode 100644
index 0000000..aa98031
--- /dev/null
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -0,0 +1,143 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.collector.conf;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+
+public class ConfigurationTest {
+
+ private Random randomSource = new Random();
+
+ @Test()
+ public void testKeyCount() throws Exception {
+ assertEquals("The number of properties keys in enum Key changed."
+ + "\n This test class should be adapted.",
+ 30, Key.values().length);
+ }
+
+ @Test()
+ public void testConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("TorperfOutputDirectory = xyz".getBytes()));
+ assertEquals(1, conf.size());
+ assertEquals("xyz", conf.getProperty("TorperfOutputDirectory"));
+ }
+
+ @Test()
+ public void testArrayValues() throws Exception {
+ String[] array = new String[randomSource.nextInt(30) + 1];
+ for (int i = 0; i < array.length; i++){
+ array[i] = Integer.toBinaryString(randomSource.nextInt(100));
+ }
+ String[] arrays = new String[] {
+ Arrays.toString(array).replace("[", "").replace("]", ""),
+ Arrays.toString(array).replace("[", "").replace("]", "").replaceAll(" ", "")
+ };
+ Configuration conf = new Configuration();
+ for(String input : arrays) {
+ conf.clear();
+ conf.load(new ByteArrayInputStream(("CachedRelayDescriptorsDirectories = " + input).getBytes()));
+ assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: "
+ + Arrays.toString(conf.getStringArray(Key.CachedRelayDescriptorsDirectories)),
+ array, conf.getStringArray(Key.CachedRelayDescriptorsDirectories));
+ }
+ }
+
+ @Test()
+ public void testBoolValues() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream(("CompressRelayDescriptorDownloads=false"
+ + "\nImportDirectoryArchives = trUe"
+ + "\nReplaceIPAddressesWithHashes= false").getBytes()));
+ assertFalse(conf.getBool(Key.CompressRelayDescriptorDownloads));
+ assertTrue(conf.getBool(Key.ImportDirectoryArchives));
+ assertFalse(conf.getBool(Key.ReplaceIPAddressesWithHashes));
+ }
+
+ @Test()
+ public void testIntValues() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = inf".getBytes()));
+ assertEquals(Integer.MAX_VALUE,
+ conf.getInt(Key.BridgeDescriptorMappingsLimit));
+ int r = randomSource.nextInt(Integer.MAX_VALUE);
+ conf.clear();
+ conf.load(new ByteArrayInputStream(("BridgeDescriptorMappingsLimit =" + r).getBytes()));
+ assertEquals(r,
+ conf.getInt(Key.BridgeDescriptorMappingsLimit));
+ }
+
+ @Test()
+ public void testFileValues() throws Exception {
+ String[] files = new String[] { "/the/path/file.txt", "another/path"};
+ Configuration conf = new Configuration();
+ for(String file : files) {
+ conf.clear();
+ conf.load(new ByteArrayInputStream(("DirectoryArchivesOutputDirectory = " + file).getBytes()));
+ assertEquals(new File(file), conf.getPath(Key.DirectoryArchivesOutputDirectory).toFile());
+ }
+ }
+
+ @Test()
+ public void testArrayArrayValues() throws Exception {
+ String[][] sourceStrings = new String[][] {
+ new String[]{"localsource", "http://127.0.0.1:12345"},
+ new String[]{"somesource", "https://some.host.org:12345"}};
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream(("TorperfSources = "
+ + Arrays.deepToString(sourceStrings)).replace("[[", "").replace("]]", "")
+ .replace("], [", Configuration.ARRAYSEP).getBytes()));
+ assertArrayEquals(sourceStrings, conf.getStringArrayArray(Key.TorperfSources));
+ }
+
+ @Test( expected = ConfigurationException.class)
+ public void testArrayArrayValueException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes()));
+ conf.getStringArrayArray(Key.TorperfOutputDirectory);
+ }
+
+ @Test( expected = ConfigurationException.class)
+ public void testArrayValueException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes()));
+ conf.getStringArray(Key.TorperfSources);
+ }
+
+ @Test( expected = ConfigurationException.class)
+ public void testBoolValueException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("TorperfSource = http://x.y.z".getBytes()));
+ conf.getBool(Key.CachedRelayDescriptorsDirectories);
+ }
+
+ @Test( expected = ConfigurationException.class)
+ public void testPathValueException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("DirectoryArchivesDirectory = \\u0000:".getBytes()));
+ conf.getPath(Key.DirectoryArchivesDirectory);
+ }
+
+ @Test( expected = ConfigurationException.class)
+ public void testIntValueException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = y7".getBytes()));
+ conf.getInt(Key.BridgeDescriptorMappingsLimit);
+ }
+
+}
diff --git a/src/test/resources/junittest.policy b/src/test/resources/junittest.policy
new file mode 100644
index 0000000..e6eb2ef
--- /dev/null
+++ b/src/test/resources/junittest.policy
@@ -0,0 +1,10 @@
+/* Prevent tests from bothering production servers. */
+
+grant {
+ permission java.io.FilePermission "<<ALL FILES>>", "read, write, delete, execute";
+ permission java.util.PropertyPermission "*", "read, write";
+ permission java.lang.RuntimePermission "setIO";
+ permission java.lang.RuntimePermission "accessDeclaredMembers";
+ permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "shutdownHooks";
+};
More information about the tor-commits
mailing list