[tor-commits] [collector/master] Implements task-19021 and task-19005.

commit 8767c73d0826dfa9aa21e70a2d857c8a2d77e524 Author: iwakeh iwakeh@torproject.org Date: Mon May 30 15:14:49 2016 +0200

Implements task-19021 and task-19005. Adds the very first tests to CollecTor. Increases testability and prepares task-19018. Avoid using literal path separators,use Paths.get instead. --- .gitignore | 1 + build.xml | 30 +- config.template | 107 ------- src/main/java/org/torproject/collector/Main.java | 73 ++++- .../bridgedescs/SanitizedBridgesWriter.java | 53 ++-- .../torproject/collector/conf/Configuration.java | 123 ++++++++ .../collector/conf/ConfigurationException.java | 18 ++ .../java/org/torproject/collector/conf/Key.java | 55 ++++ .../collector/exitlists/ExitListDownloader.java | 32 ++- .../collector/index/CreateIndexJson.java | 18 +- .../torproject/collector/main/Configuration.java | 318 --------------------- .../org/torproject/collector/main/LockFile.java | 20 +- .../collector/relaydescs/ArchiveWriter.java | 188 ++++++------ .../relaydescs/CachedRelayDescriptorReader.java | 4 +- .../relaydescs/RelayDescriptorDownloader.java | 9 +- .../collector/torperf/TorperfDownloader.java | 59 ++-- src/main/resources/collector.properties | 115 ++++++++ .../java/org/torproject/collector/MainTest.java | 72 +++++ .../collector/conf/ConfigurationTest.java | 143 +++++++++ src/test/resources/junittest.policy | 10 + 20 files changed, 822 insertions(+), 626 deletions(-)

diff --git a/.gitignore b/.gitignore index afab74f..0ca0b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ /generated /lib cobertura.ser +*~

diff --git a/build.xml b/build.xml index 10c5edd..8e46584 100644 --- a/build.xml +++ b/build.xml @@ -53,10 +53,22 @@ <include name="logback-classic-1.1.2.jar" /> </fileset> </path> + <path id="cobertura.test.classpath"> + <path location="${instrument}" /> + <path refid="test.classpath" /> + <path refid="cobertura.classpath" /> + </path> <path id="test.classpath"> + <pathelement path="${classes}"/> <pathelement path="${testclasses}"/> + <pathelement path="${resources}"/> + <pathelement path="${testresources}"/> + <fileset dir="${libs}"> + <patternset refid="runtime" /> + </fileset> <fileset dir="${libs}"> <include name="junit4-4.11.jar"/> + <include name="hamcrest-all-1.3.jar"/> </fileset> </path> <target name="init"> @@ -65,7 +77,6 @@ <mkdir dir="${docs}"/> <mkdir dir="${testresult}"/> <mkdir dir="${instrument}"/> - <copy file="config.template" tofile="config"/> </target> <target name="clean"> <delete includeEmptyDirs="true" quiet="true"> @@ -123,6 +134,7 @@ <jar destfile="${jarfile}" basedir="${classes}"> <fileset dir="${classes}"/> + <fileset dir="${resources}" includes="collector.properties"/> <zipgroupfileset dir="${libs}" > <patternset refid="runtime" /> </zipgroupfileset> @@ -185,10 +197,10 @@ <junit fork="true" haltonfailure="false" printsummary="on"> <sysproperty key="net.sourceforge.cobertura.datafile" file="${cobertura.ser.file}" /> - <classpath location="${instrument}" /> - <classpath refid="classpath" /> - <classpath refid="test.classpath" /> - <classpath refid="cobertura.classpath" /> + <!-- The following jvmargs prevent test access to the network. --> + <jvmarg value="-Djava.security.policy=${testresources}/junittest.policy"/> + <jvmarg value="-Djava.security.manager"/> + <classpath refid="cobertura.test.classpath" /> <formatter type="xml" /> <batchtest toDir="${testresult}" > <fileset dir="${testclasses}" /> @@ -199,11 +211,15 @@ <include name="**/*.java" /> </fileset> </cobertura-report> - <cobertura-check branchrate="0" totallinerate="0" /> + <cobertura-check branchrate="0" totallinerate="15" totalbranchrate="5" > + <regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/> + </cobertura-check> </target> <target name="test" depends="compile,compile-tests"> <junit fork="true" haltonfailure="true" printsummary="off"> - <classpath refid="classpath"/> + <!-- The following jvmargs prevent test access to the network. --> + <jvmarg value="-Djava.security.policy=${testresources}/junittest.policy"/> + <jvmarg value="-Djava.security.manager"/> <classpath refid="test.classpath"/> <formatter type="plain" usefile="false"/> <batchtest> diff --git a/config.template b/config.template deleted file mode 100644 index 88407a2..0000000 --- a/config.template +++ /dev/null @@ -1,107 +0,0 @@ -######## Relay descriptors ######## -# -## Read cached-* files from a local Tor data directory -#ImportCachedRelayDescriptors 0 -# -## Relative path to Tor data directory to read cached-* files from (can be -## specified multiple times) -#CachedRelayDescriptorsDirectory in/relay-descriptors/cacheddesc/ -# -## Import directory archives from disk, if available -#ImportDirectoryArchives 0 -# -## Relative path to directory to import directory archives from -#DirectoryArchivesDirectory in/relay-descriptors/archives/ -# -## Keep a history of imported directory archive files to know which files -## have been imported before. This history can be useful when importing -## from a changing source to avoid importing descriptors over and over -## again, but it can be confusing to users who don't know about it. -#KeepDirectoryArchiveImportHistory 0 -# -## Download relay descriptors from directory authorities, if required -#DownloadRelayDescriptors 0 -# -## Comma separated list of directory authority addresses (IP[:port]) to -## download missing relay descriptors from -#DownloadFromDirectoryAuthorities 86.59.21.38,76.73.17.194:9030,171.25.193.9:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170,154.35.32.5 -# -## Comma separated list of directory authority fingerprints to download -## votes -#DownloadVotesByFingerprint 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,27B6B5996C426270A5C95488AA5BCEB6BCC86956,49015F787433103580E3B66A1707A00E60F2D15B,585769C78764D58426B8B52B6651A5A71137189A,80550987E1D626E3EBA5E5E75A458DE0626D088C,D586D18309DED4CD6D57C18FDB97EFA96D330566,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,ED03BB616EB2F60BEC80151114BB25CEF515B226,EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 -# -## Download the current consensus (only if DownloadRelayDescriptors is 1) -#DownloadCurrentConsensus 1 -# -## Download the current microdesc consensus (only if -## DownloadRelayDescriptors is 1) -#DownloadCurrentMicrodescConsensus 1 -# -## Download current votes (only if DownloadRelayDescriptors is 1) -#DownloadCurrentVotes 1 -# -## Download missing server descriptors (only if DownloadRelayDescriptors -## is 1) -#DownloadMissingServerDescriptors 1 -# -## Download missing extra-info descriptors (only if -## DownloadRelayDescriptors is 1) -#DownloadMissingExtraInfoDescriptors 1 -# -## Download missing microdescriptors (only if -## DownloadRelayDescriptors is 1) -#DownloadMissingMicrodescriptors 1 -# -## Download all server descriptors from the directory authorities at most -## once a day (only if DownloadRelayDescriptors is 1) -#DownloadAllServerDescriptors 0 -# -## Download all extra-info descriptors from the directory authorities at -## most once a day (only if DownloadRelayDescriptors is 1) -#DownloadAllExtraInfoDescriptors 0 -# -## Compress relay descriptors downloads by adding .z to the URLs -#CompressRelayDescriptorDownloads 0 -# -## Relative path to directory to write directory archives to -#DirectoryArchivesOutputDirectory out/relay-descriptors/ -# -# -######## Bridge descriptors ######## -# -## Relative path to directory to import bridge descriptor snapshots from -#BridgeSnapshotsDirectory in/bridge-descriptors/ -# -## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z -## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we -## can learn about IP address changes. -#ReplaceIPAddressesWithHashes 0 -# -## Limit internal bridge descriptor mapping state to the following number -## of days, or -1 for unlimited. -#LimitBridgeDescriptorMappings -1 -# -## Relative path to directory to write sanitized bridges to -#SanitizedBridgesWriteDirectory out/bridge-descriptors/ -# -# -######## Exit lists ######## -# -## (No options available) -# -# -######## Torperf downloader ######## -# -## Relative path to the directory to store Torperf files in -#TorperfOutputDirectory out/torperf/ -# -## Torperf source names and base URLs (option can be contained multiple -## times) -#TorperfSource torperf http://torperf.torproject.org/ -# -## Torperf measurement file size in bytes, .data file, and .extradata file -## available on a given source (option can be contained multiple times) -#TorperfFiles torperf 51200 50kb.data 50kb.extradata -#TorperfFiles torperf 1048576 1mb.data 1mb.extradata -#TorperfFiles torperf 5242880 5mb.data 5mb.extradata - diff --git a/src/main/java/org/torproject/collector/Main.java b/src/main/java/org/torproject/collector/Main.java index 9c64696..d21cfb6 100644 --- a/src/main/java/org/torproject/collector/Main.java +++ b/src/main/java/org/torproject/collector/Main.java @@ -4,12 +4,19 @@ package org.torproject.collector;

import org.torproject.collector.bridgedescs.SanitizedBridgesWriter; +import org.torproject.collector.conf.Configuration; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; import org.torproject.collector.relaydescs.ArchiveWriter; import org.torproject.collector.torperf.TorperfDownloader;

+import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; import java.util.logging.Logger; @@ -24,11 +31,12 @@ import java.util.logging.Logger; public class Main {

private static Logger log = Logger.getLogger(Main.class.getName()); + public static final String CONF_FILE = "collector.properties";

/** All possible main classes. * If a new CollecTorMain class is available, just add it to this map. */ - private static final Map<String, Class> collecTorMains = new HashMap<>(); + static final Map<String, Class> collecTorMains = new HashMap<>();

static { // add a new main class here collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class); @@ -41,38 +49,73 @@ public class Main { private static final String modules = collecTorMains.keySet().toString() .replace("[", "").replace("]", "").replaceAll(", ", "|");

+ private static Configuration conf = new Configuration(); + /** * One argument is necessary. * See class description {@link Main}. */ - public static void main(String[] args) { - if (null == args || args.length != 1) { - printUsageAndExit("CollecTor needs exactly one argument."); + public static void main(String[] args) throws Exception { + File confFile = null; + if (null == args || args.length < 1 || args.length > 2) { + printUsage("CollecTor needs one or two arguments."); + return; + } else if (args.length == 1) { + confFile = new File(CONF_FILE); + } else if (args.length == 2) { + confFile = new File(args[1]); + } + if (!confFile.exists() || confFile.length() < 1L) { + writeDefaultConfig(confFile); + return; } else { - invokeGivenMainAndExit(args[0]); + readConfigurationFrom(confFile); } + invokeGivenMain(args[0]); }

- private static void printUsageAndExit(String msg) { + private static void printUsage(String msg) { final String usage = "Usage:\njava -jar collector.jar " - + "<" + modules + ">"; + + "<" + modules + "> [path/to/configFile]"; System.out.println(msg + "\n" + usage); - System.exit(0); }

- private static void invokeGivenMainAndExit(String mainId) { + private static void writeDefaultConfig(File confFile) { + try { + Files.copy(Main.class.getClassLoader().getResource(CONF_FILE).openStream(), + confFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + printUsage("Could not find config file. In the default " + + "configuration, we are not configured to read data from any " + + "data source or write data to any data sink. You need to " + + "change the configuration (" + CONF_FILE + + ") and provide at least one data source and one data sink. " + + "Refer to the manual for more information."); + } catch (IOException e) { + log.severe("Cannot write default configuration. Reason: " + e); + } + } + + private static void readConfigurationFrom(File confFile) throws Exception { + try (FileInputStream fis = new FileInputStream(confFile)) { + conf.load(fis); + } catch (Exception e) { // catch all possible problems + log.severe("Cannot read configuration. Reason: " + e); + throw e; + } + } + + private static void invokeGivenMain(String mainId) { Class clazz = collecTorMains.get(mainId); if (null == clazz) { - printUsageAndExit("Unknown argument: " + mainId); + printUsage("Unknown argument: " + mainId); } - invokeMainOnClassAndExit(clazz); + invokeMainOnClass(clazz); }

- private static void invokeMainOnClassAndExit(Class clazz) { + private static void invokeMainOnClass(Class clazz) { try { - clazz.getMethod("main", new Class[] { String[].class }) - .invoke(null, (Object) new String[]{}); - System.exit(0); + clazz.getMethod("main", new Class[] { Configuration.class }) + .invoke(null, (Object) conf); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { log.severe("Cannot invoke 'main' method on " diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 3214715..fa24a3d 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -3,7 +3,9 @@

package org.torproject.collector.bridgedescs;

-import org.torproject.collector.main.Configuration; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; import org.torproject.collector.main.LockFile;

import org.apache.commons.codec.DecoderException; @@ -35,36 +37,30 @@ import java.util.logging.Level; import java.util.logging.Logger;

/** - * Sanitizes bridge descriptors, i.e., removes all possibly sensitive + * <p>Sanitizes bridge descriptors, i.e., removes all possibly sensitive * information from them, and writes them to a local directory structure. * During the sanitizing process, all information about the bridge * identity or IP address are removed or replaced. The goal is to keep the * sanitized bridge descriptors useful for statistical analysis while not - * making it easier for an adversary to enumerate bridges. + * making it easier for an adversary to enumerate bridges.</p> * - * There are three types of bridge descriptors: bridge network statuses + * <p>There are three types of bridge descriptors: bridge network statuses * (lists of all bridges at a given time), server descriptors (published * by the bridge to advertise their capabilities), and extra-info - * descriptors (published by the bridge, mainly for statistical analysis). + * descriptors (published by the bridge, mainly for statistical analysis).</p> */ public class SanitizedBridgesWriter extends Thread {

- public static void main(String[] args) { + private static Logger logger;

- Logger logger = Logger.getLogger( - SanitizedBridgesWriter.class.getName()); - logger.info("Starting bridge-descriptors module of CollecTor."); + public static void main(Configuration config) throws ConfigurationException {

- // Initialize configuration - Configuration config = new Configuration(); + logger = Logger.getLogger(SanitizedBridgesWriter.class.getName()); + logger.info("Starting bridge-descriptors module of CollecTor.");

// Use lock file to avoid overlapping runs - LockFile lf = new LockFile("bridge-descriptors"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } + LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "bridge-descriptors"); + lf.acquireLock();

// Sanitize bridge descriptors new SanitizedBridgesWriter(config).run(); @@ -84,11 +80,6 @@ public class SanitizedBridgesWriter extends Thread { this.config = config; }

- /** - * Logger for this class. - */ - private Logger logger; - private String rsyncCatString;

private File bridgeDirectoriesDirectory; @@ -112,16 +103,26 @@ public class SanitizedBridgesWriter extends Thread {

private SecureRandom secureRandom;

+ @Override public void run() { + try { + startProcessing(); + } catch (ConfigurationException ce) { + logger.severe("Configuration failed: " + ce); + throw new RuntimeException(ce); + } + } + + private void startProcessing() throws ConfigurationException {

File bridgeDirectoriesDirectory = - new File(config.getBridgeSnapshotsDirectory()); + config.getPath(Key.BridgeSnapshotsDirectory).toFile(); File sanitizedBridgesDirectory = - new File(config.getSanitizedBridgesWriteDirectory()); + config.getPath(Key.SanitizedBridgesWriteDirectory).toFile(); boolean replaceIPAddressesWithHashes = - config.getReplaceIPAddressesWithHashes(); + config.getBool(Key.ReplaceIPAddressesWithHashes); long limitBridgeSanitizingInterval = - config.getLimitBridgeDescriptorMappings(); + config.getInt(Key.BridgeDescriptorMappingsLimit); File statsDirectory = new File("stats");

if (bridgeDirectoriesDirectory == null diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java new file mode 100644 index 0000000..8b8cc12 --- /dev/null +++ b/src/main/java/org/torproject/collector/conf/Configuration.java @@ -0,0 +1,123 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.conf; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Initialize configuration with defaults from collector.properties, + * unless a configuration properties file is available. + */ +public class Configuration extends Properties { + + public static final String FIELDSEP = ","; + public static final String ARRAYSEP = ";"; + + /** + * Returns {@code String[][]} from a property. Commas seperate array elements + * and semicolons separate arrays, e.g., + * {@code propertyname = a1, a2, a3; b1, b2, b3} + */ + public String[][] getStringArrayArray(Key key) throws ConfigurationException { + try { + checkClass(key, String[][].class); + String[] interim = getProperty(key.name()).split(ARRAYSEP); + String[][] res = new String[interim.length][]; + for (int i = 0; i < interim.length; i++) { + res[i] = interim[i].trim().split(FIELDSEP); + for (int j = 0; j < res[i].length; j++) { + res[i][j] = res[i][j].trim(); + } + } + return res; + } catch (RuntimeException re) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + re.getMessage(), re); + } + } + + /** + * Returns {@code String[]} from a property. Commas seperate array elements, + * e.g., + * {@code propertyname = a1, a2, a3} + */ + public String[] getStringArray(Key key) throws ConfigurationException { + try { + checkClass(key, String[].class); + String[] res = getProperty(key.name()).split(FIELDSEP); + for (int i = 0; i < res.length; i++) { + res[i] = res[i].trim(); + } + return res; + } catch (RuntimeException re) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + re.getMessage(), re); + } + } + + private void checkClass(Key key, Class clazz) { + if (!key.keyClass().getSimpleName().equals(clazz.getSimpleName())) { + throw new RuntimeException("Wrong type wanted! My class is " + + key.keyClass().getSimpleName()); + } + } + + /** + * Returns a {@code boolean} property (case insensitiv), e.g. + * {@code propertyOne = True}. + */ + public boolean getBool(Key key) throws ConfigurationException { + try { + checkClass(key, Boolean.class); + return Boolean.parseBoolean(getProperty(key.name())); + } catch (RuntimeException re) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + re.getMessage(), re); + } + } + + /** + * Parse an integer property and translate the String + * <code>"inf"</code> into Integer.MAX_VALUE. + * Verifies that this enum is a Key for an integer value. + */ + public int getInt(Key key) throws ConfigurationException { + try { + checkClass(key, Integer.class); + String prop = getProperty(key.name()); + if ("inf".equals(prop)) { + return Integer.MAX_VALUE; + } else { + return Integer.parseInt(prop); + } + } catch (RuntimeException re) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + re.getMessage(), re); + } + } + + /** + * Returns a {@code Path} property, e.g. + * {@code pathProperty = /my/path/file}. + */ + public Path getPath(Key key) throws ConfigurationException { + try { + checkClass(key, Path.class); + return Paths.get(getProperty(key.name())); + } catch (RuntimeException re) { + throw new ConfigurationException("Corrupt property: " + key + + " reason: " + re.getMessage(), re); + } + } + +} diff --git a/src/main/java/org/torproject/collector/conf/ConfigurationException.java b/src/main/java/org/torproject/collector/conf/ConfigurationException.java new file mode 100644 index 0000000..730b1b3 --- /dev/null +++ b/src/main/java/org/torproject/collector/conf/ConfigurationException.java @@ -0,0 +1,18 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.conf; + +public class ConfigurationException extends Exception { + + public ConfigurationException() {} + + public ConfigurationException(String msg) { + super(msg); + } + + public ConfigurationException(String msg, Exception ex) { + super(msg, ex); + } + +} diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java new file mode 100644 index 0000000..67f91c5 --- /dev/null +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -0,0 +1,55 @@ +package org.torproject.collector.conf; + +import java.nio.file.Path; + +/** + * Enum containing all the properties keys of the configuration. + * Specifies the key type. + */ +public enum Key { + + LockFilePath(Path.class), + ArchivePath(Path.class), + RecentPath(Path.class), + IndexPath(Path.class), + StatsPath(Path.class), + BridgeSnapshotsDirectory(Path.class), + CachedRelayDescriptorsDirectories(String[].class), + CompressRelayDescriptorDownloads(Boolean.class), + DirectoryArchivesDirectory(Path.class), + DirectoryArchivesOutputDirectory(Path.class), + DownloadRelayDescriptors(Boolean.class), + DirectoryAuthoritiesAddresses(String[].class), + DirectoryAuthoritiesFingerprintsForVotes(String[].class), + DownloadCurrentConsensus(Boolean.class), + DownloadCurrentMicrodescConsensus(Boolean.class), + DownloadCurrentVotes(Boolean.class), + DownloadMissingServerDescriptors(Boolean.class), + DownloadMissingExtraInfoDescriptors(Boolean.class), + DownloadMissingMicrodescriptors(Boolean.class), + DownloadAllServerDescriptors(Boolean.class), + DownloadAllExtraInfoDescriptors(Boolean.class), + ImportCachedRelayDescriptors(Boolean.class), + ImportDirectoryArchives(Boolean.class), + KeepDirectoryArchiveImportHistory(Boolean.class), + ReplaceIPAddressesWithHashes(Boolean.class), + BridgeDescriptorMappingsLimit(Integer.class), + SanitizedBridgesWriteDirectory(Path.class), + TorperfOutputDirectory(Path.class), + TorperfFilesLines(String[].class), + TorperfSources(String[][].class); + + private Class clazz; + + /** + * @param Class of key value. + */ + Key(Class clazz) { + this.clazz = clazz; + } + + public Class keyClass() { + return clazz; + } + +} diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 54fd50f..53fc300 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -3,7 +3,9 @@

package org.torproject.collector.exitlists;

-import org.torproject.collector.main.Configuration; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; import org.torproject.collector.main.LockFile; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; @@ -31,21 +33,15 @@ import java.util.logging.Logger;

public class ExitListDownloader extends Thread {

- public static void main(String[] args) { + private static Logger logger = + Logger.getLogger(ExitListDownloader.class.getName());

- Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); + public static void main(Configuration config) throws ConfigurationException { logger.info("Starting exit-lists module of CollecTor.");

- // Initialize configuration - Configuration config = new Configuration(); - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile("exit-lists"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } + LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "exit-lists"); + lf.acquireLock();

// Download exit list and store it to disk new ExitListDownloader(config).run(); @@ -56,12 +52,18 @@ public class ExitListDownloader extends Thread { logger.info("Terminating exit-lists module of CollecTor."); }

- public ExitListDownloader(Configuration config) { - } + public ExitListDownloader(Configuration config) {}

public void run() { + try { + startProcessing(); + } catch (ConfigurationException ce) { + logger.severe("Configuration failed: " + ce); + throw new RuntimeException(ce); + } + }

- Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); + private void startProcessing() throws ConfigurationException {

SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index ac5adf5..de69488 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -3,6 +3,10 @@

package org.torproject.collector.index;

+import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; + import com.google.gson.Gson; import com.google.gson.GsonBuilder;

@@ -33,12 +37,11 @@ import java.util.zip.GZIPOutputStream; * we'll likely have to do that. */ public class CreateIndexJson {

- static final File indexJsonFile = new File("index.json"); + private static File indexJsonFile;

- static final String basePath = "https://collector.torproject.org"; + private static String basePath = "https://collector.torproject.org";

- static final File[] indexedDirectories = new File[] { - new File("archive"), new File("recent") }; + private static File[] indexedDirectories;

static final String dateTimePattern = "yyyy-MM-dd HH:mm";

@@ -46,7 +49,12 @@ public class CreateIndexJson {

static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");

- public static void main(String[] args) throws IOException { + public static void main(Configuration config) + throws ConfigurationException, IOException { + indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); + indexedDirectories = new File[] { + new File(config.getPath(Key.ArchivePath).toFile(), "archive"), + new File(config.getPath(Key.RecentPath).toFile(), "recent") }; writeIndex(indexDirectories()); }

diff --git a/src/main/java/org/torproject/collector/main/Configuration.java b/src/main/java/org/torproject/collector/main/Configuration.java deleted file mode 100644 index aee1d02..0000000 --- a/src/main/java/org/torproject/collector/main/Configuration.java +++ /dev/null @@ -1,318 +0,0 @@ -/* Copyright 2010--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.main; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Initialize configuration with hard-coded defaults, overwrite with - * configuration in config file, if exists, and answer Main.java about our - * configuration. - */ -public class Configuration { - private String directoryArchivesOutputDirectory = - "out/relay-descriptors/"; - private boolean importCachedRelayDescriptors = false; - private List<String> cachedRelayDescriptorsDirectory = - new ArrayList<String>(Arrays.asList( - "in/relay-descriptors/cacheddesc/".split(","))); - private boolean importDirectoryArchives = false; - private String directoryArchivesDirectory = - "in/relay-descriptors/archives/"; - private boolean keepDirectoryArchiveImportHistory = false; - private boolean replaceIPAddressesWithHashes = false; - private long limitBridgeDescriptorMappings = -1L; - private String sanitizedBridgesWriteDirectory = - "out/bridge-descriptors/"; - private String bridgeSnapshotsDirectory = "in/bridge-descriptors/"; - private boolean downloadRelayDescriptors = false; - private List<String> downloadFromDirectoryAuthorities = Arrays.asList(( - "86.59.21.38,76.73.17.194:9030,171.25.193.9:443," - + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131," - + "194.109.206.212,212.112.245.170,154.35.32.5").split(",")); - private List<String> downloadVotesByFingerprint = Arrays.asList(( - "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4," - + "27B6B5996C426270A5C95488AA5BCEB6BCC86956," - + "49015F787433103580E3B66A1707A00E60F2D15B," - + "585769C78764D58426B8B52B6651A5A71137189A," - + "80550987E1D626E3EBA5E5E75A458DE0626D088C," - + "D586D18309DED4CD6D57C18FDB97EFA96D330566," - + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58," - + "ED03BB616EB2F60BEC80151114BB25CEF515B226," - + "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97").split(",")); - private boolean downloadCurrentConsensus = true; - private boolean downloadCurrentMicrodescConsensus = true; - private boolean downloadCurrentVotes = true; - private boolean downloadMissingServerDescriptors = true; - private boolean downloadMissingExtraInfoDescriptors = true; - private boolean downloadMissingMicrodescriptors = true; - private boolean downloadAllServerDescriptors = false; - private boolean downloadAllExtraInfoDescriptors = false; - private boolean compressRelayDescriptorDownloads; - private String torperfOutputDirectory = "out/torperf/"; - private SortedMap<String, String> torperfSources = null; - private List<String> torperfFiles = null; - - public Configuration() { - - /* Initialize logger. */ - Logger logger = Logger.getLogger(Configuration.class.getName()); - - /* Read config file, if present. */ - File configFile = new File("config"); - if (!configFile.exists()) { - logger.warning("Could not find config file. In the default " - + "configuration, we are not configured to read data from any " - + "data source or write data to any data sink. You need to " - + "create a config file (" + configFile.getAbsolutePath() - + ") and provide at least one data source and one data sink. " - + "Refer to the manual for more information."); - return; - } - String line = null; - boolean containsCachedRelayDescriptorsDirectory = false; - try { - BufferedReader br = new BufferedReader(new FileReader(configFile)); - while ((line = br.readLine()) != null) { - if (line.startsWith("#") || line.length() < 1) { - continue; - } else if (line.startsWith("DirectoryArchivesOutputDirectory")) { - this.directoryArchivesOutputDirectory = line.split(" ")[1]; - } else if (line.startsWith("ImportCachedRelayDescriptors")) { - this.importCachedRelayDescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("CachedRelayDescriptorsDirectory")) { - if (!containsCachedRelayDescriptorsDirectory) { - this.cachedRelayDescriptorsDirectory.clear(); - containsCachedRelayDescriptorsDirectory = true; - } - this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]); - } else if (line.startsWith("ImportDirectoryArchives")) { - this.importDirectoryArchives = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DirectoryArchivesDirectory")) { - this.directoryArchivesDirectory = line.split(" ")[1]; - } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { - this.keepDirectoryArchiveImportHistory = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("ReplaceIPAddressesWithHashes")) { - this.replaceIPAddressesWithHashes = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("LimitBridgeDescriptorMappings")) { - this.limitBridgeDescriptorMappings = Long.parseLong( - line.split(" ")[1]); - } else if (line.startsWith("SanitizedBridgesWriteDirectory")) { - this.sanitizedBridgesWriteDirectory = line.split(" ")[1]; - } else if (line.startsWith("BridgeSnapshotsDirectory")) { - this.bridgeSnapshotsDirectory = line.split(" ")[1]; - } else if (line.startsWith("DownloadRelayDescriptors")) { - this.downloadRelayDescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadFromDirectoryAuthorities")) { - this.downloadFromDirectoryAuthorities = new ArrayList<String>(); - for (String dir : line.split(" ")[1].split(",")) { - // test if IP:port pair has correct format - if (dir.length() < 1) { - logger.severe("Configuration file contains directory " - + "authority IP:port of length 0 in line '" + line - + "'! Exiting!"); - System.exit(1); - } - new URL("http://" + dir + "/"); - this.downloadFromDirectoryAuthorities.add(dir); - } - } else if (line.startsWith("DownloadVotesByFingerprint")) { - this.downloadVotesByFingerprint = new ArrayList<String>(); - for (String fingerprint : line.split(" ")[1].split(",")) { - this.downloadVotesByFingerprint.add(fingerprint); - } - } else if (line.startsWith("DownloadCurrentConsensus")) { - this.downloadCurrentConsensus = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadCurrentMicrodescConsensus")) { - this.downloadCurrentMicrodescConsensus = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadCurrentVotes")) { - this.downloadCurrentVotes = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadMissingServerDescriptors")) { - this.downloadMissingServerDescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith( - "DownloadMissingExtraInfoDescriptors")) { - this.downloadMissingExtraInfoDescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadMissingMicrodescriptors")) { - this.downloadMissingMicrodescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadAllServerDescriptors")) { - this.downloadAllServerDescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) { - this.downloadAllExtraInfoDescriptors = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("CompressRelayDescriptorDownloads")) { - this.compressRelayDescriptorDownloads = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("TorperfOutputDirectory")) { - this.torperfOutputDirectory = line.split(" ")[1]; - } else if (line.startsWith("TorperfSource")) { - if (this.torperfSources == null) { - this.torperfSources = new TreeMap<String, String>(); - } - String[] parts = line.split(" "); - String sourceName = parts[1]; - String baseUrl = parts[2]; - this.torperfSources.put(sourceName, baseUrl); - } else if (line.startsWith("TorperfFiles")) { - if (this.torperfFiles == null) { - this.torperfFiles = new ArrayList<String>(); - } - String[] parts = line.split(" "); - if (parts.length != 5) { - logger.severe("Configuration file contains TorperfFiles " - + "option with wrong number of values in line '" + line - + "'! Exiting!"); - System.exit(1); - } - this.torperfFiles.add(line); - } else { - logger.severe("Configuration file contains unrecognized " - + "configuration key in line '" + line + "'! Exiting!"); - System.exit(1); - } - } - br.close(); - } catch (ArrayIndexOutOfBoundsException e) { - logger.severe("Configuration file contains configuration key " - + "without value in line '" + line + "'. Exiting!"); - System.exit(1); - } catch (MalformedURLException e) { - logger.severe("Configuration file contains illegal URL or IP:port " - + "pair in line '" + line + "'. Exiting!"); - System.exit(1); - } catch (NumberFormatException e) { - logger.severe("Configuration file contains illegal value in line '" - + line + "' with legal values being 0 or 1. Exiting!"); - System.exit(1); - } catch (IOException e) { - logger.log(Level.SEVERE, "Unknown problem while reading config " - + "file! Exiting!", e); - System.exit(1); - } - } - - public String getDirectoryArchivesOutputDirectory() { - return this.directoryArchivesOutputDirectory; - } - - public boolean getImportCachedRelayDescriptors() { - return this.importCachedRelayDescriptors; - } - - public List<String> getCachedRelayDescriptorDirectory() { - return this.cachedRelayDescriptorsDirectory; - } - - public boolean getImportDirectoryArchives() { - return this.importDirectoryArchives; - } - - public String getDirectoryArchivesDirectory() { - return this.directoryArchivesDirectory; - } - - public boolean getKeepDirectoryArchiveImportHistory() { - return this.keepDirectoryArchiveImportHistory; - } - - public boolean getReplaceIPAddressesWithHashes() { - return this.replaceIPAddressesWithHashes; - } - - public long getLimitBridgeDescriptorMappings() { - return this.limitBridgeDescriptorMappings; - } - - public String getSanitizedBridgesWriteDirectory() { - return this.sanitizedBridgesWriteDirectory; - } - - public String getBridgeSnapshotsDirectory() { - return this.bridgeSnapshotsDirectory; - } - - public boolean getDownloadRelayDescriptors() { - return this.downloadRelayDescriptors; - } - - public List<String> getDownloadFromDirectoryAuthorities() { - return this.downloadFromDirectoryAuthorities; - } - - public List<String> getDownloadVotesByFingerprint() { - return this.downloadVotesByFingerprint; - } - - public boolean getDownloadCurrentConsensus() { - return this.downloadCurrentConsensus; - } - - public boolean getDownloadCurrentMicrodescConsensus() { - return this.downloadCurrentMicrodescConsensus; - } - - public boolean getDownloadCurrentVotes() { - return this.downloadCurrentVotes; - } - - public boolean getDownloadMissingServerDescriptors() { - return this.downloadMissingServerDescriptors; - } - - public boolean getDownloadMissingExtraInfoDescriptors() { - return this.downloadMissingExtraInfoDescriptors; - } - - public boolean getDownloadMissingMicrodescriptors() { - return this.downloadMissingMicrodescriptors; - } - - public boolean getDownloadAllServerDescriptors() { - return this.downloadAllServerDescriptors; - } - - public boolean getDownloadAllExtraInfoDescriptors() { - return this.downloadAllExtraInfoDescriptors; - } - - public boolean getCompressRelayDescriptorDownloads() { - return this.compressRelayDescriptorDownloads; - } - - public String getTorperfOutputDirectory() { - return this.torperfOutputDirectory; - } - - public SortedMap<String, String> getTorperfSources() { - return this.torperfSources; - } - - public List<String> getTorperfFiles() { - return this.torperfFiles; - } -} - diff --git a/src/main/java/org/torproject/collector/main/LockFile.java b/src/main/java/org/torproject/collector/main/LockFile.java index b07d4b1..f168bc3 100644 --- a/src/main/java/org/torproject/collector/main/LockFile.java +++ b/src/main/java/org/torproject/collector/main/LockFile.java @@ -13,12 +13,17 @@ import java.util.logging.Logger;

public class LockFile {

- private File lockFile; - private Logger logger; + private final File lockFile; + private final String moduleName; + private final Logger logger = Logger.getLogger(LockFile.class.getName());

public LockFile(String moduleName) { - this.lockFile = new File("lock/" + moduleName); - this.logger = Logger.getLogger(LockFile.class.getName()); + this("lock", moduleName); + } + + public LockFile(String lockFilePath, String moduleName) { + this.lockFile = new File(lockFilePath, moduleName); + this.moduleName = moduleName; }

public boolean acquireLock() { @@ -30,7 +35,7 @@ public class LockFile { long runStarted = Long.parseLong(br.readLine()); br.close(); if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) { - return false; + throw new RuntimeException("Cannot acquire lock for " + moduleName); } } this.lockFile.getParentFile().mkdirs(); @@ -41,9 +46,8 @@ public class LockFile { this.logger.fine("Acquired lock."); return true; } catch (IOException e) { - this.logger.warning("Caught exception while trying to acquire " - + "lock!"); - return false; + throw new RuntimeException("Caught exception while trying to acquire " + + "lock for " + moduleName); } }

diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index cf603d1..43c7975 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -3,7 +3,9 @@

package org.torproject.collector.relaydescs;

-import org.torproject.collector.main.Configuration; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; import org.torproject.collector.main.LockFile; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; @@ -17,6 +19,8 @@ import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -36,11 +40,12 @@ import java.util.logging.Logger;

public class ArchiveWriter extends Thread {

+ private static Logger logger = Logger.getLogger(ArchiveWriter.class.getName()); + private Configuration config;

private long now = System.currentTimeMillis(); - private Logger logger; - private File outputDirectory; + private String outputDirectory; private String rsyncCatString; private DescriptorParser descriptorParser; private int storedConsensusesCounter = 0; @@ -67,12 +72,9 @@ public class ArchiveWriter extends Thread { private SortedMap<Long, Set<String>> storedMicrodescriptors = new TreeMap<Long, Set<String>>();

- private File storedServerDescriptorsFile = new File( - "stats/stored-server-descriptors"); - private File storedExtraInfoDescriptorsFile = new File( - "stats/stored-extra-info-descriptors"); - private File storedMicrodescriptorsFile = new File( - "stats/stored-microdescriptors"); + private File storedServerDescriptorsFile; + private File storedExtraInfoDescriptorsFile; + private File storedMicrodescriptorsFile;

private static final byte[] CONSENSUS_ANNOTATION = "@type network-status-consensus-3 1.0\n".getBytes(); @@ -97,28 +99,31 @@ public class ArchiveWriter extends Thread {

private StringBuilder intermediateStats = new StringBuilder();

- public static void main(String[] args) { + private static Path recentPath; + private static String recentPathName; + private static final String RELAY_DESCRIPTORS = "relay-descriptors"; + private static final String MICRO = "micro"; + private static final String CONSENSUS_MICRODESC = "consensus-microdesc"; + private static final String MICRODESC = "microdesc"; + private static final String MICRODESCS = "microdescs"; + public static void main(Configuration config) throws ConfigurationException {

- Logger logger = Logger.getLogger(ArchiveWriter.class.getName()); logger.info("Starting relay-descriptors module of CollecTor.");

- // Initialize configuration - Configuration config = new Configuration(); - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile("relay-descriptors"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } + LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), RELAY_DESCRIPTORS); + lf.acquireLock(); + + recentPath = config.getPath(Key.RecentPath); + recentPathName = recentPath.toString();

// Import/download relay descriptors from the various sources new ArchiveWriter(config).run();

- new ReferenceChecker(new File("recent/relay-descriptors"), - new File("stats/references"), - new File("stats/references-history")).check(); + new ReferenceChecker( + recentPath.toFile(), + new File(config.getPath(Key.StatsPath).toFile(), "references"), + new File(config.getPath(Key.StatsPath).toFile(), "references-history")).check();

// Remove lock file lf.releaseLock(); @@ -126,18 +131,29 @@ public class ArchiveWriter extends Thread { logger.info("Terminating relay-descriptors module of CollecTor."); }

- public ArchiveWriter(Configuration config) { + public ArchiveWriter(Configuration config) throws ConfigurationException { this.config = config; + storedServerDescriptorsFile = + new File(config.getPath(Key.StatsPath).toFile(), "stored-server-descriptors"); + storedExtraInfoDescriptorsFile = + new File(config.getPath(Key.StatsPath).toFile(), "stored-extra-info-descriptors"); + storedMicrodescriptorsFile = + new File(config.getPath(Key.StatsPath).toFile(), "stored-microdescriptors"); }

public void run() { + try { + startProcessing(); + } catch (ConfigurationException ce) { + logger.severe("Configuration failed: " + ce); + throw new RuntimeException(ce); + } + }

- File outputDirectory = - new File(config.getDirectoryArchivesOutputDirectory()); - File statsDirectory = new File("stats"); + private void startProcessing() throws ConfigurationException {

- this.logger = Logger.getLogger(ArchiveWriter.class.getName()); - this.outputDirectory = outputDirectory; + File statsDirectory = new File("stats"); + this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString(); SimpleDateFormat rsyncCatFormat = new SimpleDateFormat( "yyyy-MM-dd-HH-mm-ss"); rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -152,33 +168,33 @@ public class ArchiveWriter extends Thread { RelayDescriptorParser rdp = new RelayDescriptorParser(this);

RelayDescriptorDownloader rdd = null; - if (config.getDownloadRelayDescriptors()) { - List<String> dirSources = - config.getDownloadFromDirectoryAuthorities(); + if (config.getBool(Key.DownloadRelayDescriptors)) { + String[] dirSources = + config.getStringArray(Key.DirectoryAuthoritiesAddresses); rdd = new RelayDescriptorDownloader(rdp, dirSources, - config.getDownloadVotesByFingerprint(), - config.getDownloadCurrentConsensus(), - config.getDownloadCurrentMicrodescConsensus(), - config.getDownloadCurrentVotes(), - config.getDownloadMissingServerDescriptors(), - config.getDownloadMissingExtraInfoDescriptors(), - config.getDownloadMissingMicrodescriptors(), - config.getDownloadAllServerDescriptors(), - config.getDownloadAllExtraInfoDescriptors(), - config.getCompressRelayDescriptorDownloads()); + config.getStringArray(Key.DirectoryAuthoritiesFingerprintsForVotes), + config.getBool(Key.DownloadCurrentConsensus), + config.getBool(Key.DownloadCurrentMicrodescConsensus), + config.getBool(Key.DownloadCurrentVotes), + config.getBool(Key.DownloadMissingServerDescriptors), + config.getBool(Key.DownloadMissingExtraInfoDescriptors), + config.getBool(Key.DownloadMissingMicrodescriptors), + config.getBool(Key.DownloadAllServerDescriptors), + config.getBool(Key.DownloadAllExtraInfoDescriptors), + config.getBool(Key.CompressRelayDescriptorDownloads)); rdp.setRelayDescriptorDownloader(rdd); } - if (config.getImportCachedRelayDescriptors()) { + if (config.getBool(Key.ImportCachedRelayDescriptors)) { new CachedRelayDescriptorReader(rdp, - config.getCachedRelayDescriptorDirectory(), statsDirectory); + config.getStringArray(Key.CachedRelayDescriptorsDirectories), statsDirectory); this.intermediateStats("importing relay descriptors from local " + "Tor data directories"); } - if (config.getImportDirectoryArchives()) { + if (config.getBool(Key.ImportDirectoryArchives)) { new ArchiveReader(rdp, - new File(config.getDirectoryArchivesDirectory()), + config.getPath(Key.DirectoryArchivesDirectory).toFile(), statsDirectory, - config.getKeepDirectoryArchiveImportHistory()); + config.getBool(Key.KeepDirectoryArchiveImportHistory)); this.intermediateStats("importing relay descriptors from local " + "directory"); } @@ -557,7 +573,7 @@ public class ArchiveWriter extends Thread { - 3L * 24L * 60L * 60L * 1000L; long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/relay-descriptors")); + allFiles.add(new File(recentPathName, RELAY_DESCRIPTORS)); while (!allFiles.isEmpty()) { File file = allFiles.pop(); if (file.isDirectory()) { @@ -633,11 +649,11 @@ public class ArchiveWriter extends Thread { SimpleDateFormat printFormat = new SimpleDateFormat( "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); printFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory + "/consensus/" - + printFormat.format(new Date(validAfter)) + "-consensus"); + File tarballFile = Paths.get(this.outputDirectory, "consensus", + printFormat.format(new Date(validAfter)) + "-consensus").toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); - File rsyncFile = new File("recent/relay-descriptors/consensuses/" - + tarballFile.getName()); + File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, + "consensuses", tarballFile.getName()).toFile(); File[] outputFiles = new File[] { tarballFile, rsyncFile }; if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) { this.storedConsensusesCounter++; @@ -657,14 +673,12 @@ public class ArchiveWriter extends Thread { SimpleDateFormat dayDirectoryFileFormat = new SimpleDateFormat( "dd/yyyy-MM-dd-HH-mm-ss"); dayDirectoryFileFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory - + "/microdesc/" + yearMonthDirectoryFormat.format(validAfter) - + "/consensus-microdesc/" - + dayDirectoryFileFormat.format(validAfter) - + "-consensus-microdesc"); + File tarballFile = Paths.get(this.outputDirectory, MICRODESC, + yearMonthDirectoryFormat.format(validAfter), CONSENSUS_MICRODESC, + dayDirectoryFileFormat.format(validAfter) + "-consensus-microdesc").toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); - File rsyncFile = new File("recent/relay-descriptors/microdescs/" - + "consensus-microdesc/" + tarballFile.getName()); + File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, MICRODESCS, + CONSENSUS_MICRODESC, tarballFile.getName()).toFile(); File[] outputFiles = new File[] { tarballFile, rsyncFile }; if (this.store(MICRODESCCONSENSUS_ANNOTATION, data, outputFiles, null)) { @@ -683,12 +697,12 @@ public class ArchiveWriter extends Thread { SimpleDateFormat printFormat = new SimpleDateFormat( "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); printFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory + "/vote/" - + printFormat.format(new Date(validAfter)) + "-vote-" - + fingerprint + "-" + digest); + File tarballFile = Paths.get(this.outputDirectory, "vote", + printFormat.format(new Date(validAfter)) + "-vote-" + + fingerprint + "-" + digest).toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); - File rsyncFile = new File("recent/relay-descriptors/votes/" - + tarballFile.getName()); + File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "votes", + tarballFile.getName()).toFile(); File[] outputFiles = new File[] { tarballFile, rsyncFile }; if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) { this.storedVotesCounter++; @@ -709,8 +723,8 @@ public class ArchiveWriter extends Thread { SimpleDateFormat printFormat = new SimpleDateFormat( "yyyy-MM-dd-HH-mm-ss"); printFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory + "/certs/" - + fingerprint + "-" + printFormat.format(new Date(published))); + File tarballFile = Paths.get(this.outputDirectory, "certs", + fingerprint + "-" + printFormat.format(new Date(published))).toFile(); File[] outputFiles = new File[] { tarballFile }; if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles, null)) { this.storedCertsCounter++; @@ -721,14 +735,13 @@ public class ArchiveWriter extends Thread { long published, String extraInfoDigest) { SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/"); printFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory - + "/server-descriptor/" + printFormat.format(new Date(published)) - + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/" - + digest); + File tarballFile = Paths.get(this.outputDirectory, + "server-descriptor", printFormat.format(new Date(published)), + digest.substring(0, 1), digest.substring(1, 2), digest).toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); - File rsyncCatFile = new File("recent/relay-descriptors/" - + "server-descriptors/" + this.rsyncCatString - + "-server-descriptors.tmp"); + File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, + "server-descriptors", + this.rsyncCatString + "-server-descriptors.tmp").toFile(); File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; boolean[] append = new boolean[] { false, true }; if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles, @@ -750,14 +763,14 @@ public class ArchiveWriter extends Thread { String extraInfoDigest, long published) { SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/"); descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory + "/extra-info/" - + descriptorFormat.format(new Date(published)) - + extraInfoDigest.substring(0, 1) + "/" - + extraInfoDigest.substring(1, 2) + "/" - + extraInfoDigest); + File tarballFile = Paths.get(this.outputDirectory, "extra-info", + descriptorFormat.format(new Date(published)), + extraInfoDigest.substring(0, 1), + extraInfoDigest.substring(1, 2), + extraInfoDigest).toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); - File rsyncCatFile = new File("recent/relay-descriptors/" - + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp"); + File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, + "extra-infos", this.rsyncCatString + "-extra-infos.tmp").toFile(); File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; boolean[] append = new boolean[] { false, true }; if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) { @@ -784,15 +797,14 @@ public class ArchiveWriter extends Thread { * valid-after months. */ SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/"); descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File(this.outputDirectory + "/microdesc/" - + descriptorFormat.format(validAfter) + "micro/" - + microdescriptorDigest.substring(0, 1) + "/" - + microdescriptorDigest.substring(1, 2) + "/" - + microdescriptorDigest); + File tarballFile = Paths.get(this.outputDirectory, MICRODESC, + descriptorFormat.format(validAfter), MICRO, + microdescriptorDigest.substring(0, 1), + microdescriptorDigest.substring(1, 2), + microdescriptorDigest).toFile(); boolean tarballFileExistedBefore = tarballFile.exists(); - File rsyncCatFile = new File("recent/relay-descriptors/" - + "microdescs/micro/" + this.rsyncCatString - + "-micro.tmp"); + File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, + MICRODESCS, MICRO, this.rsyncCatString + "-micro.tmp").toFile(); File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; boolean[] append = new boolean[] { false, true }; if (this.store(MICRODESCRIPTOR_ANNOTATION, data, outputFiles, diff --git a/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java index b9001dd..00eeab1 100644 --- a/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java @@ -35,10 +35,10 @@ import java.util.logging.Logger; */ public class CachedRelayDescriptorReader { public CachedRelayDescriptorReader(RelayDescriptorParser rdp, - List<String> inputDirectories, File statsDirectory) { + String[] inputDirectories, File statsDirectory) {

if (rdp == null || inputDirectories == null - || inputDirectories.isEmpty() || statsDirectory == null) { + || inputDirectories.length == 0 || statsDirectory == null) { throw new IllegalArgumentException(); }

diff --git a/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java index 458332a..bd0a482 100644 --- a/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java @@ -19,7 +19,7 @@ import java.net.HttpURLConnection; import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -288,7 +288,7 @@ public class RelayDescriptorDownloader { * <code>stats/last-downloaded-all-descriptors</code>. */ public RelayDescriptorDownloader(RelayDescriptorParser rdp, - List<String> authorities, List<String> authorityFingerprints, + String[] authorities, String[] authorityFingerprints, boolean downloadCurrentConsensus, boolean downloadCurrentMicrodescConsensus, boolean downloadCurrentVotes, @@ -300,9 +300,8 @@ public class RelayDescriptorDownloader {

/* Memorize argument values. */ this.rdp = rdp; - this.authorities = new ArrayList<String>(authorities); - this.authorityFingerprints = new ArrayList<String>( - authorityFingerprints); + this.authorities = Arrays.asList(authorities); + this.authorityFingerprints = Arrays.asList(authorityFingerprints); this.downloadCurrentConsensus = downloadCurrentConsensus; this.downloadCurrentMicrodescConsensus = downloadCurrentMicrodescConsensus; diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java index 7bcfbf3..c80f99e 100644 --- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java @@ -3,7 +3,9 @@

package org.torproject.collector.torperf;

-import org.torproject.collector.main.Configuration; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; import org.torproject.collector.main.LockFile;

import java.io.BufferedReader; @@ -17,6 +19,7 @@ import java.net.HttpURLConnection; import java.net.URL; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -30,22 +33,14 @@ import java.util.logging.Logger; * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ public class TorperfDownloader extends Thread { + private static Logger logger = Logger.getLogger(TorperfDownloader.class.getName());

- public static void main(String[] args) { - - Logger logger = Logger.getLogger(TorperfDownloader.class.getName()); + public static void main(Configuration config) throws ConfigurationException { logger.info("Starting torperf module of CollecTor.");

- // Initialize configuration - Configuration config = new Configuration(); - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile("torperf"); - if (!lf.acquireLock()) { - logger.severe("Warning: CollecTor is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } + LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "torperf"); + lf.acquireLock();

// Process Torperf files new TorperfDownloader(config).run(); @@ -63,30 +58,34 @@ public class TorperfDownloader extends Thread { }

private File torperfOutputDirectory = null; - private SortedMap<String, String> torperfSources = null; - private List<String> torperfFilesLines = null; - private Logger logger = null; + private Map<String, String> torperfSources = new HashMap<>(); + private String[] torperfFilesLines = null; private SimpleDateFormat dateFormat;

public void run() { + try { + startProcessing(); + } catch (ConfigurationException ce) { + logger.severe("Configuration failed: " + ce); + throw new RuntimeException(ce); + } + }

- File torperfOutputDirectory = - new File(config.getTorperfOutputDirectory()); - SortedMap<String, String> torperfSources = config.getTorperfSources(); - List<String> torperfFilesLines = config.getTorperfFiles(); - - this.torperfOutputDirectory = torperfOutputDirectory; - this.torperfSources = torperfSources; - this.torperfFilesLines = torperfFilesLines; + private void startProcessing() throws ConfigurationException { + this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines); + this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory) + .toFile(); if (!this.torperfOutputDirectory.exists()) { this.torperfOutputDirectory.mkdirs(); } - this.logger = Logger.getLogger(TorperfDownloader.class.getName()); this.dateFormat = new SimpleDateFormat("yyyy-MM-dd"); this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); this.readLastMergedTimestamps(); + for (String[] source : config.getStringArrayArray(Key.TorperfSources)) { + torperfSources.put(source[0], source[1]); + } for (String torperfFilesLine : this.torperfFilesLines) { - this.downloadAndMergeFiles(torperfFilesLine); + this.downloadAndMergeFiles(torperfFilesLine); } this.writeLastMergedTimestamps();

@@ -161,10 +160,10 @@ public class TorperfDownloader extends Thread {

private void downloadAndMergeFiles(String torperfFilesLine) { String[] parts = torperfFilesLine.split(" "); - String sourceName = parts[1]; + String sourceName = parts[0]; int fileSize = -1; try { - fileSize = Integer.parseInt(parts[2]); + fileSize = Integer.parseInt(parts[1]); } catch (NumberFormatException e) { this.logger.log(Level.WARNING, "Could not parse file size in " + "TorperfFiles configuration line '" + torperfFilesLine @@ -173,7 +172,7 @@ public class TorperfDownloader extends Thread { }

/* Download and append the .data file. */ - String dataFileName = parts[3]; + String dataFileName = parts[2]; String sourceBaseUrl = torperfSources.get(sourceName); String dataUrl = sourceBaseUrl + dataFileName; String dataOutputFileName = sourceName + "-" + dataFileName; @@ -183,7 +182,7 @@ public class TorperfDownloader extends Thread { dataOutputFile, true);

/* Download and append the .extradata file. */ - String extradataFileName = parts[4]; + String extradataFileName = parts[3]; String extradataUrl = sourceBaseUrl + extradataFileName; String extradataOutputFileName = sourceName + "-" + extradataFileName; File extradataOutputFile = new File(torperfOutputDirectory, diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties new file mode 100644 index 0000000..2645d01 --- /dev/null +++ b/src/main/resources/collector.properties @@ -0,0 +1,115 @@ +######## Collector Properties +# +######## General Properties ######## +LockFilePath = lock +IndexPath = out/index +ArchivePath = out/archive +RecentPath = out/recent +StatsPath = out/stats + +######## Relay descriptors ######## +# +## Read cached-* files from a local Tor data directory +ImportCachedRelayDescriptors = false +# +## Relative path to Tor data directory to read cached-* files from +## the listed path(s). If there is more that one separated by comma. +CachedRelayDescriptorsDirectories = in/relay-descriptors/cacheddesc/ +# +## Import directory archives from disk, if available +ImportDirectoryArchives = false +# +## Relative path to directory to import directory archives from +DirectoryArchivesDirectory = in/relay-descriptors/archives/ +# +## Keep a history of imported directory archive files to know which files +## have been imported before. This history can be useful when importing +## from a changing source to avoid importing descriptors over and over +## again, but it can be confusing to users who don't know about it. +KeepDirectoryArchiveImportHistory = false +# +## Download relay descriptors from directory authorities, if required +DownloadRelayDescriptors = false +# +## Comma separated list of directory authority addresses (IP[:port]) to +## download missing relay descriptors from +DirectoryAuthoritiesAddresses = 86.59.21.38,76.73.17.194:9030,171.25.193.9:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170,154.35.32.5 +# +## Comma separated list of directory authority fingerprints to download +## votes +DirectoryAuthoritiesFingerprintsForVotes = 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,27B6B5996C426270A5C95488AA5BCEB6BCC86956,49015F787433103580E3B66A1707A00E60F2D15B,585769C78764D58426B8B52B6651A5A71137189A,80550987E1D626E3EBA5E5E75A458DE0626D088C,D586D18309DED4CD6D57C18FDB97EFA96D330566,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,ED03BB616EB2F60BEC80151114BB25CEF515B226,EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 +# +## Download the current consensus (only if DownloadRelayDescriptors is 1) +DownloadCurrentConsensus = true +# +## Download the current microdesc consensus (only if +## DownloadRelayDescriptors is true) +DownloadCurrentMicrodescConsensus = true +# +## Download current votes (only if DownloadRelayDescriptors is true) +DownloadCurrentVotes = true +# +## Download missing server descriptors (only if DownloadRelayDescriptors +## is true) +DownloadMissingServerDescriptors = true +# +## Download missing extra-info descriptors (only if +## DownloadRelayDescriptors is true) +DownloadMissingExtraInfoDescriptors = true +# +## Download missing microdescriptors (only if +## DownloadRelayDescriptors is true) +DownloadMissingMicrodescriptors = true +# +## Download all server descriptors from the directory authorities at most +## once a day (only if DownloadRelayDescriptors is true) +DownloadAllServerDescriptors false +# +## Download all extra-info descriptors from the directory authorities at +## most once a day (only if DownloadRelayDescriptors is true) +DownloadAllExtraInfoDescriptors = false +# +## Compress relay descriptors downloads by adding .z to the URLs +CompressRelayDescriptorDownloads = false +# +## Relative path to directory to write directory archives to +DirectoryArchivesOutputDirectory = out/relay-descriptors/ +# +# +######## Bridge descriptors ######## +# +## Relative path to directory to import bridge descriptor snapshots from +BridgeSnapshotsDirectory = in/bridge-descriptors/ +# +## Replace IP addresses in sanitized bridge descriptors with 10.x.y.z +## where x.y.z = H(IP address | bridge identity | secret)[:3], so that we +## can learn about IP address changes. +ReplaceIPAddressesWithHashes = false +# +## Limit internal bridge descriptor mapping state to the following number +## of days, or inf for unlimited. +BridgeDescriptorMappingsLimit = inf +# +## Relative path to directory to write sanitized bridges to +SanitizedBridgesWriteDirectory = out/bridge-descriptors/ + +######## Exit lists ######## +# +## (No options available) +# +# +######## Torperf downloader ######## +# +## Path to the directory to store Torperf files in. +## A relative path starts with ./ +TorperfOutputDirectory = out/torperf/ + +## Torperf source names and base URLs +## multiple pairs can be specified separated by semi-colon, e.g. +## TorperfSourceName = torperf_A, http://some.torproject.org/; another, http://another.torproject.org/ +TorperfSources = torperf, http://torperf.torproject.org/ + +## Torperf measurement file size in bytes, .data file, and .extradata file +## available on a given source (multiple times lists can be given +## TorperfFiles = torperf 51200 50kb.data 50kb.extradata, torperf 1048576 1mb.data 1mb.extradata +TorperfFilesLines = torperf 51200 50kb.data 50kb.extradata, torperf 1048576 1mb.data 1mb.extradata, torperf 5242880 5mb.data 5mb.extradata diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java new file mode 100644 index 0000000..9a19285 --- /dev/null +++ b/src/test/java/org/torproject/collector/MainTest.java @@ -0,0 +1,72 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.collector; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import org.torproject.collector.conf.Key; +import org.torproject.collector.conf.ConfigurationException; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.net.URL; +import java.io.BufferedWriter; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.AccessControlException; +import java.security.Policy; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.junit.rules.TemporaryFolder; +import org.junit.Rule; +import org.junit.Test; + +public class MainTest { + + private Random randomSource = new Random(); + + @Rule + public TemporaryFolder tmpf = new TemporaryFolder(); + + @Test() + public void testSmoke() throws Exception { + System.out.println("\n!!!! Three SEVERE log messages are expected." + + "\nOne each from: ExitListDownloader, " + + "TorperfDownloader, and CreateIndexJson.\n"); + File conf = tmpf.newFile("test.conf"); + File lockPath = tmpf.newFolder("test.lock"); + assertEquals(0L, conf.length()); + Main.main(new String[]{"relaydescs", conf.toString()}); + assertTrue(4_000L <= conf.length()); + changeLockFilePath(conf, lockPath); + for ( String key : Main.collecTorMains.keySet()) { + Main.main(new String[]{key, conf.toString()}); + } + } + + private void changeLockFilePath(File f, File l) throws Exception { + List<String> lines = Files.readAllLines(f.toPath()); + BufferedWriter bw = Files.newBufferedWriter(f.toPath()); + File out = tmpf.newFolder(); + for(String line : lines) { + if (line.contains(Key.LockFilePath.name())) { + line = Key.LockFilePath.name() + " = " + l.toString(); + } else if (line.contains("out")) { + line = line.replace("out", out.toString() + "out"); + } + bw.write(line); + bw.newLine(); + } + bw.flush(); + bw.close(); + } + +} diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java new file mode 100644 index 0000000..aa98031 --- /dev/null +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -0,0 +1,143 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.collector.conf; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.junit.Test; + +public class ConfigurationTest { + + private Random randomSource = new Random(); + + @Test() + public void testKeyCount() throws Exception { + assertEquals("The number of properties keys in enum Key changed." + + "\n This test class should be adapted.", + 30, Key.values().length); + } + + @Test() + public void testConfiguration() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("TorperfOutputDirectory = xyz".getBytes())); + assertEquals(1, conf.size()); + assertEquals("xyz", conf.getProperty("TorperfOutputDirectory")); + } + + @Test() + public void testArrayValues() throws Exception { + String[] array = new String[randomSource.nextInt(30) + 1]; + for (int i = 0; i < array.length; i++){ + array[i] = Integer.toBinaryString(randomSource.nextInt(100)); + } + String[] arrays = new String[] { + Arrays.toString(array).replace("[", "").replace("]", ""), + Arrays.toString(array).replace("[", "").replace("]", "").replaceAll(" ", "") + }; + Configuration conf = new Configuration(); + for(String input : arrays) { + conf.clear(); + conf.load(new ByteArrayInputStream(("CachedRelayDescriptorsDirectories = " + input).getBytes())); + assertArrayEquals("expected " + Arrays.toString(array) + "\nreceived: " + + Arrays.toString(conf.getStringArray(Key.CachedRelayDescriptorsDirectories)), + array, conf.getStringArray(Key.CachedRelayDescriptorsDirectories)); + } + } + + @Test() + public void testBoolValues() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream(("CompressRelayDescriptorDownloads=false" + + "\nImportDirectoryArchives = trUe" + + "\nReplaceIPAddressesWithHashes= false").getBytes())); + assertFalse(conf.getBool(Key.CompressRelayDescriptorDownloads)); + assertTrue(conf.getBool(Key.ImportDirectoryArchives)); + assertFalse(conf.getBool(Key.ReplaceIPAddressesWithHashes)); + } + + @Test() + public void testIntValues() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = inf".getBytes())); + assertEquals(Integer.MAX_VALUE, + conf.getInt(Key.BridgeDescriptorMappingsLimit)); + int r = randomSource.nextInt(Integer.MAX_VALUE); + conf.clear(); + conf.load(new ByteArrayInputStream(("BridgeDescriptorMappingsLimit =" + r).getBytes())); + assertEquals(r, + conf.getInt(Key.BridgeDescriptorMappingsLimit)); + } + + @Test() + public void testFileValues() throws Exception { + String[] files = new String[] { "/the/path/file.txt", "another/path"}; + Configuration conf = new Configuration(); + for(String file : files) { + conf.clear(); + conf.load(new ByteArrayInputStream(("DirectoryArchivesOutputDirectory = " + file).getBytes())); + assertEquals(new File(file), conf.getPath(Key.DirectoryArchivesOutputDirectory).toFile()); + } + } + + @Test() + public void testArrayArrayValues() throws Exception { + String[][] sourceStrings = new String[][] { + new String[]{"localsource", "http://127.0.0.1:12345%22%7D, + new String[]{"somesource", "https://some.host.org:12345%22%7D%7D; + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream(("TorperfSources = " + + Arrays.deepToString(sourceStrings)).replace("[[", "").replace("]]", "") + .replace("], [", Configuration.ARRAYSEP).getBytes())); + assertArrayEquals(sourceStrings, conf.getStringArrayArray(Key.TorperfSources)); + } + + @Test( expected = ConfigurationException.class) + public void testArrayArrayValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes())); + conf.getStringArrayArray(Key.TorperfOutputDirectory); + } + + @Test( expected = ConfigurationException.class) + public void testArrayValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes())); + conf.getStringArray(Key.TorperfSources); + } + + @Test( expected = ConfigurationException.class) + public void testBoolValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("TorperfSource = http://x.y.z%22.getBytes())); + conf.getBool(Key.CachedRelayDescriptorsDirectories); + } + + @Test( expected = ConfigurationException.class) + public void testPathValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("DirectoryArchivesDirectory = \u0000:".getBytes())); + conf.getPath(Key.DirectoryArchivesDirectory); + } + + @Test( expected = ConfigurationException.class) + public void testIntValueException() throws Exception { + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = y7".getBytes())); + conf.getInt(Key.BridgeDescriptorMappingsLimit); + } + +} diff --git a/src/test/resources/junittest.policy b/src/test/resources/junittest.policy new file mode 100644 index 0000000..e6eb2ef --- /dev/null +++ b/src/test/resources/junittest.policy @@ -0,0 +1,10 @@ +/* Prevent tests from bothering production servers. */ + +grant { + permission java.io.FilePermission "<<ALL FILES>>", "read, write, delete, execute"; + permission java.util.PropertyPermission "*", "read, write"; + permission java.lang.RuntimePermission "setIO"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.RuntimePermission "shutdownHooks"; +};