commit 2f725efbf6a527bee8663fca3751da7069787625 Author: iwakeh iwakeh@torproject.org Date: Tue Jul 19 15:32:31 2016 +0200

Implements #19424: make all main paths configurable, add property instance url. --- .../bridgedescs/SanitizedBridgesWriter.java | 28 ++++++++++++---------- .../java/org/torproject/collector/conf/Key.java | 2 ++ .../collector/exitlists/ExitListDownloader.java | 22 ++++++++++------- .../collector/index/CreateIndexJson.java | 7 +++--- .../collector/torperf/TorperfDownloader.java | 25 +++++++++++-------- src/main/resources/collector.properties | 7 +++--- .../collector/conf/ConfigurationTest.java | 2 +- 7 files changed, 55 insertions(+), 38 deletions(-)

diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 9d5828c..121f8ca 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -120,7 +120,7 @@ public class SanitizedBridgesWriter extends Thread { config.getPath(Key.BridgeSnapshotsDirectory).toFile(); File sanitizedBridgesDirectory = config.getPath(Key.SanitizedBridgesWriteDirectory).toFile(); - File statsDirectory = new File("stats"); + File statsDirectory = config.getPath(Key.StatsPath).toFile();

if (bridgeDirectoriesDirectory == null || sanitizedBridgesDirectory == null || statsDirectory == null) { @@ -923,12 +923,12 @@ public class SanitizedBridgesWriter extends Thread { + "/" + descriptorDigest.charAt(0) + "/" + descriptorDigest.charAt(1) + "/" + descriptorDigest); - File rsyncCatFile = new File("recent/bridge-descriptors/" - + "server-descriptors/" + this.rsyncCatString - + "-server-descriptors.tmp"); - File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; - boolean[] append = new boolean[] { false, true }; try { + File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(), + "bridge-descriptors/server-descriptors/" + this.rsyncCatString + + "-server-descriptors.tmp"); + File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; + boolean[] append = new boolean[] { false, true }; for (int i = 0; i < outputFiles.length; i++) { File outputFile = outputFiles[i]; boolean appendToFile = append[i]; @@ -950,7 +950,7 @@ public class SanitizedBridgesWriter extends Thread { + "\n"); bw.close(); } - } catch (IOException e) { + } catch (ConfigurationException | IOException e) { this.logger.warn("Could not write sanitized server " + "descriptor to disk.", e); return; @@ -1201,11 +1201,12 @@ public class SanitizedBridgesWriter extends Thread { + descriptorDigest.charAt(0) + "/" + descriptorDigest.charAt(1) + "/" + descriptorDigest); - File rsyncCatFile = new File("recent/bridge-descriptors/" - + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp"); - File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; - boolean[] append = new boolean[] { false, true }; try { + File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(), + "bridge-descriptors/extra-infos/" + this.rsyncCatString + + "-extra-infos.tmp"); + File[] outputFiles = new File[] { tarballFile, rsyncCatFile }; + boolean[] append = new boolean[] { false, true }; for (int i = 0; i < outputFiles.length; i++) { File outputFile = outputFiles[i]; boolean appendToFile = append[i]; @@ -1313,11 +1314,12 @@ public class SanitizedBridgesWriter extends Thread { /** Delete all files from the rsync directory that have not been modified * in the last three days, and remove the .tmp extension from newly * written files. */ - public void cleanUpRsyncDirectory() { + public void cleanUpRsyncDirectory() throws ConfigurationException { long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/bridge-descriptors")); + allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), + "bridge-descriptors")); while (!allFiles.isEmpty()) { File file = allFiles.pop(); if (file.isDirectory()) { diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index 21e536d..4f5df64 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -8,6 +8,8 @@ import java.nio.file.Path; */ public enum Key {

+ ExitlistOutputDirectory(Path.class), + InstanceBaseUrl(String.class), LockFilePath(Path.class), ArchivePath(Path.class), RecentPath(Path.class), diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 5e17971..70de74f 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -53,7 +53,12 @@ public class ExitListDownloader extends Thread { logger.info("Terminating exit-lists module of CollecTor."); }

- public ExitListDownloader(Configuration config) {} + private Configuration config; + + /** Instanciate the exit-lists module using the given configuration. */ + public ExitListDownloader(Configuration config) { + this.config = config; + }

@Override public void run() { @@ -113,8 +118,8 @@ public class ExitListDownloader extends Thread { SimpleDateFormat tarballFormat = new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - File tarballFile = new File("out/exit-lists/" + tarballFormat.format( - downloadedDate)); + File tarballFile = new File(config.getPath(Key.ExitlistOutputDirectory) + + tarballFormat.format(downloadedDate));

long maxScanMillis = 0L; try { @@ -146,8 +151,8 @@ public class ExitListDownloader extends Thread { }

/* Write to disk. */ - File rsyncFile = new File("recent/exit-lists/" - + tarballFile.getName()); + File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(), + "exit-lists/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile }; for (File outputFile : outputFiles) { try { @@ -166,7 +171,7 @@ public class ExitListDownloader extends Thread { StringBuilder dumpStats = new StringBuilder("Finished downloading " + "exit list.\nLast three exit lists are:"); Stack<File> filesInInputDir = new Stack<File>(); - filesInInputDir.add(new File("out/exit-lists")); + filesInInputDir.add(config.getPath(Key.ExitlistOutputDirectory).toFile()); SortedSet<File> lastThreeExitLists = new TreeSet<File>(); while (!filesInInputDir.isEmpty()) { File pop = filesInInputDir.pop(); @@ -198,11 +203,12 @@ public class ExitListDownloader extends Thread {

/** Delete all files from the rsync directory that have not been modified * in the last three days. */ - public void cleanUpRsyncDirectory() { + public void cleanUpRsyncDirectory() throws ConfigurationException { long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/exit-lists")); + allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), + "/exit-lists")); while (!allFiles.isEmpty()) { File file = allFiles.pop(); if (file.isDirectory()) { diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 821be59..7da2b5e 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -39,7 +39,7 @@ public class CreateIndexJson {

private static File indexJsonFile;

- private static String basePath = "https://collector.torproject.org"; + private static String basePath;

private static File[] indexedDirectories;

@@ -54,9 +54,10 @@ public class CreateIndexJson { public static void main(Configuration config) throws ConfigurationException, IOException { indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); + basePath = config.getProperty(Key.InstanceBaseUrl.name()); indexedDirectories = new File[] { - new File(config.getPath(Key.ArchivePath).toFile(), "archive"), - new File(config.getPath(Key.RecentPath).toFile(), "recent") }; + config.getPath(Key.ArchivePath).toFile(), + config.getPath(Key.RecentPath).toFile() }; writeIndex(indexDirectories()); }

diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java index dd1c541..7616dd8 100644 --- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java @@ -62,6 +62,7 @@ public class TorperfDownloader extends Thread { private Map<String, String> torperfSources = new HashMap<>(); private String[] torperfFilesLines = null; private SimpleDateFormat dateFormat; + private File torperfLastMergedFile;

@Override public void run() { @@ -77,6 +78,8 @@ public class TorperfDownloader extends Thread { this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines); this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory) .toFile(); + this.torperfLastMergedFile = new File(config.getPath(Key.StatsPath).toFile(), + "torperf-last-merged"); if (!this.torperfOutputDirectory.exists()) { this.torperfOutputDirectory.mkdirs(); } @@ -94,9 +97,6 @@ public class TorperfDownloader extends Thread { this.cleanUpRsyncDirectory(); }

- private File torperfLastMergedFile = - new File("stats/torperf-last-merged"); - SortedMap<String, String> lastMergedTimestamps = new TreeMap<String, String>();

@@ -160,7 +160,8 @@ public class TorperfDownloader extends Thread { } }

- private void downloadAndMergeFiles(String torperfFilesLine) { + private void downloadAndMergeFiles(String torperfFilesLine) + throws ConfigurationException { String[] parts = torperfFilesLine.split(" "); String sourceName = parts[0]; int fileSize = -1; @@ -306,7 +307,8 @@ public class TorperfDownloader extends Thread { }

private String mergeFiles(File dataFile, File extradataFile, - String source, int fileSize, String skipUntil) throws IOException { + String source, int fileSize, String skipUntil) throws IOException, + ConfigurationException { SortedMap<String, String> config = new TreeMap<String, String>(); config.put("SOURCE", source); config.put("FILESIZE", String.valueOf(fileSize)); @@ -548,7 +550,8 @@ public class TorperfDownloader extends Thread { private SortedMap<String, String> cachedTpfLines;

private void writeTpfLine(String source, int fileSize, - SortedMap<String, String> keysAndValues) throws IOException { + SortedMap<String, String> keysAndValues) throws ConfigurationException, + IOException { StringBuilder sb = new StringBuilder(); int written = 0; for (Map.Entry<String, String> keyAndValue : @@ -602,7 +605,8 @@ public class TorperfDownloader extends Thread { br.close(); }

- private void writeCachedTpfLines() throws IOException { + private void writeCachedTpfLines() throws ConfigurationException, + IOException { if (this.cachedSource == null || this.cachedFileSize == 0 || this.cachedStartDate == null || this.cachedTpfLines == null) { return; @@ -612,7 +616,8 @@ public class TorperfDownloader extends Thread { + "/" + this.cachedSource + "-" + String.valueOf(this.cachedFileSize) + "-" + this.cachedStartDate + ".tpf"); - File rsyncFile = new File("recent/torperf/" + tarballFile.getName()); + File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(), + "torperf/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile }; for (File outputFile : outputFiles) { outputFile.getParentFile().mkdirs(); @@ -631,11 +636,11 @@ public class TorperfDownloader extends Thread {

/** Delete all files from the rsync directory that have not been modified * in the last three days. */ - public void cleanUpRsyncDirectory() { + public void cleanUpRsyncDirectory() throws ConfigurationException { long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<File>(); - allFiles.add(new File("recent/torperf")); + allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), "torperf")); while (!allFiles.isEmpty()) { File file = allFiles.pop(); if (file.isDirectory()) { diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index c7df21f..fb9e1b4 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -1,6 +1,7 @@ ######## Collector Properties # ######## General Properties ######## +InstanceBaseUrl = "https://collector.torproject.org" LockFilePath = lock IndexPath = out/index ArchivePath = out/archive @@ -95,9 +96,9 @@ SanitizedBridgesWriteDirectory = out/bridge-descriptors/

######## Exit lists ######## # -## (No options available) -# -# +## +ExitlistOutputDirectory = out/exit-lists/ + ######## Torperf downloader ######## # ## Path to the directory to store Torperf files in. diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index f966b43..57dda75 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -22,7 +22,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 30, Key.values().length); + 32, Key.values().length); }

@Test()