[tor-commits] [collector/master] Implements #19424: make all main paths configurable, add property instance url.
karsten at torproject.org
karsten at torproject.org
Wed Jul 20 07:38:26 UTC 2016
commit 2f725efbf6a527bee8663fca3751da7069787625
Author: iwakeh <iwakeh at torproject.org>
Date: Tue Jul 19 15:32:31 2016 +0200
Implements #19424: make all main paths configurable, add property instance url.
---
.../bridgedescs/SanitizedBridgesWriter.java | 28 ++++++++++++----------
.../java/org/torproject/collector/conf/Key.java | 2 ++
.../collector/exitlists/ExitListDownloader.java | 22 ++++++++++-------
.../collector/index/CreateIndexJson.java | 7 +++---
.../collector/torperf/TorperfDownloader.java | 25 +++++++++++--------
src/main/resources/collector.properties | 7 +++---
.../collector/conf/ConfigurationTest.java | 2 +-
7 files changed, 55 insertions(+), 38 deletions(-)
diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 9d5828c..121f8ca 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -120,7 +120,7 @@ public class SanitizedBridgesWriter extends Thread {
config.getPath(Key.BridgeSnapshotsDirectory).toFile();
File sanitizedBridgesDirectory =
config.getPath(Key.SanitizedBridgesWriteDirectory).toFile();
- File statsDirectory = new File("stats");
+ File statsDirectory = config.getPath(Key.StatsPath).toFile();
if (bridgeDirectoriesDirectory == null
|| sanitizedBridgesDirectory == null || statsDirectory == null) {
@@ -923,12 +923,12 @@ public class SanitizedBridgesWriter extends Thread {
+ "/" + descriptorDigest.charAt(0) + "/"
+ descriptorDigest.charAt(1) + "/"
+ descriptorDigest);
- File rsyncCatFile = new File("recent/bridge-descriptors/"
- + "server-descriptors/" + this.rsyncCatString
- + "-server-descriptors.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
try {
+ File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),
+ "bridge-descriptors/server-descriptors/" + this.rsyncCatString
+ + "-server-descriptors.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
for (int i = 0; i < outputFiles.length; i++) {
File outputFile = outputFiles[i];
boolean appendToFile = append[i];
@@ -950,7 +950,7 @@ public class SanitizedBridgesWriter extends Thread {
+ "\n");
bw.close();
}
- } catch (IOException e) {
+ } catch (ConfigurationException | IOException e) {
this.logger.warn("Could not write sanitized server "
+ "descriptor to disk.", e);
return;
@@ -1201,11 +1201,12 @@ public class SanitizedBridgesWriter extends Thread {
+ descriptorDigest.charAt(0) + "/"
+ descriptorDigest.charAt(1) + "/"
+ descriptorDigest);
- File rsyncCatFile = new File("recent/bridge-descriptors/"
- + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
try {
+ File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),
+ "bridge-descriptors/extra-infos/" + this.rsyncCatString
+ + "-extra-infos.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
for (int i = 0; i < outputFiles.length; i++) {
File outputFile = outputFiles[i];
boolean appendToFile = append[i];
@@ -1313,11 +1314,12 @@ public class SanitizedBridgesWriter extends Thread {
/** Delete all files from the rsync directory that have not been modified
* in the last three days, and remove the .tmp extension from newly
* written files. */
- public void cleanUpRsyncDirectory() {
+ public void cleanUpRsyncDirectory() throws ConfigurationException {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/bridge-descriptors"));
+ allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
+ "bridge-descriptors"));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java
index 21e536d..4f5df64 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -8,6 +8,8 @@ import java.nio.file.Path;
*/
public enum Key {
+ ExitlistOutputDirectory(Path.class),
+ InstanceBaseUrl(String.class),
LockFilePath(Path.class),
ArchivePath(Path.class),
RecentPath(Path.class),
diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
index 5e17971..70de74f 100644
--- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -53,7 +53,12 @@ public class ExitListDownloader extends Thread {
logger.info("Terminating exit-lists module of CollecTor.");
}
- public ExitListDownloader(Configuration config) {}
+ private Configuration config;
+
+ /** Instanciate the exit-lists module using the given configuration. */
+ public ExitListDownloader(Configuration config) {
+ this.config = config;
+ }
@Override
public void run() {
@@ -113,8 +118,8 @@ public class ExitListDownloader extends Thread {
SimpleDateFormat tarballFormat =
new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File("out/exit-lists/" + tarballFormat.format(
- downloadedDate));
+ File tarballFile = new File(config.getPath(Key.ExitlistOutputDirectory)
+ + tarballFormat.format(downloadedDate));
long maxScanMillis = 0L;
try {
@@ -146,8 +151,8 @@ public class ExitListDownloader extends Thread {
}
/* Write to disk. */
- File rsyncFile = new File("recent/exit-lists/"
- + tarballFile.getName());
+ File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
+ "exit-lists/" + tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
for (File outputFile : outputFiles) {
try {
@@ -166,7 +171,7 @@ public class ExitListDownloader extends Thread {
StringBuilder dumpStats = new StringBuilder("Finished downloading "
+ "exit list.\nLast three exit lists are:");
Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(new File("out/exit-lists"));
+ filesInInputDir.add(config.getPath(Key.ExitlistOutputDirectory).toFile());
SortedSet<File> lastThreeExitLists = new TreeSet<File>();
while (!filesInInputDir.isEmpty()) {
File pop = filesInInputDir.pop();
@@ -198,11 +203,12 @@ public class ExitListDownloader extends Thread {
/** Delete all files from the rsync directory that have not been modified
* in the last three days. */
- public void cleanUpRsyncDirectory() {
+ public void cleanUpRsyncDirectory() throws ConfigurationException {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/exit-lists"));
+ allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
+ "/exit-lists"));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
index 821be59..7da2b5e 100644
--- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java
+++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
@@ -39,7 +39,7 @@ public class CreateIndexJson {
private static File indexJsonFile;
- private static String basePath = "https://collector.torproject.org";
+ private static String basePath;
private static File[] indexedDirectories;
@@ -54,9 +54,10 @@ public class CreateIndexJson {
public static void main(Configuration config)
throws ConfigurationException, IOException {
indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json");
+ basePath = config.getProperty(Key.InstanceBaseUrl.name());
indexedDirectories = new File[] {
- new File(config.getPath(Key.ArchivePath).toFile(), "archive"),
- new File(config.getPath(Key.RecentPath).toFile(), "recent") };
+ config.getPath(Key.ArchivePath).toFile(),
+ config.getPath(Key.RecentPath).toFile() };
writeIndex(indexDirectories());
}
diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
index dd1c541..7616dd8 100644
--- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
+++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
@@ -62,6 +62,7 @@ public class TorperfDownloader extends Thread {
private Map<String, String> torperfSources = new HashMap<>();
private String[] torperfFilesLines = null;
private SimpleDateFormat dateFormat;
+ private File torperfLastMergedFile;
@Override
public void run() {
@@ -77,6 +78,8 @@ public class TorperfDownloader extends Thread {
this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines);
this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory)
.toFile();
+ this.torperfLastMergedFile = new File(config.getPath(Key.StatsPath).toFile(),
+ "torperf-last-merged");
if (!this.torperfOutputDirectory.exists()) {
this.torperfOutputDirectory.mkdirs();
}
@@ -94,9 +97,6 @@ public class TorperfDownloader extends Thread {
this.cleanUpRsyncDirectory();
}
- private File torperfLastMergedFile =
- new File("stats/torperf-last-merged");
-
SortedMap<String, String> lastMergedTimestamps =
new TreeMap<String, String>();
@@ -160,7 +160,8 @@ public class TorperfDownloader extends Thread {
}
}
- private void downloadAndMergeFiles(String torperfFilesLine) {
+ private void downloadAndMergeFiles(String torperfFilesLine)
+ throws ConfigurationException {
String[] parts = torperfFilesLine.split(" ");
String sourceName = parts[0];
int fileSize = -1;
@@ -306,7 +307,8 @@ public class TorperfDownloader extends Thread {
}
private String mergeFiles(File dataFile, File extradataFile,
- String source, int fileSize, String skipUntil) throws IOException {
+ String source, int fileSize, String skipUntil) throws IOException,
+ ConfigurationException {
SortedMap<String, String> config = new TreeMap<String, String>();
config.put("SOURCE", source);
config.put("FILESIZE", String.valueOf(fileSize));
@@ -548,7 +550,8 @@ public class TorperfDownloader extends Thread {
private SortedMap<String, String> cachedTpfLines;
private void writeTpfLine(String source, int fileSize,
- SortedMap<String, String> keysAndValues) throws IOException {
+ SortedMap<String, String> keysAndValues) throws ConfigurationException,
+ IOException {
StringBuilder sb = new StringBuilder();
int written = 0;
for (Map.Entry<String, String> keyAndValue :
@@ -602,7 +605,8 @@ public class TorperfDownloader extends Thread {
br.close();
}
- private void writeCachedTpfLines() throws IOException {
+ private void writeCachedTpfLines() throws ConfigurationException,
+ IOException {
if (this.cachedSource == null || this.cachedFileSize == 0
|| this.cachedStartDate == null || this.cachedTpfLines == null) {
return;
@@ -612,7 +616,8 @@ public class TorperfDownloader extends Thread {
+ "/" + this.cachedSource + "-"
+ String.valueOf(this.cachedFileSize) + "-"
+ this.cachedStartDate + ".tpf");
- File rsyncFile = new File("recent/torperf/" + tarballFile.getName());
+ File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
+ "torperf/" + tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
for (File outputFile : outputFiles) {
outputFile.getParentFile().mkdirs();
@@ -631,11 +636,11 @@ public class TorperfDownloader extends Thread {
/** Delete all files from the rsync directory that have not been modified
* in the last three days. */
- public void cleanUpRsyncDirectory() {
+ public void cleanUpRsyncDirectory() throws ConfigurationException {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/torperf"));
+ allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), "torperf"));
while (!allFiles.isEmpty()) {
File file = allFiles.pop();
if (file.isDirectory()) {
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index c7df21f..fb9e1b4 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -1,6 +1,7 @@
######## Collector Properties
#
######## General Properties ########
+InstanceBaseUrl = "https://collector.torproject.org"
LockFilePath = lock
IndexPath = out/index
ArchivePath = out/archive
@@ -95,9 +96,9 @@ SanitizedBridgesWriteDirectory = out/bridge-descriptors/
######## Exit lists ########
#
-## (No options available)
-#
-#
+##
+ExitlistOutputDirectory = out/exit-lists/
+
######## Torperf downloader ########
#
## Path to the directory to store Torperf files in.
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
index f966b43..57dda75 100644
--- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -22,7 +22,7 @@ public class ConfigurationTest {
public void testKeyCount() throws Exception {
assertEquals("The number of properties keys in enum Key changed."
+ "\n This test class should be adapted.",
- 30, Key.values().length);
+ 32, Key.values().length);
}
@Test()
More information about the tor-commits
mailing list