[tor-commits] [metrics-db/master] Make each module copy its own files to rsync/.
karsten at torproject.org
karsten at torproject.org
Sat Oct 27 20:07:46 UTC 2012
commit 9766f4c4e9282253090c2e682ee55037c8864383
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Fri Oct 26 14:15:16 2012 -0400
Make each module copy its own files to rsync/.
---
.../db/bridgedescs/SanitizedBridgesWriter.java | 15 ++
.../BridgePoolAssignmentsProcessor.java | 10 ++
.../ernie/db/exitlists/ExitListDownloader.java | 12 ++-
src/org/torproject/ernie/db/main/Main.java | 18 +--
.../ernie/db/main/RsyncDataProvider.java | 161 ++++----------------
.../ernie/db/relaydescs/ArchiveWriter.java | 20 +++
.../ernie/db/torperf/TorperfDownloader.java | 8 +
7 files changed, 96 insertions(+), 148 deletions(-)
diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index e46450f..15cb8f1 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -28,6 +28,7 @@ import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
/**
* Sanitizes bridge descriptors, i.e., removes all possibly sensitive
@@ -178,6 +179,20 @@ public class SanitizedBridgesWriter {
// Finish writing sanitized bridge descriptors to disk
this.finishWriting();
+
+ // Copy sanitized bridge descriptors from the last 3 days to rsync
+ // directory.
+ if (config.getProvideFilesViaRsync()) {
+ RsyncDataProvider rdp = new RsyncDataProvider(
+ new File(config.getRsyncDirectory()));
+ rdp.copyFiles(new File(sanitizedBridgesDirectory, "statuses"),
+ "bridge-descriptors/statuses");
+ rdp.copyFiles(
+ new File(sanitizedBridgesDirectory, "server-descriptor"),
+ "bridge-descriptors/server-descriptors");
+ rdp.copyFiles(new File(sanitizedBridgesDirectory, "extra-info"),
+ "bridge-descriptors/extra-infos");
+ }
}
private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 76fa539..63b7c4d 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -27,6 +27,7 @@ import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
public class BridgePoolAssignmentsProcessor {
@@ -172,6 +173,15 @@ public class BridgePoolAssignmentsProcessor {
}
}
+ // Copy sanitized bridge pool assignments from the last 3 days to the
+ // rsync directory.
+ if (config.getProvideFilesViaRsync()) {
+ RsyncDataProvider rdp = new RsyncDataProvider(
+ new File(config.getRsyncDirectory()));
+ rdp.copyFiles(sanitizedAssignmentsDirectory,
+ "bridge-pool-assignments");
+ }
+
logger.info("Finished processing bridge pool assignment file(s).");
}
}
diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
index 64f6a3b..9fb714b 100644
--- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
+++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
@@ -18,8 +18,11 @@ import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
+
public class ExitListDownloader {
- public ExitListDownloader() {
+ public ExitListDownloader(Configuration config) {
Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
try {
logger.fine("Downloading exit list...");
@@ -95,6 +98,13 @@ public class ExitListDownloader {
dumpStats.append("\n" + f.getName());
}
logger.info(dumpStats.toString());
+
+ /* Copy exit lists from the last 3 days to the rsync directory. */
+ if (config.getProvideFilesViaRsync()) {
+ RsyncDataProvider rdp = new RsyncDataProvider(
+ new File(config.getRsyncDirectory()));
+ rdp.copyFiles(new File("exitlist"), "exit-lists");
+ }
}
}
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
index 9109a75..093f002 100644
--- a/src/org/torproject/ernie/db/main/Main.java
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -51,7 +51,7 @@ public class Main {
// Download exit list and store it to disk
if (config.getDownloadExitList()) {
- new ExitListDownloader();
+ new ExitListDownloader(config);
}
// Process bridge pool assignments
@@ -64,22 +64,6 @@ public class Main {
new TorperfDownloader(config);
}
- // Copy recently published files to a local directory that can then
- // be served via rsync.
- if (config.getProvideFilesViaRsync()) {
- new RsyncDataProvider(
- !config.getWriteDirectoryArchives() ? null :
- new File(config.getDirectoryArchivesOutputDirectory()),
- !config.getWriteSanitizedBridges() ? null :
- new File(config.getSanitizedBridgesWriteDirectory()),
- !config.getProcessBridgePoolAssignments() ? null :
- new File(config.getSanitizedAssignmentsDirectory()),
- config.getDownloadExitList(),
- !config.getProcessTorperfFiles() ? null :
- new File(config.getTorperfOutputDirectory()),
- new File(config.getRsyncDirectory()));
- }
-
// Remove lock file
lf.releaseLock();
diff --git a/src/org/torproject/ernie/db/main/RsyncDataProvider.java b/src/org/torproject/ernie/db/main/RsyncDataProvider.java
index cd4a6f9..78ea1f3 100644
--- a/src/org/torproject/ernie/db/main/RsyncDataProvider.java
+++ b/src/org/torproject/ernie/db/main/RsyncDataProvider.java
@@ -18,29 +18,38 @@ import java.util.logging.Logger;
* then be served via rsync.
*/
public class RsyncDataProvider {
- public RsyncDataProvider(File directoryArchivesOutputDirectory,
- File sanitizedBridgesWriteDirectory,
- File sanitizedAssignmentsDirectory,
- boolean downloadExitList,
- File torperfOutputDirectory, File rsyncDirectory) {
+
+ private Logger logger;
+
+ private long cutOffMillis;
+
+ private File rsyncDirectory;
+
+ public RsyncDataProvider(File rsyncDirectory) {
/* Initialize logger. */
- Logger logger = Logger.getLogger(RsyncDataProvider.class.getName());
+ this.logger = Logger.getLogger(RsyncDataProvider.class.getName());
/* Determine the cut-off time for files in rsync/. */
- long cutOffMillis = System.currentTimeMillis()
+ this.cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
/* Create rsync/ directory if it doesn't exist. */
if (!rsyncDirectory.exists()) {
rsyncDirectory.mkdirs();
}
+ this.rsyncDirectory = rsyncDirectory;
+ }
- /* Make a list of all files in the rsync/ directory to delete those
+ public void copyFiles(File fromDirectory, String toRsyncSubDirectory) {
+
+ File toDirectory = new File(this.rsyncDirectory, toRsyncSubDirectory);
+
+ /* Make a list of all files in the rsync/ subdirectory to delete those
* that we didn't copy in this run. */
Set<String> fileNamesInRsync = new HashSet<String>();
Stack<File> files = new Stack<File>();
- files.add(rsyncDirectory);
+ files.add(toDirectory);
while (!files.isEmpty()) {
File pop = files.pop();
if (pop.isDirectory()) {
@@ -50,132 +59,24 @@ public class RsyncDataProvider {
}
}
logger.info("Found " + fileNamesInRsync.size() + " files in "
- + rsyncDirectory.getAbsolutePath() + " that we're either "
+ + toDirectory.getAbsolutePath() + " that we're either "
+ "overwriting or deleting in this execution.");
- /* Copy relay descriptors from the last 3 days. */
- if (directoryArchivesOutputDirectory != null) {
- files.add(directoryArchivesOutputDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- if (pop.getAbsolutePath().contains("/consensus/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/consensuses/" + fileName));
- } else if (pop.getAbsolutePath().contains("/vote/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/votes/" + fileName));
- } else if (pop.getAbsolutePath().contains(
- "/server-descriptor/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/server-descriptors/" + fileName));
- } else if (pop.getAbsolutePath().contains("/extra-info/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/extra-infos/" + fileName));
- } else {
- continue;
- }
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying relay descriptors, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy sanitized bridge descriptors from the last 3 days. */
- if (sanitizedBridgesWriteDirectory != null) {
- files.add(sanitizedBridgesWriteDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- if (pop.getAbsolutePath().contains("/statuses/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-descriptors/statuses/" + fileName));
- } else if (pop.getAbsolutePath().contains(
- "/server-descriptors/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-descriptors/server-descriptors/" + fileName));
- } else if (pop.getAbsolutePath().contains("/extra-infos/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-descriptors/extra-infos/" + fileName));
- } else {
- continue;
- }
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying sanitized bridge descriptors, there are "
- + "still " + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy sanitized bridge pool assignments from the last 3 days. */
- if (sanitizedAssignmentsDirectory != null) {
- files.add(sanitizedAssignmentsDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-pool-assignments/" + fileName));
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying sanitized bridge pool assignments, there "
- + "are still " + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy exit lists from the last 3 days. */
- if (downloadExitList) {
- files.add(new File("exitlist"));
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- this.copyFile(pop, new File(rsyncDirectory,
- "exit-lists/" + fileName));
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying exit lists, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy Torperf files. */
- if (torperfOutputDirectory != null) {
- files.add(torperfOutputDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.getName().endsWith(".tpf") &&
- pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- this.copyFile(pop, new File(rsyncDirectory,
- "torperf/" + fileName));
- fileNamesInRsync.remove(pop.getName());
- }
+ /* Copy files modified in the last 3 days. */
+ files.add(fromDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= this.cutOffMillis) {
+ String fileName = pop.getName();
+ this.copyFile(pop, new File(toDirectory, fileName));
+ fileNamesInRsync.remove(fileName);
}
}
- logger.info("After copying Torperf files, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
/* Delete all files that we didn't (over-)write in this run. */
- files.add(rsyncDirectory);
+ files.add(toDirectory);
while (!files.isEmpty()) {
File pop = files.pop();
if (pop.isDirectory()) {
@@ -187,7 +88,7 @@ public class RsyncDataProvider {
}
logger.info("After deleting files that we didn't overwrite in this "
+ "run, there are " + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
+ + toDirectory.getAbsolutePath() + ".");
}
private void copyFile(File from, File to) {
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index d53f726..8d7e413 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -25,6 +25,7 @@ import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.impl.DescriptorParseException;
import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
public class ArchiveWriter {
private Logger logger;
@@ -84,6 +85,25 @@ public class ArchiveWriter {
// Write output to disk that only depends on relay descriptors
this.dumpStats();
+
+ /* Copy relay descriptors from the last 3 days to the rsync
+ * directory. */
+ if (config.getProvideFilesViaRsync()) {
+ RsyncDataProvider rsdp = new RsyncDataProvider(
+ new File(config.getRsyncDirectory()));
+ rsdp.copyFiles(
+ new File(outputDirectory, "consensus"),
+ "relay-descriptors/consensuses");
+ rsdp.copyFiles(
+ new File(outputDirectory, "vote"),
+ "relay-descriptors/votes");
+ rsdp.copyFiles(
+ new File(outputDirectory, "server-descriptor"),
+ "relay-descriptors/server-descriptors");
+ rsdp.copyFiles(
+ new File(outputDirectory, "extra-info"),
+ "relay-descriptors/extra-infos");
+ }
}
private boolean store(byte[] typeAnnotation, byte[] data,
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
index a931965..07b444e 100644
--- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -21,6 +21,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
/* Download possibly truncated Torperf .data and .extradata files from
* configured sources, append them to the files we already have, and merge
@@ -54,6 +55,13 @@ public class TorperfDownloader {
this.downloadAndMergeFiles(torperfFilesLine);
}
this.writeLastMergedTimestamps();
+
+ /* Copy Torperf files from the last 3 days to the rsync directory. */
+ if (config.getProvideFilesViaRsync()) {
+ RsyncDataProvider rdp = new RsyncDataProvider(
+ new File(config.getRsyncDirectory()));
+ rdp.copyFiles(torperfOutputDirectory, "torperf");
+ }
}
private File torperfLastMergedFile =
More information about the tor-commits
mailing list