commit 9766f4c4e9282253090c2e682ee55037c8864383 Author: Karsten Loesing karsten.loesing@gmx.net Date: Fri Oct 26 14:15:16 2012 -0400

Make each module copy its own files to rsync/. --- .../db/bridgedescs/SanitizedBridgesWriter.java | 15 ++ .../BridgePoolAssignmentsProcessor.java | 10 ++ .../ernie/db/exitlists/ExitListDownloader.java | 12 ++- src/org/torproject/ernie/db/main/Main.java | 18 +-- .../ernie/db/main/RsyncDataProvider.java | 161 ++++---------------- .../ernie/db/relaydescs/ArchiveWriter.java | 20 +++ .../ernie/db/torperf/TorperfDownloader.java | 8 + 7 files changed, 96 insertions(+), 148 deletions(-)

diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java index e46450f..15cb8f1 100644 --- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java @@ -28,6 +28,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.RsyncDataProvider;

/** * Sanitizes bridge descriptors, i.e., removes all possibly sensitive @@ -178,6 +179,20 @@ public class SanitizedBridgesWriter {

// Finish writing sanitized bridge descriptors to disk this.finishWriting(); + + // Copy sanitized bridge descriptors from the last 3 days to rsync + // directory. + if (config.getProvideFilesViaRsync()) { + RsyncDataProvider rdp = new RsyncDataProvider( + new File(config.getRsyncDirectory())); + rdp.copyFiles(new File(sanitizedBridgesDirectory, "statuses"), + "bridge-descriptors/statuses"); + rdp.copyFiles( + new File(sanitizedBridgesDirectory, "server-descriptor"), + "bridge-descriptors/server-descriptors"); + rdp.copyFiles(new File(sanitizedBridgesDirectory, "extra-info"), + "bridge-descriptors/extra-infos"); + } }

private String scrubOrAddress(String orAddress, byte[] fingerprintBytes, diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java index 76fa539..63b7c4d 100644 --- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java @@ -27,6 +27,7 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.RsyncDataProvider;

public class BridgePoolAssignmentsProcessor {

@@ -172,6 +173,15 @@ public class BridgePoolAssignmentsProcessor { } }

+ // Copy sanitized bridge pool assignments from the last 3 days to the + // rsync directory. + if (config.getProvideFilesViaRsync()) { + RsyncDataProvider rdp = new RsyncDataProvider( + new File(config.getRsyncDirectory())); + rdp.copyFiles(sanitizedAssignmentsDirectory, + "bridge-pool-assignments"); + } + logger.info("Finished processing bridge pool assignment file(s)."); } } diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java index 64f6a3b..9fb714b 100644 --- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java +++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java @@ -18,8 +18,11 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger;

+import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.RsyncDataProvider; + public class ExitListDownloader { - public ExitListDownloader() { + public ExitListDownloader(Configuration config) { Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); try { logger.fine("Downloading exit list..."); @@ -95,6 +98,13 @@ public class ExitListDownloader { dumpStats.append("\n" + f.getName()); } logger.info(dumpStats.toString()); + + /* Copy exit lists from the last 3 days to the rsync directory. */ + if (config.getProvideFilesViaRsync()) { + RsyncDataProvider rdp = new RsyncDataProvider( + new File(config.getRsyncDirectory())); + rdp.copyFiles(new File("exitlist"), "exit-lists"); + } } }

diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java index 9109a75..093f002 100644 --- a/src/org/torproject/ernie/db/main/Main.java +++ b/src/org/torproject/ernie/db/main/Main.java @@ -51,7 +51,7 @@ public class Main {

// Download exit list and store it to disk if (config.getDownloadExitList()) { - new ExitListDownloader(); + new ExitListDownloader(config); }

// Process bridge pool assignments @@ -64,22 +64,6 @@ public class Main { new TorperfDownloader(config); }

- // Copy recently published files to a local directory that can then - // be served via rsync. - if (config.getProvideFilesViaRsync()) { - new RsyncDataProvider( - !config.getWriteDirectoryArchives() ? null : - new File(config.getDirectoryArchivesOutputDirectory()), - !config.getWriteSanitizedBridges() ? null : - new File(config.getSanitizedBridgesWriteDirectory()), - !config.getProcessBridgePoolAssignments() ? null : - new File(config.getSanitizedAssignmentsDirectory()), - config.getDownloadExitList(), - !config.getProcessTorperfFiles() ? null : - new File(config.getTorperfOutputDirectory()), - new File(config.getRsyncDirectory())); - } - // Remove lock file lf.releaseLock();

diff --git a/src/org/torproject/ernie/db/main/RsyncDataProvider.java b/src/org/torproject/ernie/db/main/RsyncDataProvider.java index cd4a6f9..78ea1f3 100644 --- a/src/org/torproject/ernie/db/main/RsyncDataProvider.java +++ b/src/org/torproject/ernie/db/main/RsyncDataProvider.java @@ -18,29 +18,38 @@ import java.util.logging.Logger; * then be served via rsync. */ public class RsyncDataProvider { - public RsyncDataProvider(File directoryArchivesOutputDirectory, - File sanitizedBridgesWriteDirectory, - File sanitizedAssignmentsDirectory, - boolean downloadExitList, - File torperfOutputDirectory, File rsyncDirectory) { + + private Logger logger; + + private long cutOffMillis; + + private File rsyncDirectory; + + public RsyncDataProvider(File rsyncDirectory) {

/* Initialize logger. */ - Logger logger = Logger.getLogger(RsyncDataProvider.class.getName()); + this.logger = Logger.getLogger(RsyncDataProvider.class.getName());

/* Determine the cut-off time for files in rsync/. */ - long cutOffMillis = System.currentTimeMillis() + this.cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L;

/* Create rsync/ directory if it doesn't exist. */ if (!rsyncDirectory.exists()) { rsyncDirectory.mkdirs(); } + this.rsyncDirectory = rsyncDirectory; + }

- /* Make a list of all files in the rsync/ directory to delete those + public void copyFiles(File fromDirectory, String toRsyncSubDirectory) { + + File toDirectory = new File(this.rsyncDirectory, toRsyncSubDirectory); + + /* Make a list of all files in the rsync/ subdirectory to delete those * that we didn't copy in this run. */ Set<String> fileNamesInRsync = new HashSet<String>(); Stack<File> files = new Stack<File>(); - files.add(rsyncDirectory); + files.add(toDirectory); while (!files.isEmpty()) { File pop = files.pop(); if (pop.isDirectory()) { @@ -50,132 +59,24 @@ public class RsyncDataProvider { } } logger.info("Found " + fileNamesInRsync.size() + " files in " - + rsyncDirectory.getAbsolutePath() + " that we're either " + + toDirectory.getAbsolutePath() + " that we're either " + "overwriting or deleting in this execution.");

- /* Copy relay descriptors from the last 3 days. */ - if (directoryArchivesOutputDirectory != null) { - files.add(directoryArchivesOutputDirectory); - while (!files.isEmpty()) { - File pop = files.pop(); - if (pop.isDirectory()) { - files.addAll(Arrays.asList(pop.listFiles())); - } else if (pop.lastModified() >= cutOffMillis) { - String fileName = pop.getName(); - if (pop.getAbsolutePath().contains("/consensus/")) { - this.copyFile(pop, new File(rsyncDirectory, - "relay-descriptors/consensuses/" + fileName)); - } else if (pop.getAbsolutePath().contains("/vote/")) { - this.copyFile(pop, new File(rsyncDirectory, - "relay-descriptors/votes/" + fileName)); - } else if (pop.getAbsolutePath().contains( - "/server-descriptor/")) { - this.copyFile(pop, new File(rsyncDirectory, - "relay-descriptors/server-descriptors/" + fileName)); - } else if (pop.getAbsolutePath().contains("/extra-info/")) { - this.copyFile(pop, new File(rsyncDirectory, - "relay-descriptors/extra-infos/" + fileName)); - } else { - continue; - } - fileNamesInRsync.remove(pop.getName()); - } - } - } - logger.info("After copying relay descriptors, there are still " - + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); - - /* Copy sanitized bridge descriptors from the last 3 days. */ - if (sanitizedBridgesWriteDirectory != null) { - files.add(sanitizedBridgesWriteDirectory); - while (!files.isEmpty()) { - File pop = files.pop(); - if (pop.isDirectory()) { - files.addAll(Arrays.asList(pop.listFiles())); - } else if (pop.lastModified() >= cutOffMillis) { - String fileName = pop.getName(); - if (pop.getAbsolutePath().contains("/statuses/")) { - this.copyFile(pop, new File(rsyncDirectory, - "bridge-descriptors/statuses/" + fileName)); - } else if (pop.getAbsolutePath().contains( - "/server-descriptors/")) { - this.copyFile(pop, new File(rsyncDirectory, - "bridge-descriptors/server-descriptors/" + fileName)); - } else if (pop.getAbsolutePath().contains("/extra-infos/")) { - this.copyFile(pop, new File(rsyncDirectory, - "bridge-descriptors/extra-infos/" + fileName)); - } else { - continue; - } - fileNamesInRsync.remove(pop.getName()); - } - } - } - logger.info("After copying sanitized bridge descriptors, there are " - + "still " + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); - - /* Copy sanitized bridge pool assignments from the last 3 days. */ - if (sanitizedAssignmentsDirectory != null) { - files.add(sanitizedAssignmentsDirectory); - while (!files.isEmpty()) { - File pop = files.pop(); - if (pop.isDirectory()) { - files.addAll(Arrays.asList(pop.listFiles())); - } else if (pop.lastModified() >= cutOffMillis) { - String fileName = pop.getName(); - this.copyFile(pop, new File(rsyncDirectory, - "bridge-pool-assignments/" + fileName)); - fileNamesInRsync.remove(pop.getName()); - } - } - } - logger.info("After copying sanitized bridge pool assignments, there " - + "are still " + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); - - /* Copy exit lists from the last 3 days. */ - if (downloadExitList) { - files.add(new File("exitlist")); - while (!files.isEmpty()) { - File pop = files.pop(); - if (pop.isDirectory()) { - files.addAll(Arrays.asList(pop.listFiles())); - } else if (pop.lastModified() >= cutOffMillis) { - String fileName = pop.getName(); - this.copyFile(pop, new File(rsyncDirectory, - "exit-lists/" + fileName)); - fileNamesInRsync.remove(pop.getName()); - } - } - } - logger.info("After copying exit lists, there are still " - + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); - - /* Copy Torperf files. */ - if (torperfOutputDirectory != null) { - files.add(torperfOutputDirectory); - while (!files.isEmpty()) { - File pop = files.pop(); - if (pop.isDirectory()) { - files.addAll(Arrays.asList(pop.listFiles())); - } else if (pop.getName().endsWith(".tpf") && - pop.lastModified() >= cutOffMillis) { - String fileName = pop.getName(); - this.copyFile(pop, new File(rsyncDirectory, - "torperf/" + fileName)); - fileNamesInRsync.remove(pop.getName()); - } + /* Copy files modified in the last 3 days. */ + files.add(fromDirectory); + while (!files.isEmpty()) { + File pop = files.pop(); + if (pop.isDirectory()) { + files.addAll(Arrays.asList(pop.listFiles())); + } else if (pop.lastModified() >= this.cutOffMillis) { + String fileName = pop.getName(); + this.copyFile(pop, new File(toDirectory, fileName)); + fileNamesInRsync.remove(fileName); } } - logger.info("After copying Torperf files, there are still " - + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + ".");

/* Delete all files that we didn't (over-)write in this run. */ - files.add(rsyncDirectory); + files.add(toDirectory); while (!files.isEmpty()) { File pop = files.pop(); if (pop.isDirectory()) { @@ -187,7 +88,7 @@ public class RsyncDataProvider { } logger.info("After deleting files that we didn't overwrite in this " + "run, there are " + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); + + toDirectory.getAbsolutePath() + "."); }

private void copyFile(File from, File to) { diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index d53f726..8d7e413 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -25,6 +25,7 @@ import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.impl.DescriptorParseException; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.RsyncDataProvider;

public class ArchiveWriter { private Logger logger; @@ -84,6 +85,25 @@ public class ArchiveWriter {

// Write output to disk that only depends on relay descriptors this.dumpStats(); + + /* Copy relay descriptors from the last 3 days to the rsync + * directory. */ + if (config.getProvideFilesViaRsync()) { + RsyncDataProvider rsdp = new RsyncDataProvider( + new File(config.getRsyncDirectory())); + rsdp.copyFiles( + new File(outputDirectory, "consensus"), + "relay-descriptors/consensuses"); + rsdp.copyFiles( + new File(outputDirectory, "vote"), + "relay-descriptors/votes"); + rsdp.copyFiles( + new File(outputDirectory, "server-descriptor"), + "relay-descriptors/server-descriptors"); + rsdp.copyFiles( + new File(outputDirectory, "extra-info"), + "relay-descriptors/extra-infos"); + } }

private boolean store(byte[] typeAnnotation, byte[] data, diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java index a931965..07b444e 100644 --- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java +++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java @@ -21,6 +21,7 @@ import java.util.logging.Level; import java.util.logging.Logger;

import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.RsyncDataProvider;

/* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge @@ -54,6 +55,13 @@ public class TorperfDownloader { this.downloadAndMergeFiles(torperfFilesLine); } this.writeLastMergedTimestamps(); + + /* Copy Torperf files from the last 3 days to the rsync directory. */ + if (config.getProvideFilesViaRsync()) { + RsyncDataProvider rdp = new RsyncDataProvider( + new File(config.getRsyncDirectory())); + rdp.copyFiles(torperfOutputDirectory, "torperf"); + } }

private File torperfLastMergedFile =