commit 7db75dbedf2fc77a3855e4fdff051bcbdcccac5e Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Fri Oct 26 15:13:21 2012 -0400 Run five data-processing modules in parallel. --- .../db/bridgedescs/SanitizedBridgesWriter.java | 19 ++++++++++++------- .../BridgePoolAssignmentsProcessor.java | 8 +++++++- .../ernie/db/exitlists/ExitListDownloader.java | 6 +++++- src/org/torproject/ernie/db/main/Main.java | 14 +++++--------- .../ernie/db/relaydescs/ArchiveWriter.java | 12 ++++++++++-- .../ernie/db/torperf/TorperfDownloader.java | 10 ++++++++-- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java index 10d2baa..cb08df1 100644 --- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java @@ -43,7 +43,16 @@ import org.torproject.ernie.db.main.RsyncDataProvider; * by the bridge to advertise their capabilities), and extra-info * descriptors (published by the bridge, mainly for statistical analysis). */ -public class SanitizedBridgesWriter { +public class SanitizedBridgesWriter extends Thread { + + private Configuration config; + + /** + * Initializes this class. + */ + public SanitizedBridgesWriter(Configuration config) { + this.config = config; + } /** * Logger for this class. @@ -71,12 +80,7 @@ public class SanitizedBridgesWriter { private SecureRandom secureRandom; - /** - * Initializes this class. - */ - public SanitizedBridgesWriter(Configuration config, - File statsDirectory) { - + public void run() { File bridgeDirectoriesDirectory = new File(config.getBridgeSnapshotsDirectory()); File sanitizedBridgesDirectory = @@ -85,6 +89,7 @@ public class SanitizedBridgesWriter { config.getReplaceIPAddressesWithHashes(); long limitBridgeSanitizingInterval = config.getLimitBridgeDescriptorMappings(); + File statsDirectory = new File("stats"); if (bridgeDirectoriesDirectory == null || sanitizedBridgesDirectory == null || statsDirectory == null) { diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java index 3893495..d822d11 100644 --- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java @@ -29,9 +29,15 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.torproject.ernie.db.main.Configuration; import org.torproject.ernie.db.main.RsyncDataProvider; -public class BridgePoolAssignmentsProcessor { +public class BridgePoolAssignmentsProcessor extends Thread { + + private Configuration config; public BridgePoolAssignmentsProcessor(Configuration config) { + this.config = config; + } + + public void run() { File assignmentsDirectory = new File(config.getAssignmentsDirectory()); File sanitizedAssignmentsDirectory = diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java index f6f2865..bf67bb7 100644 --- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java +++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java @@ -21,8 +21,12 @@ import java.util.logging.Logger; import org.torproject.ernie.db.main.Configuration; import org.torproject.ernie.db.main.RsyncDataProvider; -public class ExitListDownloader { +public class ExitListDownloader extends Thread { + public ExitListDownloader(Configuration config) { + } + + public void run() { Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); try { logger.fine("Downloading exit list..."); diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java index 093f002..88f2372 100644 --- a/src/org/torproject/ernie/db/main/Main.java +++ b/src/org/torproject/ernie/db/main/Main.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.ernie.db.main; -import java.io.File; import java.util.logging.Logger; import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter; @@ -35,33 +34,30 @@ public class Main { System.exit(1); } - // Define stats directory for temporary files - File statsDirectory = new File("stats"); - // Import/download relay descriptors from the various sources if (config.getWriteDirectoryArchives()) { - new ArchiveWriter(config, statsDirectory); + new ArchiveWriter(config).start(); } // Sanitize bridge descriptors if (config.getImportBridgeSnapshots() && config.getWriteSanitizedBridges()) { - new SanitizedBridgesWriter(config, statsDirectory); + new SanitizedBridgesWriter(config).start(); } // Download exit list and store it to disk if (config.getDownloadExitList()) { - new ExitListDownloader(config); + new ExitListDownloader(config).start(); } // Process bridge pool assignments if (config.getProcessBridgePoolAssignments()) { - new BridgePoolAssignmentsProcessor(config); + new BridgePoolAssignmentsProcessor(config).start(); } // Process Torperf files if (config.getProcessTorperfFiles()) { - new TorperfDownloader(config); + new TorperfDownloader(config).start(); } // Remove lock file diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index a9cb604..5c3d2e9 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -27,17 +27,25 @@ import org.torproject.descriptor.impl.DescriptorParseException; import org.torproject.ernie.db.main.Configuration; import org.torproject.ernie.db.main.RsyncDataProvider; -public class ArchiveWriter { +public class ArchiveWriter extends Thread { + + private Configuration config; + + public ArchiveWriter(Configuration config) { + this.config = config; + } + private Logger logger; private File outputDirectory; private DescriptorParser descriptorParser; private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0, storedServerDescriptors = 0, storedExtraInfoDescriptors = 0; - public ArchiveWriter(Configuration config, File statsDirectory) { + public void run() { File outputDirectory = new File(config.getDirectoryArchivesOutputDirectory()); + File statsDirectory = new File("stats"); this.logger = Logger.getLogger(ArchiveWriter.class.getName()); this.outputDirectory = outputDirectory; diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java index 5f09038..ef67ae7 100644 --- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java +++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java @@ -26,7 +26,13 @@ import org.torproject.ernie.db.main.RsyncDataProvider; /* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ -public class TorperfDownloader { +public class TorperfDownloader extends Thread { + + private Configuration config; + + public TorperfDownloader(Configuration config) { + this.config = config; + } private File torperfOutputDirectory = null; private SortedMap<String, String> torperfSources = null; @@ -34,7 +40,7 @@ public class TorperfDownloader { private Logger logger = null; private SimpleDateFormat dateFormat; - public TorperfDownloader(Configuration config) { + public void run() { File torperfOutputDirectory = new File(config.getTorperfOutputDirectory());