commit 0df849df625f749342f0c6ae333928d4a92c37c7 Author: Karsten Loesing karsten.loesing@gmx.net Date: Fri Oct 26 13:24:39 2012 -0400
Move all processing logic to sub modules. --- .../db/bridgedescs/SanitizedBridgesWriter.java | 33 ++++++- .../BridgePoolAssignmentsProcessor.java | 8 +- src/org/torproject/ernie/db/main/Main.java | 100 ++------------------ .../ernie/db/relaydescs/ArchiveWriter.java | 51 +++++++++- .../ernie/db/torperf/TorperfDownloader.java | 15 ++- 5 files changed, 98 insertions(+), 109 deletions(-)
diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java index 85b0883..e46450f 100644 --- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java @@ -27,6 +27,7 @@ import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; +import org.torproject.ernie.db.main.Configuration;
/** * Sanitizes bridge descriptors, i.e., removes all possibly sensitive @@ -48,6 +49,8 @@ public class SanitizedBridgesWriter { */ private Logger logger;
+ private File bridgeDirectoriesDirectory; + /** * Output directory for writing sanitized bridge descriptors. */ @@ -70,15 +73,25 @@ public class SanitizedBridgesWriter { /** * Initializes this class. */ - public SanitizedBridgesWriter(File sanitizedBridgesDirectory, - File statsDirectory, boolean replaceIPAddressesWithHashes, - long limitBridgeSanitizingInterval) { - - if (sanitizedBridgesDirectory == null || statsDirectory == null) { + public SanitizedBridgesWriter(Configuration config, + File statsDirectory) { + + File bridgeDirectoriesDirectory = + new File(config.getBridgeSnapshotsDirectory()); + File sanitizedBridgesDirectory = + new File(config.getSanitizedBridgesWriteDirectory()); + boolean replaceIPAddressesWithHashes = + config.getReplaceIPAddressesWithHashes(); + long limitBridgeSanitizingInterval = + config.getLimitBridgeDescriptorMappings(); + + if (bridgeDirectoriesDirectory == null || + sanitizedBridgesDirectory == null || statsDirectory == null) { throw new IllegalArgumentException(); }
/* Memorize argument values. */ + this.bridgeDirectoriesDirectory = bridgeDirectoriesDirectory; this.sanitizedBridgesDirectory = sanitizedBridgesDirectory; this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
@@ -155,6 +168,16 @@ public class SanitizedBridgesWriter { } else { this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59"; } + + // Prepare bridge descriptor parser + BridgeDescriptorParser bdp = new BridgeDescriptorParser(this); + + // Import bridge descriptors + new BridgeSnapshotReader(bdp, this.bridgeDirectoriesDirectory, + statsDirectory); + + // Finish writing sanitized bridge descriptors to disk + this.finishWriting(); }
private String scrubOrAddress(String orAddress, byte[] fingerprintBytes, diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java index 470f6ab..76fa539 100644 --- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java @@ -26,11 +26,15 @@ import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.torproject.ernie.db.main.Configuration;
public class BridgePoolAssignmentsProcessor {
- public BridgePoolAssignmentsProcessor(File assignmentsDirectory, - File sanitizedAssignmentsDirectory) { + public BridgePoolAssignmentsProcessor(Configuration config) { + File assignmentsDirectory = + new File(config.getAssignmentsDirectory()); + File sanitizedAssignmentsDirectory = + new File(config.getSanitizedAssignmentsDirectory());
Logger logger = Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName()); diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java index e008eca..9109a75 100644 --- a/src/org/torproject/ernie/db/main/Main.java +++ b/src/org/torproject/ernie/db/main/Main.java @@ -3,19 +3,12 @@ package org.torproject.ernie.db.main;
import java.io.File; -import java.util.List; import java.util.logging.Logger;
-import org.torproject.ernie.db.bridgedescs.BridgeDescriptorParser; -import org.torproject.ernie.db.bridgedescs.BridgeSnapshotReader; import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter; import org.torproject.ernie.db.bridgepools.BridgePoolAssignmentsProcessor; import org.torproject.ernie.db.exitlists.ExitListDownloader; -import org.torproject.ernie.db.relaydescs.ArchiveReader; import org.torproject.ernie.db.relaydescs.ArchiveWriter; -import org.torproject.ernie.db.relaydescs.CachedRelayDescriptorReader; -import org.torproject.ernie.db.relaydescs.RelayDescriptorDownloader; -import org.torproject.ernie.db.relaydescs.RelayDescriptorParser; import org.torproject.ernie.db.torperf.TorperfDownloader;
/** @@ -45,89 +38,15 @@ public class Main { // Define stats directory for temporary files File statsDirectory = new File("stats");
- // Prepare writing relay descriptor archive to disk - ArchiveWriter aw = config.getWriteDirectoryArchives() ? - new ArchiveWriter( - new File(config.getDirectoryArchivesOutputDirectory())) : null; - - // Prepare relay descriptor parser (only if we are writing stats or - // directory archives to disk) - RelayDescriptorParser rdp = aw != null ? - new RelayDescriptorParser(aw) : null; - // Import/download relay descriptors from the various sources - if (rdp != null) { - RelayDescriptorDownloader rdd = null; - if (config.getDownloadRelayDescriptors()) { - List<String> dirSources = - config.getDownloadFromDirectoryAuthorities(); - rdd = new RelayDescriptorDownloader(rdp, dirSources, - config.getDownloadCurrentConsensus(), - config.getDownloadCurrentVotes(), - config.getDownloadMissingServerDescriptors(), - config.getDownloadMissingExtraInfoDescriptors(), - config.getDownloadAllServerDescriptors(), - config.getDownloadAllExtraInfoDescriptors(), - config.getCompressRelayDescriptorDownloads()); - rdp.setRelayDescriptorDownloader(rdd); - } - if (config.getImportCachedRelayDescriptors()) { - new CachedRelayDescriptorReader(rdp, - config.getCachedRelayDescriptorDirectory(), statsDirectory); - if (aw != null) { - aw.intermediateStats("importing relay descriptors from local " - + "Tor data directories"); - } - } - if (config.getImportDirectoryArchives()) { - new ArchiveReader(rdp, - new File(config.getDirectoryArchivesDirectory()), - statsDirectory, - config.getKeepDirectoryArchiveImportHistory()); - if (aw != null) { - aw.intermediateStats("importing relay descriptors from local " - + "directory"); - } - } - if (rdd != null) { - rdd.downloadDescriptors(); - rdd.writeFile(); - rdd = null; - if (aw != null) { - aw.intermediateStats("downloading relay descriptors from the " - + "directory authorities"); - } - } - } - - // Write output to disk that only depends on relay descriptors - if (aw != null) { - aw.dumpStats(); - aw = null; - } - - // Prepare sanitized bridge descriptor writer - SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ? - new SanitizedBridgesWriter( - new File(config.getSanitizedBridgesWriteDirectory()), - statsDirectory, config.getReplaceIPAddressesWithHashes(), - config.getLimitBridgeDescriptorMappings()) : null; - - // Prepare bridge descriptor parser - BridgeDescriptorParser bdp = config.getWriteSanitizedBridges() - ? new BridgeDescriptorParser(sbw) : null; - - // Import bridge descriptors - if (bdp != null && config.getImportBridgeSnapshots()) { - new BridgeSnapshotReader(bdp, - new File(config.getBridgeSnapshotsDirectory()), - statsDirectory); + if (config.getWriteDirectoryArchives()) { + new ArchiveWriter(config, statsDirectory); }
- // Finish writing sanitized bridge descriptors to disk - if (sbw != null) { - sbw.finishWriting(); - sbw = null; + // Sanitize bridge descriptors + if (config.getImportBridgeSnapshots() && + config.getWriteSanitizedBridges()) { + new SanitizedBridgesWriter(config, statsDirectory); }
// Download exit list and store it to disk @@ -137,15 +56,12 @@ public class Main {
// Process bridge pool assignments if (config.getProcessBridgePoolAssignments()) { - new BridgePoolAssignmentsProcessor( - new File(config.getAssignmentsDirectory()), - new File(config.getSanitizedAssignmentsDirectory())); + new BridgePoolAssignmentsProcessor(config); }
// Process Torperf files if (config.getProcessTorperfFiles()) { - new TorperfDownloader(new File(config.getTorperfOutputDirectory()), - config.getTorperfSources(), config.getTorperfFiles()); + new TorperfDownloader(config); }
// Copy recently published files to a local directory that can then diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index c632656..d53f726 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.List; import java.util.SortedSet; import java.util.Stack; import java.util.TimeZone; @@ -23,6 +24,7 @@ import org.apache.commons.codec.binary.Hex; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.impl.DescriptorParseException; +import org.torproject.ernie.db.main.Configuration;
public class ArchiveWriter { private Logger logger; @@ -31,16 +33,57 @@ public class ArchiveWriter { private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0, storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
- public ArchiveWriter(File outputDirectory) { + public ArchiveWriter(Configuration config, File statsDirectory) {
- if (outputDirectory == null) { - throw new IllegalArgumentException(); - } + File outputDirectory = + new File(config.getDirectoryArchivesOutputDirectory());
this.logger = Logger.getLogger(ArchiveWriter.class.getName()); this.outputDirectory = outputDirectory; this.descriptorParser = DescriptorSourceFactory.createDescriptorParser(); + + // Prepare relay descriptor parser + RelayDescriptorParser rdp = new RelayDescriptorParser(this); + + RelayDescriptorDownloader rdd = null; + if (config.getDownloadRelayDescriptors()) { + List<String> dirSources = + config.getDownloadFromDirectoryAuthorities(); + rdd = new RelayDescriptorDownloader(rdp, dirSources, + config.getDownloadCurrentConsensus(), + config.getDownloadCurrentVotes(), + config.getDownloadMissingServerDescriptors(), + config.getDownloadMissingExtraInfoDescriptors(), + config.getDownloadAllServerDescriptors(), + config.getDownloadAllExtraInfoDescriptors(), + config.getCompressRelayDescriptorDownloads()); + rdp.setRelayDescriptorDownloader(rdd); + } + if (config.getImportCachedRelayDescriptors()) { + new CachedRelayDescriptorReader(rdp, + config.getCachedRelayDescriptorDirectory(), statsDirectory); + this.intermediateStats("importing relay descriptors from local " + + "Tor data directories"); + } + if (config.getImportDirectoryArchives()) { + new ArchiveReader(rdp, + new File(config.getDirectoryArchivesDirectory()), + statsDirectory, + config.getKeepDirectoryArchiveImportHistory()); + this.intermediateStats("importing relay descriptors from local " + + "directory"); + } + if (rdd != null) { + rdd.downloadDescriptors(); + rdd.writeFile(); + rdd = null; + this.intermediateStats("downloading relay descriptors from the " + + "directory authorities"); + } + + // Write output to disk that only depends on relay descriptors + this.dumpStats(); }
private boolean store(byte[] typeAnnotation, byte[] data, diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java index 1ac593a..a931965 100644 --- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java +++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java @@ -20,6 +20,8 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger;
+import org.torproject.ernie.db.main.Configuration; + /* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ @@ -31,12 +33,13 @@ public class TorperfDownloader { private Logger logger = null; private SimpleDateFormat dateFormat;
- public TorperfDownloader(File torperfOutputDirectory, - SortedMap<String, String> torperfSources, - List<String> torperfFilesLines) { - if (torperfOutputDirectory == null) { - throw new IllegalArgumentException(); - } + public TorperfDownloader(Configuration config) { + + File torperfOutputDirectory = + new File(config.getTorperfOutputDirectory()); + SortedMap<String, String> torperfSources = config.getTorperfSources(); + List<String> torperfFilesLines = config.getTorperfFiles(); + this.torperfOutputDirectory = torperfOutputDirectory; this.torperfSources = torperfSources; this.torperfFilesLines = torperfFilesLines;