[metrics-db/master] Move all processing logic to sub modules. - tor-commits

27 Oct 2012

commit 0df849df625f749342f0c6ae333928d4a92c37c7
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Fri Oct 26 13:24:39 2012 -0400
Move all processing logic to sub modules.
---
 .../db/bridgedescs/SanitizedBridgesWriter.java     |   33 ++++++-
 .../BridgePoolAssignmentsProcessor.java            |    8 +-
 src/org/torproject/ernie/db/main/Main.java         |  100 ++------------------
 .../ernie/db/relaydescs/ArchiveWriter.java         |   51 +++++++++-
 .../ernie/db/torperf/TorperfDownloader.java        |   15 ++-
 5 files changed, 98 insertions(+), 109 deletions(-)

diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index 85b0883..e46450f 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -27,6 +27,7 @@ import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.ernie.db.main.Configuration;
/**
  * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
@@ -48,6 +49,8 @@ public class SanitizedBridgesWriter {
    */
   private Logger logger;
+  private File bridgeDirectoriesDirectory;
+
   /**
    * Output directory for writing sanitized bridge descriptors.
    */
@@ -70,15 +73,25 @@ public class SanitizedBridgesWriter {
   /**
    * Initializes this class.
    */
-  public SanitizedBridgesWriter(File sanitizedBridgesDirectory,
-      File statsDirectory, boolean replaceIPAddressesWithHashes,
-      long limitBridgeSanitizingInterval) {
-
-    if (sanitizedBridgesDirectory == null || statsDirectory == null) {
+  public SanitizedBridgesWriter(Configuration config,
+      File statsDirectory) {
+
+    File bridgeDirectoriesDirectory =
+        new File(config.getBridgeSnapshotsDirectory());
+    File sanitizedBridgesDirectory =
+        new File(config.getSanitizedBridgesWriteDirectory());
+    boolean replaceIPAddressesWithHashes =
+        config.getReplaceIPAddressesWithHashes();
+    long limitBridgeSanitizingInterval =
+        config.getLimitBridgeDescriptorMappings();
+
+    if (bridgeDirectoriesDirectory == null ||
+        sanitizedBridgesDirectory == null || statsDirectory == null) {
       throw new IllegalArgumentException();
     }
/* Memorize argument values. */
+    this.bridgeDirectoriesDirectory = bridgeDirectoriesDirectory;
     this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
     this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
@@ -155,6 +168,16 @@ public class SanitizedBridgesWriter {
     } else {
       this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59";
     }
+
+    // Prepare bridge descriptor parser
+    BridgeDescriptorParser bdp = new BridgeDescriptorParser(this);
+
+    // Import bridge descriptors
+    new BridgeSnapshotReader(bdp, this.bridgeDirectoriesDirectory,
+        statsDirectory);
+
+    // Finish writing sanitized bridge descriptors to disk
+    this.finishWriting();
   }
private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 470f6ab..76fa539 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -26,11 +26,15 @@ import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.torproject.ernie.db.main.Configuration;
public class BridgePoolAssignmentsProcessor {
-  public BridgePoolAssignmentsProcessor(File assignmentsDirectory,
-        File sanitizedAssignmentsDirectory) {
+  public BridgePoolAssignmentsProcessor(Configuration config) {
+    File assignmentsDirectory =
+        new File(config.getAssignmentsDirectory());
+    File sanitizedAssignmentsDirectory =
+        new File(config.getSanitizedAssignmentsDirectory());
Logger logger =
         Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName());
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
index e008eca..9109a75 100644
--- a/src/org/torproject/ernie/db/main/Main.java
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -3,19 +3,12 @@
 package org.torproject.ernie.db.main;
import java.io.File;
-import java.util.List;
 import java.util.logging.Logger;
-import org.torproject.ernie.db.bridgedescs.BridgeDescriptorParser;
-import org.torproject.ernie.db.bridgedescs.BridgeSnapshotReader;
 import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter;
 import org.torproject.ernie.db.bridgepools.BridgePoolAssignmentsProcessor;
 import org.torproject.ernie.db.exitlists.ExitListDownloader;
-import org.torproject.ernie.db.relaydescs.ArchiveReader;
 import org.torproject.ernie.db.relaydescs.ArchiveWriter;
-import org.torproject.ernie.db.relaydescs.CachedRelayDescriptorReader;
-import org.torproject.ernie.db.relaydescs.RelayDescriptorDownloader;
-import org.torproject.ernie.db.relaydescs.RelayDescriptorParser;
 import org.torproject.ernie.db.torperf.TorperfDownloader;
/**
@@ -45,89 +38,15 @@ public class Main {
     // Define stats directory for temporary files
     File statsDirectory = new File("stats");
-    // Prepare writing relay descriptor archive to disk
-    ArchiveWriter aw = config.getWriteDirectoryArchives() ?
-        new ArchiveWriter(
-        new File(config.getDirectoryArchivesOutputDirectory())) : null;
-
-    // Prepare relay descriptor parser (only if we are writing stats or
-    // directory archives to disk)
-    RelayDescriptorParser rdp = aw != null ?
-        new RelayDescriptorParser(aw) : null;
-
     // Import/download relay descriptors from the various sources
-    if (rdp != null) {
-      RelayDescriptorDownloader rdd = null;
-      if (config.getDownloadRelayDescriptors()) {
-        List<String> dirSources =
-            config.getDownloadFromDirectoryAuthorities();
-        rdd = new RelayDescriptorDownloader(rdp, dirSources,
-            config.getDownloadCurrentConsensus(),
-            config.getDownloadCurrentVotes(),
-            config.getDownloadMissingServerDescriptors(),
-            config.getDownloadMissingExtraInfoDescriptors(),
-            config.getDownloadAllServerDescriptors(),
-            config.getDownloadAllExtraInfoDescriptors(),
-            config.getCompressRelayDescriptorDownloads());
-        rdp.setRelayDescriptorDownloader(rdd);
-      }
-      if (config.getImportCachedRelayDescriptors()) {
-        new CachedRelayDescriptorReader(rdp,
-            config.getCachedRelayDescriptorDirectory(), statsDirectory);
-        if (aw != null) {
-          aw.intermediateStats("importing relay descriptors from local "
-              + "Tor data directories");
-        }
-      }
-      if (config.getImportDirectoryArchives()) {
-        new ArchiveReader(rdp,
-            new File(config.getDirectoryArchivesDirectory()),
-            statsDirectory,
-            config.getKeepDirectoryArchiveImportHistory());
-        if (aw != null) {
-          aw.intermediateStats("importing relay descriptors from local "
-              + "directory");
-        }
-      }
-      if (rdd != null) {
-        rdd.downloadDescriptors();
-        rdd.writeFile();
-        rdd = null;
-        if (aw != null) {
-          aw.intermediateStats("downloading relay descriptors from the "
-              + "directory authorities");
-        }
-      }
-    }
-
-    // Write output to disk that only depends on relay descriptors
-    if (aw != null) {
-      aw.dumpStats();
-      aw = null;
-    }
-
-    // Prepare sanitized bridge descriptor writer
-    SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
-        new SanitizedBridgesWriter(
-        new File(config.getSanitizedBridgesWriteDirectory()),
-        statsDirectory, config.getReplaceIPAddressesWithHashes(),
-        config.getLimitBridgeDescriptorMappings()) : null;
-
-    // Prepare bridge descriptor parser
-    BridgeDescriptorParser bdp = config.getWriteSanitizedBridges()
-        ? new BridgeDescriptorParser(sbw) : null;
-
-    // Import bridge descriptors
-    if (bdp != null && config.getImportBridgeSnapshots()) {
-      new BridgeSnapshotReader(bdp,
-          new File(config.getBridgeSnapshotsDirectory()),
-          statsDirectory);
+    if (config.getWriteDirectoryArchives()) {
+      new ArchiveWriter(config, statsDirectory);
     }
-    // Finish writing sanitized bridge descriptors to disk
-    if (sbw != null) {
-      sbw.finishWriting();
-      sbw = null;
+    // Sanitize bridge descriptors
+    if (config.getImportBridgeSnapshots() &&
+        config.getWriteSanitizedBridges()) {
+      new SanitizedBridgesWriter(config, statsDirectory);
     }
// Download exit list and store it to disk
@@ -137,15 +56,12 @@ public class Main {
// Process bridge pool assignments
     if (config.getProcessBridgePoolAssignments()) {
-      new BridgePoolAssignmentsProcessor(
-          new File(config.getAssignmentsDirectory()),
-          new File(config.getSanitizedAssignmentsDirectory()));
+      new BridgePoolAssignmentsProcessor(config);
     }
// Process Torperf files
     if (config.getProcessTorperfFiles()) {
-      new TorperfDownloader(new File(config.getTorperfOutputDirectory()),
-          config.getTorperfSources(), config.getTorperfFiles());
+      new TorperfDownloader(config);
     }
// Copy recently published files to a local directory that can then
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index c632656..d53f726 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -11,6 +11,7 @@ import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.List;
 import java.util.SortedSet;
 import java.util.Stack;
 import java.util.TimeZone;
@@ -23,6 +24,7 @@ import org.apache.commons.codec.binary.Hex;
 import org.torproject.descriptor.DescriptorParser;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.impl.DescriptorParseException;
+import org.torproject.ernie.db.main.Configuration;
public class ArchiveWriter {
   private Logger logger;
@@ -31,16 +33,57 @@ public class ArchiveWriter {
   private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
       storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
-  public ArchiveWriter(File outputDirectory) {
+  public ArchiveWriter(Configuration config, File statsDirectory) {
-    if (outputDirectory == null) {
-      throw new IllegalArgumentException();
-    }
+    File outputDirectory =
+        new File(config.getDirectoryArchivesOutputDirectory());
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
     this.outputDirectory = outputDirectory;
     this.descriptorParser =
         DescriptorSourceFactory.createDescriptorParser();
+
+    // Prepare relay descriptor parser
+    RelayDescriptorParser rdp = new RelayDescriptorParser(this);
+
+    RelayDescriptorDownloader rdd = null;
+    if (config.getDownloadRelayDescriptors()) {
+      List<String> dirSources =
+          config.getDownloadFromDirectoryAuthorities();
+      rdd = new RelayDescriptorDownloader(rdp, dirSources,
+          config.getDownloadCurrentConsensus(),
+          config.getDownloadCurrentVotes(),
+          config.getDownloadMissingServerDescriptors(),
+          config.getDownloadMissingExtraInfoDescriptors(),
+          config.getDownloadAllServerDescriptors(),
+          config.getDownloadAllExtraInfoDescriptors(),
+          config.getCompressRelayDescriptorDownloads());
+      rdp.setRelayDescriptorDownloader(rdd);
+    }
+    if (config.getImportCachedRelayDescriptors()) {
+      new CachedRelayDescriptorReader(rdp,
+          config.getCachedRelayDescriptorDirectory(), statsDirectory);
+      this.intermediateStats("importing relay descriptors from local "
+          + "Tor data directories");
+    }
+    if (config.getImportDirectoryArchives()) {
+      new ArchiveReader(rdp,
+          new File(config.getDirectoryArchivesDirectory()),
+          statsDirectory,
+          config.getKeepDirectoryArchiveImportHistory());
+      this.intermediateStats("importing relay descriptors from local "
+          + "directory");
+    }
+    if (rdd != null) {
+      rdd.downloadDescriptors();
+      rdd.writeFile();
+      rdd = null;
+      this.intermediateStats("downloading relay descriptors from the "
+          + "directory authorities");
+    }
+
+    // Write output to disk that only depends on relay descriptors
+    this.dumpStats();
   }
private boolean store(byte[] typeAnnotation, byte[] data,
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
index 1ac593a..a931965 100644
--- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -20,6 +20,8 @@ import java.util.TreeMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.torproject.ernie.db.main.Configuration;
+
 /* Download possibly truncated Torperf .data and .extradata files from
  * configured sources, append them to the files we already have, and merge
  * the two files into the .tpf format. */
@@ -31,12 +33,13 @@ public class TorperfDownloader {
   private Logger logger = null;
   private SimpleDateFormat dateFormat;
-  public TorperfDownloader(File torperfOutputDirectory,
-      SortedMap<String, String> torperfSources,
-      List<String> torperfFilesLines) {
-    if (torperfOutputDirectory == null) {
-      throw new IllegalArgumentException();
-    }
+  public TorperfDownloader(Configuration config) {
+
+    File torperfOutputDirectory =
+        new File(config.getTorperfOutputDirectory());
+    SortedMap<String, String> torperfSources = config.getTorperfSources();
+    List<String> torperfFilesLines = config.getTorperfFiles();
+
     this.torperfOutputDirectory = torperfOutputDirectory;
     this.torperfSources = torperfSources;
     this.torperfFilesLines = torperfFilesLines;

    