[tor-commits] [metrics-db/master] Move default input directories to in/.

[karsten at torproject.org]

commit efa2b13d3f30a4ecd471e68ef442795d2e5480a6
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Sat Nov 3 10:47:29 2012 -0400

    Move default input directories to in/.
---
 config.template                                    |    8 ++++----
 .../torproject/ernie/db/main/Configuration.java    |   10 ++++++----
 .../db/relaydescs/CachedRelayDescriptorReader.java |   17 ++++++++++++++++-
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/config.template b/config.template
index a31dc9e..29da891 100644
--- a/config.template
+++ b/config.template
@@ -5,13 +5,13 @@
 #
 ## Relative path to Tor data directory to read cached-* files from (can be
 ## specified multiple times)
-#CachedRelayDescriptorsDirectory cacheddesc/
+#CachedRelayDescriptorsDirectory in/relay-descriptors/cacheddesc/
 #
 ## Import directory archives from disk, if available
 #ImportDirectoryArchives 0
 #
 ## Relative path to directory to import directory archives from
-#DirectoryArchivesDirectory archives/
+#DirectoryArchivesDirectory in/relay-descriptors/archives/
 #
 ## Keep a history of imported directory archive files to know which files
 ## have been imported before. This history can be useful when importing
@@ -68,7 +68,7 @@
 #ImportBridgeSnapshots 0
 #
 ## Relative path to directory to import bridge descriptor snapshots from
-#BridgeSnapshotsDirectory bridge-directories/
+#BridgeSnapshotsDirectory in/bridge-descriptors/
 #
 ## Write sanitized bridges to disk
 #WriteSanitizedBridges 0
@@ -93,7 +93,7 @@
 #ProcessBridgePoolAssignments 0
 #
 ## Relative path to directory to read bridge pool assignment files from
-#AssignmentsDirectory assignments/
+#AssignmentsDirectory in/bridge-assignments/
 #
 ## Relative path to directory to write sanitized bridge pool assignment
 ## files to
diff --git a/src/org/torproject/ernie/db/main/Configuration.java b/src/org/torproject/ernie/db/main/Configuration.java
index bfb7307..0f30d73 100644
--- a/src/org/torproject/ernie/db/main/Configuration.java
+++ b/src/org/torproject/ernie/db/main/Configuration.java
@@ -26,16 +26,18 @@ public class Configuration {
   private String directoryArchivesOutputDirectory = "directory-archive/";
   private boolean importCachedRelayDescriptors = false;
   private List<String> cachedRelayDescriptorsDirectory =
-      new ArrayList<String>(Arrays.asList("cacheddesc/".split(",")));
+      new ArrayList<String>(Arrays.asList(
+      "in/relay-descriptors/cacheddesc/".split(",")));
   private boolean importDirectoryArchives = false;
-  private String directoryArchivesDirectory = "archives/";
+  private String directoryArchivesDirectory =
+      "in/relay-descriptors/archives/";
   private boolean keepDirectoryArchiveImportHistory = false;
   private boolean writeSanitizedBridges = false;
   private boolean replaceIPAddressesWithHashes = false;
   private long limitBridgeDescriptorMappings = -1L;
   private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
   private boolean importBridgeSnapshots = false;
-  private String bridgeSnapshotsDirectory = "bridge-directories/";
+  private String bridgeSnapshotsDirectory = "in/bridge-descriptors/";
   private boolean downloadRelayDescriptors = false;
   private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
       "86.59.21.38,76.73.17.194:9030,213.115.239.118:443,"
@@ -60,7 +62,7 @@ public class Configuration {
   private boolean compressRelayDescriptorDownloads;
   private boolean downloadExitList = false;
   private boolean processBridgePoolAssignments = false;
-  private String assignmentsDirectory = "assignments/";
+  private String assignmentsDirectory = "in/bridge-assignments/";
   private String sanitizedAssignmentsDirectory = "sanitized-assignments/";
   private boolean processTorperfFiles = false;
   private String torperfOutputDirectory = "torperf/";
diff --git a/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java b/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java
index 194e0a2..d0a2600 100644
--- a/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java
+++ b/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java
@@ -14,10 +14,14 @@ import java.io.IOException;
 import java.io.StringReader;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.Stack;
 import java.util.TimeZone;
+import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -73,7 +77,18 @@ public class CachedRelayDescriptorReader {
       }
       logger.fine("Reading " + cachedDescDir.getAbsolutePath()
           + " directory.");
-      for (File f : cachedDescDir.listFiles()) {
+      SortedSet<File> cachedDescFiles = new TreeSet<File>();
+      Stack<File> files = new Stack<File>();
+      files.add(cachedDescDir);
+      while (!files.isEmpty()) {
+        File file = files.pop();
+        if (file.isDirectory()) {
+          files.addAll(Arrays.asList(file.listFiles()));
+        } else {
+          cachedDescFiles.add(file);
+        }
+      }
+      for (File f : cachedDescFiles) {
         try {
           // descriptors may contain non-ASCII chars; read as bytes to
           // determine digests