commit efa2b13d3f30a4ecd471e68ef442795d2e5480a6 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sat Nov 3 10:47:29 2012 -0400 Move default input directories to in/. --- config.template | 8 ++++---- .../torproject/ernie/db/main/Configuration.java | 10 ++++++---- .../db/relaydescs/CachedRelayDescriptorReader.java | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/config.template b/config.template index a31dc9e..29da891 100644 --- a/config.template +++ b/config.template @@ -5,13 +5,13 @@ # ## Relative path to Tor data directory to read cached-* files from (can be ## specified multiple times) -#CachedRelayDescriptorsDirectory cacheddesc/ +#CachedRelayDescriptorsDirectory in/relay-descriptors/cacheddesc/ # ## Import directory archives from disk, if available #ImportDirectoryArchives 0 # ## Relative path to directory to import directory archives from -#DirectoryArchivesDirectory archives/ +#DirectoryArchivesDirectory in/relay-descriptors/archives/ # ## Keep a history of imported directory archive files to know which files ## have been imported before. This history can be useful when importing @@ -68,7 +68,7 @@ #ImportBridgeSnapshots 0 # ## Relative path to directory to import bridge descriptor snapshots from -#BridgeSnapshotsDirectory bridge-directories/ +#BridgeSnapshotsDirectory in/bridge-descriptors/ # ## Write sanitized bridges to disk #WriteSanitizedBridges 0 @@ -93,7 +93,7 @@ #ProcessBridgePoolAssignments 0 # ## Relative path to directory to read bridge pool assignment files from -#AssignmentsDirectory assignments/ +#AssignmentsDirectory in/bridge-assignments/ # ## Relative path to directory to write sanitized bridge pool assignment ## files to diff --git a/src/org/torproject/ernie/db/main/Configuration.java b/src/org/torproject/ernie/db/main/Configuration.java index bfb7307..0f30d73 100644 --- a/src/org/torproject/ernie/db/main/Configuration.java +++ b/src/org/torproject/ernie/db/main/Configuration.java @@ -26,16 +26,18 @@ public class Configuration { private String directoryArchivesOutputDirectory = "directory-archive/"; private boolean importCachedRelayDescriptors = false; private List<String> cachedRelayDescriptorsDirectory = - new ArrayList<String>(Arrays.asList("cacheddesc/".split(","))); + new ArrayList<String>(Arrays.asList( + "in/relay-descriptors/cacheddesc/".split(","))); private boolean importDirectoryArchives = false; - private String directoryArchivesDirectory = "archives/"; + private String directoryArchivesDirectory = + "in/relay-descriptors/archives/"; private boolean keepDirectoryArchiveImportHistory = false; private boolean writeSanitizedBridges = false; private boolean replaceIPAddressesWithHashes = false; private long limitBridgeDescriptorMappings = -1L; private String sanitizedBridgesWriteDirectory = "sanitized-bridges/"; private boolean importBridgeSnapshots = false; - private String bridgeSnapshotsDirectory = "bridge-directories/"; + private String bridgeSnapshotsDirectory = "in/bridge-descriptors/"; private boolean downloadRelayDescriptors = false; private List<String> downloadFromDirectoryAuthorities = Arrays.asList(( "86.59.21.38,76.73.17.194:9030,213.115.239.118:443," @@ -60,7 +62,7 @@ public class Configuration { private boolean compressRelayDescriptorDownloads; private boolean downloadExitList = false; private boolean processBridgePoolAssignments = false; - private String assignmentsDirectory = "assignments/"; + private String assignmentsDirectory = "in/bridge-assignments/"; private String sanitizedAssignmentsDirectory = "sanitized-assignments/"; private boolean processTorperfFiles = false; private String torperfOutputDirectory = "torperf/"; diff --git a/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java b/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java index 194e0a2..d0a2600 100644 --- a/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java +++ b/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java @@ -14,10 +14,14 @@ import java.io.IOException; import java.io.StringReader; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.SortedSet; +import java.util.Stack; import java.util.TimeZone; +import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; @@ -73,7 +77,18 @@ public class CachedRelayDescriptorReader { } logger.fine("Reading " + cachedDescDir.getAbsolutePath() + " directory."); - for (File f : cachedDescDir.listFiles()) { + SortedSet<File> cachedDescFiles = new TreeSet<File>(); + Stack<File> files = new Stack<File>(); + files.add(cachedDescDir); + while (!files.isEmpty()) { + File file = files.pop(); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else { + cachedDescFiles.add(file); + } + } + for (File f : cachedDescFiles) { try { // descriptors may contain non-ASCII chars; read as bytes to // determine digests