[or-cvs] [ernie/master 4/4] Make local paths configurable.
karsten at torproject.org
karsten at torproject.org
Tue Apr 13 12:53:00 UTC 2010
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue, 13 Apr 2010 10:50:12 +0200
Subject: Make local paths configurable.
Commit: 50835aebb40911553c2e5c246318e25bd7dd12d5
---
config | 25 +++++++++++++++
src/ArchiveWriter.java | 34 ++++++++++++---------
src/CachedRelayDescriptorReader.java | 26 ++++++++++-----
src/Configuration.java | 55 ++++++++++++++++++++++++++++++++++
src/GeoIPDatabaseManager.java | 15 +++++----
src/Main.java | 24 +++++++++-----
6 files changed, 140 insertions(+), 39 deletions(-)
diff --git a/config b/config
index e4ef699..6f621e9 100644
--- a/config
+++ b/config
@@ -3,9 +3,16 @@
## Read cached-* files from a local Tor data directory
#ImportCachedRelayDescriptors 0
#
+## Relative path to Tor data directory to read cached-* files from (can be
+## specified multiple times)
+#CachedRelayDescriptorsDirectory cacheddesc/
+#
## Import directory archives from disk, if available
#ImportDirectoryArchives 0
#
+## Relative path to directory to import directory archives from
+#DirectoryArchivesDirectory archives/
+#
## Keep a history of imported directory archive files to know which files
## have been imported before. This history can be useful when importing
## from a changing source to avoid importing descriptors over and over
@@ -22,12 +29,21 @@
## Import sanitized bridges from disk, if available
#ImportSanitizedBridges 0
#
+## Relative path to directory to import sanitized bridges from
+#SanitizedBridgesDirectory bridges/
+#
## Import bridge snapshots from disk, if available
#ImportBridgeSnapshots 0
#
+## Relative path to directory to import bridge descriptor snapshots from
+#BridgeSnapshotsDirectory bridge-directories/
+#
## Import local Maxmind GeoIP databases
#ImportGeoIPDatabases 0
#
+## Relative path to directory to import GeoIP databases from
+#GeoIPDatabasesDirectory geoipdb/
+#
## Download (commercial) Maxmind GeoIP database for sanitizing bridge
## descriptors
#DownloadGeoIPDatabase 0
@@ -38,6 +54,9 @@
## Import torperf data, if available, and write stats to disk
#ImportWriteTorperfStats 0
#
+## Relate path to directory to import torperf results from
+#TorperfDirectory torperf/
+#
## Download and process GetTor stats
#DownloadProcessGetTorStats 0
#
@@ -52,6 +71,9 @@
## Write directory archives to disk
#WriteDirectoryArchives 0
#
+## Relative path to directory to write directory archives to
+#DirectoryArchivesOutputDirectory directory-archive/
+#
## Write relay descriptors to a database for later evaluation
#WriteRelayDescriptorDatabase 0
#
@@ -61,6 +83,9 @@
## Write sanitized bridges to disk
#WriteSanitizedBridges 0
#
+## Relate path to directory to write sanitized bridges to
+#SanitizedBridgesWriteDirectory sanitized-bridges/
+#
## Write consensus stats to disk
#WriteConsensusStats 0
#
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
index 726ecfc..de72da1 100644
--- a/src/ArchiveWriter.java
+++ b/src/ArchiveWriter.java
@@ -7,8 +7,10 @@ import org.apache.commons.codec.binary.*;
public class ArchiveWriter {
private Logger logger;
- public ArchiveWriter() {
+ private String outputDirectory;
+ public ArchiveWriter(String outputDirectory) {
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
+ this.outputDirectory = outputDirectory;
}
private void store(byte[] data, String filename) {
@@ -32,7 +34,7 @@ public class ArchiveWriter {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = "directory-archive/consensus/"
+ String filename = outputDirectory + "/consensus/"
+ printFormat.format(new Date(validAfter)) + "-consensus";
this.store(data, filename);
}
@@ -42,7 +44,7 @@ public class ArchiveWriter {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = "directory-archive/vote/"
+ String filename = outputDirectory + "/vote/"
+ printFormat.format(new Date(validAfter)) + "-vote-"
+ fingerprint + "-" + digest;
this.store(data, filename);
@@ -52,7 +54,7 @@ public class ArchiveWriter {
long published) {
SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = "directory-archive/server-descriptor/"
+ String filename = outputDirectory + "/server-descriptor/"
+ printFormat.format(new Date(published))
+ digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ digest;
@@ -63,7 +65,7 @@ public class ArchiveWriter {
String extraInfoDigest, long published) {
SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = "directory-archive/extra-info/"
+ String filename = outputDirectory + "/extra-info/"
+ descriptorFormat.format(new Date(published))
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
@@ -84,7 +86,7 @@ public class ArchiveWriter {
} else if (pop.length() > 0) {
String absPath = pop.getAbsolutePath().replaceAll(":", "-");
String relPath = absPath.substring(absPath.indexOf(
- "directory-archive/"));
+ outputDirectory + "/"));
files.add(relPath);
}
}
@@ -98,13 +100,13 @@ public class ArchiveWriter {
public void dumpStats() {
try {
SortedSet<String> votes = getFileNames(
- new File("directory-archive/vote"));
+ new File(outputDirectory + "/vote"));
SortedSet<String> serverDescs = getFileNames(
- new File("directory-archive/server-descriptor"));
+ new File(outputDirectory + "/server-descriptor"));
SortedSet<String> extraInfos = getFileNames(
- new File("directory-archive/extra-info"));
+ new File(outputDirectory + "/extra-info"));
SortedSet<String> consensuses = getFileNames(
- new File("directory-archive/consensus"));
+ new File(outputDirectory + "/consensus"));
SimpleDateFormat validAfterFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -134,7 +136,7 @@ public class ArchiveWriter {
validAfterTime = line.substring("valid-after ".length());
long validAfter = validAfterFormat.parse(
validAfterTime).getTime();
- votePrefix = "directory-archive/vote/"
+ votePrefix = outputDirectory + "/vote/"
+ consensusVoteFormat.format(new Date(validAfter))
+ "-vote-";
} else if (line.startsWith("dir-source ")) {
@@ -162,7 +164,8 @@ public class ArchiveWriter {
long published = validAfterFormat.parse(
line3.split(" ")[4] + " "
+ line3.split(" ")[5]).getTime();
- String filename = "directory-archive/server-descriptor/"
+ String filename = outputDirectory
+ + "/server-descriptor/"
+ descriptorFormat.format(new Date(published))
+ digest.substring(0, 1) + "/"
+ digest.substring(1, 2) + "/" + digest;
@@ -177,7 +180,8 @@ public class ArchiveWriter {
String extraInfoDigest = line2.startsWith("opt ") ?
line2.split(" ")[2].toLowerCase() :
line2.split(" ")[1].toLowerCase();
- String filename2 = "directory-archive/extra-info/"
+ String filename2 = outputDirectory
+ + "/extra-info/"
+ descriptorFormat.format(new Date(published))
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
@@ -209,7 +213,7 @@ public class ArchiveWriter {
line.split(" ")[3] + "=")).toLowerCase();
long published = validAfterFormat.parse(
line.split(" ")[4] + " " + line.split(" ")[5]).getTime();
- String filename = "directory-archive/server-descriptor/"
+ String filename = outputDirectory + "/server-descriptor/"
+ descriptorFormat.format(new Date(published))
+ digest.substring(0, 1) + "/"
+ digest.substring(1, 2) + "/" + digest;
@@ -224,7 +228,7 @@ public class ArchiveWriter {
String extraInfoDigest = line2.startsWith("opt ") ?
line2.split(" ")[2].toLowerCase() :
line2.split(" ")[1].toLowerCase();
- String filename2 = "directory-archive/extra-info/"
+ String filename2 = outputDirectory + "/extra-info/"
+ descriptorFormat.format(new Date(published))
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
index c71ced0..64a1966 100644
--- a/src/CachedRelayDescriptorReader.java
+++ b/src/CachedRelayDescriptorReader.java
@@ -8,12 +8,19 @@ import java.util.logging.*;
* into directory structure in directory-archive/.
*/
public class CachedRelayDescriptorReader {
- public CachedRelayDescriptorReader(RelayDescriptorParser rdp) {
+ public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
+ List<String> inputDirectories) {
Logger logger = Logger.getLogger(
CachedRelayDescriptorReader.class.getName());
- File cachedDescDir = new File("cacheddesc");
- if (cachedDescDir.exists()) {
- logger.fine("Reading cacheddesc/ directory.");
+ for (String inputDirectory : inputDirectories) {
+ File cachedDescDir = new File(inputDirectory);
+ if (!cachedDescDir.exists()) {
+ logger.warning("Directory " + cachedDescDir.getAbsolutePath()
+ + " does not exist. Skipping.");
+ continue;
+ }
+ logger.fine("Reading " + cachedDescDir.getAbsolutePath()
+ + " directory.");
for (File f : cachedDescDir.listFiles()) {
try {
// descriptors may contain non-ASCII chars; read as bytes to
@@ -86,14 +93,15 @@ public class CachedRelayDescriptorReader {
rdp.parse(descBytes);
}
}
- logger.fine("Finished reading cacheddesc/ directory.");
+ logger.fine("Finished reading "
+ + cachedDescDir.getAbsolutePath() + " directory.");
}
} catch (IOException e) {
- logger.log(Level.WARNING, "Failed reading cacheddesc/ "
- + "directory.", e);
+ logger.log(Level.WARNING, "Failed reading "
+ + cachedDescDir.getAbsolutePath() + " directory.", e);
} catch (ParseException e) {
- logger.log(Level.WARNING, "Failed reading cacheddesc/ "
- + "directory.", e);
+ logger.log(Level.WARNING, "Failed reading "
+ + cachedDescDir.getAbsolutePath() + " directory.", e);
}
}
}
diff --git a/src/Configuration.java b/src/Configuration.java
index a5edf31..7950eed 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -23,16 +23,25 @@ public class Configuration {
private List<String> relayPlatforms = new ArrayList<String>(
Arrays.asList("Linux,Windows,Darwin,FreeBSD".split(",")));
private boolean writeDirectoryArchives = false;
+ private String directoryArchivesOutputDirectory = "directory-archive/";
private boolean importCachedRelayDescriptors = false;
+ //this.cachedRelayDescriptorDirectory.add
+ private List<String> cachedRelayDescriptorsDirectory =
+ new ArrayList<String>(Arrays.asList("cacheddesc/".split(",")));
private boolean importDirectoryArchives = false;
+ private String directoryArchivesDirectory = "archives/";
private boolean keepDirectoryArchiveImportHistory = false;
private boolean writeRelayDescriptorDatabase = false;
private String relayDescriptorDatabaseJdbc =
"jdbc:postgresql://localhost/tordir?user=ernie&password=password";
private boolean writeSanitizedBridges = false;
+ private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
private boolean importSanitizedBridges = false;
+ private String sanitizedBridgesDirectory = "bridges/";
private boolean importBridgeSnapshots = false;
+ private String bridgeSnapshotsDirectory = "bridge-directories/";
private boolean importWriteTorperfStats = false;
+ private String torperfDirectory = "torperf/";
private boolean downloadRelayDescriptors = false;
private List<String> downloadFromDirectoryAuthorities = Arrays.asList(
"86.59.21.38,194.109.206.212,80.190.246.100:8180".split(","));
@@ -41,6 +50,7 @@ public class Configuration {
+ "~gettor/gettor_stats.txt";
private boolean downloadExitList = false;
private boolean importGeoIPDatabases = false;
+ private String geoIPDatabasesDirectory = "geoipdb/";
private boolean downloadGeoIPDatabase = false;
private String maxmindLicenseKey = "";
public Configuration() {
@@ -60,6 +70,7 @@ public class Configuration {
return;
}
String line = null;
+ boolean containsCachedRelayDescriptorsDirectory = false;
try {
BufferedReader br = new BufferedReader(new FileReader(configFile));
while ((line = br.readLine()) != null) {
@@ -99,12 +110,22 @@ public class Configuration {
} else if (line.startsWith("WriteDirectoryArchives")) {
this.writeDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
+ this.directoryArchivesOutputDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportCachedRelayDescriptors")) {
this.importCachedRelayDescriptors = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
+ if (!containsCachedRelayDescriptorsDirectory) {
+ this.cachedRelayDescriptorsDirectory.clear();
+ containsCachedRelayDescriptorsDirectory = true;
+ }
+ this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
} else if (line.startsWith("ImportDirectoryArchives")) {
this.importDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DirectoryArchivesDirectory")) {
+ this.directoryArchivesDirectory = line.split(" ")[1];
} else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
this.keepDirectoryArchiveImportHistory = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -116,15 +137,23 @@ public class Configuration {
} else if (line.startsWith("WriteSanitizedBridges")) {
this.writeSanitizedBridges = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
+ this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportSanitizedBridges")) {
this.importSanitizedBridges = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("SanitizedBridgesDirectory")) {
+ this.sanitizedBridgesDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportBridgeSnapshots")) {
this.importBridgeSnapshots = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("BridgeSnapshotsDirectory")) {
+ this.bridgeSnapshotsDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportWriteTorperfStats")) {
this.importWriteTorperfStats = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("TorperfDirectory")) {
+ this.torperfDirectory = line.split(" ")[1];
} else if (line.startsWith("DownloadRelayDescriptors")) {
this.downloadRelayDescriptors = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -155,6 +184,8 @@ public class Configuration {
} else if (line.startsWith("ImportGeoIPDatabases")) {
this.importGeoIPDatabases = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("GeoIPDatabasesDirectory")) {
+ this.geoIPDatabasesDirectory = line.split(" ")[1];
} else if (line.startsWith("DownloadGeoIPDatabase")) {
this.downloadGeoIPDatabase = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -264,12 +295,21 @@ public class Configuration {
public boolean getWriteDirectoryArchives() {
return this.writeDirectoryArchives;
}
+ public String getDirectoryArchivesOutputDirectory() {
+ return this.directoryArchivesOutputDirectory;
+ }
public boolean getImportCachedRelayDescriptors() {
return this.importCachedRelayDescriptors;
}
+ public List<String> getCachedRelayDescriptorDirectory() {
+ return this.cachedRelayDescriptorsDirectory;
+ }
public boolean getImportDirectoryArchives() {
return this.importDirectoryArchives;
}
+ public String getDirectoryArchivesDirectory() {
+ return this.directoryArchivesDirectory;
+ }
public boolean getKeepDirectoryArchiveImportHistory() {
return this.keepDirectoryArchiveImportHistory;
}
@@ -282,15 +322,27 @@ public class Configuration {
public boolean getWriteSanitizedBridges() {
return this.writeSanitizedBridges;
}
+ public String getSanitizedBridgesWriteDirectory() {
+ return this.sanitizedBridgesWriteDirectory;
+ }
public boolean getImportSanitizedBridges() {
return this.importSanitizedBridges;
}
+ public String getSanitizedBridgesDirectory() {
+ return this.sanitizedBridgesDirectory;
+ }
public boolean getImportBridgeSnapshots() {
return this.importBridgeSnapshots;
}
+ public String getBridgeSnapshotsDirectory() {
+ return this.bridgeSnapshotsDirectory;
+ }
public boolean getImportWriteTorperfStats() {
return this.importWriteTorperfStats;
}
+ public String getTorperfDirectory() {
+ return this.torperfDirectory;
+ }
public boolean getDownloadRelayDescriptors() {
return this.downloadRelayDescriptors;
}
@@ -309,6 +361,9 @@ public class Configuration {
public boolean getImportGeoIPDatabases() {
return this.importGeoIPDatabases;
}
+ public String getGeoIPDatabasesDirectory() {
+ return this.geoIPDatabasesDirectory;
+ }
public boolean getDownloadGeoIPDatabase() {
return this.downloadGeoIPDatabase;
}
diff --git a/src/GeoIPDatabaseManager.java b/src/GeoIPDatabaseManager.java
index 15e5ea1..2850fc0 100644
--- a/src/GeoIPDatabaseManager.java
+++ b/src/GeoIPDatabaseManager.java
@@ -68,6 +68,8 @@ public class GeoIPDatabaseManager {
*/
private String lastDownloadedTime;
+ private String geoipDir;
+
/**
* Logger for this class.
*/
@@ -79,9 +81,10 @@ public class GeoIPDatabaseManager {
* Initializes this class by reading in the database versions known so
* far.
*/
- public GeoIPDatabaseManager() {
+ public GeoIPDatabaseManager(String geoipDir) {
/* Initialize instance variables. */
+ this.geoipDir = geoipDir;
this.combinedDatabaseFile = new File("stats/geoip-database");
this.combinedDatabase = new TreeMap<Long, DatabaseEntry>();
this.allDatabases = new ArrayList<String>();
@@ -192,7 +195,7 @@ public class GeoIPDatabaseManager {
while ((entry = zis.getNextEntry()) != null) {
if (!entry.isDirectory() &&
entry.getName().endsWith("GeoIP-108.csv")) {
- String filename = "geoipdb/GeoIP-108_" + date + ".csv";
+ String filename = geoipDir + "/GeoIP-108_" + date + ".csv";
OutputStream out = new BufferedOutputStream(
new FileOutputStream(filename));
byte[] buffer = new byte[1024];
@@ -212,11 +215,11 @@ public class GeoIPDatabaseManager {
}
/**
- * Imports the GeoIP databases from <code>directory</code> to include
- * them in the combined GeoIP database.
+ * Imports the GeoIP databases to include them in the combined GeoIP
+ * database.
*/
- public void importGeoIPDatabaseFromDisk(String directory) {
- File databaseDirectory = new File(directory);
+ public void importGeoIPDatabaseFromDisk() {
+ File databaseDirectory = new File(this.geoipDir);
if (!databaseDirectory.exists()) {
return;
}
diff --git a/src/Main.java b/src/Main.java
index 2ad4ec1..03bede7 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -40,7 +40,8 @@ public class Main {
// Prepare writing relay descriptor archive to disk
ArchiveWriter aw = config.getWriteDirectoryArchives() ?
- new ArchiveWriter() : null;
+ new ArchiveWriter(config.getDirectoryArchivesOutputDirectory())
+ : null;
// Prepare writing relay descriptors to database
RelayDescriptorDatabaseImporter rddi =
@@ -78,10 +79,11 @@ public class Main {
rdp.setRelayDescriptorDownloader(rdd);
}
if (config.getImportCachedRelayDescriptors()) {
- new CachedRelayDescriptorReader(rdp);
+ new CachedRelayDescriptorReader(rdp,
+ config.getCachedRelayDescriptorDirectory());
}
if (config.getImportDirectoryArchives()) {
- new ArchiveReader(rdp, "archives",
+ new ArchiveReader(rdp, config.getDirectoryArchivesDirectory(),
config.getKeepDirectoryArchiveImportHistory());
}
if (rdd != null) {
@@ -105,18 +107,20 @@ public class Main {
}
// Import/download GeoIP databases
- GeoIPDatabaseManager gd = new GeoIPDatabaseManager();
+ GeoIPDatabaseManager gd = new GeoIPDatabaseManager(
+ config.getGeoIPDatabasesDirectory());
if (config.getDownloadGeoIPDatabase()) {
gd.downloadGeoIPDatabase(config.getMaxmindLicenseKey());
}
if (config.getImportGeoIPDatabases()) {
- gd.importGeoIPDatabaseFromDisk("geoipdb/");
+ gd.importGeoIPDatabaseFromDisk();
gd.writeCombinedDatabase();
}
// Prepare sanitized bridge descriptor writer
SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
- new SanitizedBridgesWriter(gd, "sanitized-bridges") : null;
+ new SanitizedBridgesWriter(gd,
+ config.getSanitizedBridgesWriteDirectory()) : null;
// Prepare bridge descriptor parser
BridgeDescriptorParser bdp = config.getWriteConsensusStats() ||
@@ -125,10 +129,12 @@ public class Main {
// Import bridge descriptors
if (bdp != null && config.getImportSanitizedBridges()) {
- new SanitizedBridgesReader(bdp, "bridges", countries);
+ new SanitizedBridgesReader(bdp,
+ config.getSanitizedBridgesDirectory(), countries);
}
if (bdp != null && config.getImportBridgeSnapshots()) {
- new BridgeSnapshotReader(bdp, "bridge-directories", countries);
+ new BridgeSnapshotReader(bdp, config.getBridgeSnapshotsDirectory(),
+ countries);
}
// Finish writing sanitized bridge descriptors to disk
@@ -149,7 +155,7 @@ public class Main {
// Import and process torperf stats
if (config.getImportWriteTorperfStats()) {
- new TorperfProcessor("torperf");
+ new TorperfProcessor(config.getTorperfDirectory());
}
// Download and process GetTor stats
--
1.6.5
More information about the tor-commits
mailing list