[tor-commits] [metrics-web/master] Ignore votes in most of the modules.
karsten at torproject.org
karsten at torproject.org
Fri Mar 18 07:39:49 UTC 2016
commit afd9ccb9212389d7a785d60586bdd35c2185366d
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Mar 17 20:58:38 2016 +0100
Ignore votes in most of the modules.
---
.../src/org/torproject/metrics/clients/Main.java | 8 ++++++--
modules/legacy/build.xml | 2 +-
modules/legacy/config.template | 6 ++++--
.../org/torproject/ernie/cron/Configuration.java | 19 +++++++++++++++----
.../legacy/src/org/torproject/ernie/cron/Main.java | 2 +-
.../cron/RelayDescriptorDatabaseImporter.java | 22 ++++++++++++++--------
6 files changed, 41 insertions(+), 18 deletions(-)
diff --git a/modules/clients/src/org/torproject/metrics/clients/Main.java b/modules/clients/src/org/torproject/metrics/clients/Main.java
index 430cb1e..fcd18a3 100644
--- a/modules/clients/src/org/torproject/metrics/clients/Main.java
+++ b/modules/clients/src/org/torproject/metrics/clients/Main.java
@@ -62,9 +62,13 @@ public class Main {
descriptorReader.setExcludeFiles(new File(
"status/relay-descriptors"));
descriptorReader.addDirectory(new File(
- "../../shared/in/recent/relay-descriptors"));
+ "../../shared/in/recent/relay-descriptors/consensuses"));
descriptorReader.addDirectory(new File(
- "../../shared/in/archive/relay-descriptors"));
+ "../../shared/in/recent/relay-descriptors/extra-infos"));
+ descriptorReader.addDirectory(new File(
+ "../../shared/in/archive/relay-descriptors/consensuses"));
+ descriptorReader.addDirectory(new File(
+ "../../shared/in/archive/relay-descriptors/extra-infos"));
Iterator<DescriptorFile> descriptorFiles =
descriptorReader.readDescriptors();
while (descriptorFiles.hasNext()) {
diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml
index b0dc32b..3ecf147 100644
--- a/modules/legacy/build.xml
+++ b/modules/legacy/build.xml
@@ -46,7 +46,7 @@
<!-- Prepare data for being displayed on the website. -->
<target name="run" depends="compile">
<java fork="true"
- maxmemory="1024m"
+ maxmemory="2g"
classname="org.torproject.ernie.cron.Main">
<classpath refid="classpath"/>
</java>
diff --git a/modules/legacy/config.template b/modules/legacy/config.template
index ce779d9..743fe1e 100644
--- a/modules/legacy/config.template
+++ b/modules/legacy/config.template
@@ -1,8 +1,10 @@
## Import directory archives from disk, if available
#ImportDirectoryArchives 0
#
-## Relative path to directory to import directory archives from
-#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/
+## Relative paths to directories to import directory archives from
+#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/
+#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/
+#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/
#
## Keep a history of imported directory archive files to know which files
## have been imported before. This history can be useful when importing
diff --git a/modules/legacy/src/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/org/torproject/ernie/cron/Configuration.java
index 878e882..86d5d10 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/Configuration.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/Configuration.java
@@ -7,6 +7,9 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -17,7 +20,8 @@ import java.util.logging.Logger;
*/
public class Configuration {
private boolean importDirectoryArchives = false;
- private String directoryArchivesDirectory = "in/relay-descriptors/";
+ private List<String> directoryArchivesDirectories =
+ new ArrayList<String>();
private boolean keepDirectoryArchiveImportHistory = false;
private boolean importSanitizedBridges = false;
private String sanitizedBridgesDirectory = "in/bridge-descriptors/";
@@ -55,7 +59,7 @@ public class Configuration {
this.importDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("DirectoryArchivesDirectory")) {
- this.directoryArchivesDirectory = line.split(" ")[1];
+ this.directoryArchivesDirectories.add(line.split(" ")[1]);
} else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
this.keepDirectoryArchiveImportHistory = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -117,8 +121,15 @@ public class Configuration {
public boolean getImportDirectoryArchives() {
return this.importDirectoryArchives;
}
- public String getDirectoryArchivesDirectory() {
- return this.directoryArchivesDirectory;
+ public List<String> getDirectoryArchivesDirectories() {
+ if (this.directoryArchivesDirectories.isEmpty()) {
+ String prefix = "../../shared/in/recent/relay-descriptors/";
+ return Arrays.asList(
+ (prefix + "consensuses/," + prefix + "server-descriptors/,"
+ + prefix + "extra-infos/").split(","));
+ } else {
+ return this.directoryArchivesDirectories;
+ }
}
public boolean getKeepDirectoryArchiveImportHistory() {
return this.keepDirectoryArchiveImportHistory;
diff --git a/modules/legacy/src/org/torproject/ernie/cron/Main.java b/modules/legacy/src/org/torproject/ernie/cron/Main.java
index fb0697c..7319efa 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/Main.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/Main.java
@@ -45,7 +45,7 @@ public class Main {
config.getRelayDescriptorDatabaseJDBC() : null,
config.getWriteRelayDescriptorsRawFiles() ?
config.getRelayDescriptorRawFilesDirectory() : null,
- new File(config.getDirectoryArchivesDirectory()),
+ config.getDirectoryArchivesDirectories(),
statsDirectory,
config.getKeepDirectoryArchiveImportHistory()) : null;
if (rddi != null) {
diff --git a/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index 19f2a0f..35128e7 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -173,7 +173,7 @@ public final class RelayDescriptorDatabaseImporter {
private boolean importIntoDatabase;
private boolean writeRawImportFiles;
- private File archivesDirectory;
+ private List<String> archivesDirectories;
private File statsDirectory;
private boolean keepImportHistory;
@@ -182,14 +182,14 @@ public final class RelayDescriptorDatabaseImporter {
* preparing statements.
*/
public RelayDescriptorDatabaseImporter(String connectionURL,
- String rawFilesDirectory, File archivesDirectory,
+ String rawFilesDirectory, List<String> archivesDirectories,
File statsDirectory, boolean keepImportHistory) {
- if (archivesDirectory == null ||
+ if (archivesDirectories == null ||
statsDirectory == null) {
throw new IllegalArgumentException();
}
- this.archivesDirectory = archivesDirectory;
+ this.archivesDirectories = archivesDirectories;
this.statsDirectory = statsDirectory;
this.keepImportHistory = keepImportHistory;
@@ -805,12 +805,18 @@ public final class RelayDescriptorDatabaseImporter {
}
public void importRelayDescriptors() {
- if (archivesDirectory.exists()) {
- logger.fine("Importing files in directory " + archivesDirectory
- + "/...");
+ logger.fine("Importing files in directories " + archivesDirectories
+ + "/...");
+ if (!this.archivesDirectories.isEmpty()) {
DescriptorReader reader =
DescriptorSourceFactory.createDescriptorReader();
- reader.addDirectory(archivesDirectory);
+ reader.setMaxDescriptorFilesInQueue(10);
+ for (String archivesPath : this.archivesDirectories) {
+ File archivesDirectory = new File(archivesPath);
+ if (archivesDirectory.exists()) {
+ reader.addDirectory(archivesDirectory);
+ }
+ }
if (keepImportHistory) {
reader.setExcludeFiles(new File(statsDirectory,
"database-importer-relay-descriptor-history"));
More information about the tor-commits
mailing list