commit afd9ccb9212389d7a785d60586bdd35c2185366d Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Mar 17 20:58:38 2016 +0100

Ignore votes in most of the modules. --- .../src/org/torproject/metrics/clients/Main.java | 8 ++++++-- modules/legacy/build.xml | 2 +- modules/legacy/config.template | 6 ++++-- .../org/torproject/ernie/cron/Configuration.java | 19 +++++++++++++++---- .../legacy/src/org/torproject/ernie/cron/Main.java | 2 +- .../cron/RelayDescriptorDatabaseImporter.java | 22 ++++++++++++++-------- 6 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/modules/clients/src/org/torproject/metrics/clients/Main.java b/modules/clients/src/org/torproject/metrics/clients/Main.java index 430cb1e..fcd18a3 100644 --- a/modules/clients/src/org/torproject/metrics/clients/Main.java +++ b/modules/clients/src/org/torproject/metrics/clients/Main.java @@ -62,9 +62,13 @@ public class Main { descriptorReader.setExcludeFiles(new File( "status/relay-descriptors")); descriptorReader.addDirectory(new File( - "../../shared/in/recent/relay-descriptors")); + "../../shared/in/recent/relay-descriptors/consensuses")); descriptorReader.addDirectory(new File( - "../../shared/in/archive/relay-descriptors")); + "../../shared/in/recent/relay-descriptors/extra-infos")); + descriptorReader.addDirectory(new File( + "../../shared/in/archive/relay-descriptors/consensuses")); + descriptorReader.addDirectory(new File( + "../../shared/in/archive/relay-descriptors/extra-infos")); Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); while (descriptorFiles.hasNext()) { diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml index b0dc32b..3ecf147 100644 --- a/modules/legacy/build.xml +++ b/modules/legacy/build.xml @@ -46,7 +46,7 @@ <!-- Prepare data for being displayed on the website. --> <target name="run" depends="compile"> <java fork="true" - maxmemory="1024m" + maxmemory="2g" classname="org.torproject.ernie.cron.Main"> <classpath refid="classpath"/> </java> diff --git a/modules/legacy/config.template b/modules/legacy/config.template index ce779d9..743fe1e 100644 --- a/modules/legacy/config.template +++ b/modules/legacy/config.template @@ -1,8 +1,10 @@ ## Import directory archives from disk, if available #ImportDirectoryArchives 0 # -## Relative path to directory to import directory archives from -#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/ +## Relative paths to directories to import directory archives from +#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/ +#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/ +#DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/ # ## Keep a history of imported directory archive files to know which files ## have been imported before. This history can be useful when importing diff --git a/modules/legacy/src/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/org/torproject/ernie/cron/Configuration.java index 878e882..86d5d10 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/Configuration.java +++ b/modules/legacy/src/org/torproject/ernie/cron/Configuration.java @@ -7,6 +7,9 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger;

@@ -17,7 +20,8 @@ import java.util.logging.Logger; */ public class Configuration { private boolean importDirectoryArchives = false; - private String directoryArchivesDirectory = "in/relay-descriptors/"; + private List<String> directoryArchivesDirectories = + new ArrayList<String>(); private boolean keepDirectoryArchiveImportHistory = false; private boolean importSanitizedBridges = false; private String sanitizedBridgesDirectory = "in/bridge-descriptors/"; @@ -55,7 +59,7 @@ public class Configuration { this.importDirectoryArchives = Integer.parseInt( line.split(" ")[1]) != 0; } else if (line.startsWith("DirectoryArchivesDirectory")) { - this.directoryArchivesDirectory = line.split(" ")[1]; + this.directoryArchivesDirectories.add(line.split(" ")[1]); } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { this.keepDirectoryArchiveImportHistory = Integer.parseInt( line.split(" ")[1]) != 0; @@ -117,8 +121,15 @@ public class Configuration { public boolean getImportDirectoryArchives() { return this.importDirectoryArchives; } - public String getDirectoryArchivesDirectory() { - return this.directoryArchivesDirectory; + public List<String> getDirectoryArchivesDirectories() { + if (this.directoryArchivesDirectories.isEmpty()) { + String prefix = "../../shared/in/recent/relay-descriptors/"; + return Arrays.asList( + (prefix + "consensuses/," + prefix + "server-descriptors/," + + prefix + "extra-infos/").split(",")); + } else { + return this.directoryArchivesDirectories; + } } public boolean getKeepDirectoryArchiveImportHistory() { return this.keepDirectoryArchiveImportHistory; diff --git a/modules/legacy/src/org/torproject/ernie/cron/Main.java b/modules/legacy/src/org/torproject/ernie/cron/Main.java index fb0697c..7319efa 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/Main.java +++ b/modules/legacy/src/org/torproject/ernie/cron/Main.java @@ -45,7 +45,7 @@ public class Main { config.getRelayDescriptorDatabaseJDBC() : null, config.getWriteRelayDescriptorsRawFiles() ? config.getRelayDescriptorRawFilesDirectory() : null, - new File(config.getDirectoryArchivesDirectory()), + config.getDirectoryArchivesDirectories(), statsDirectory, config.getKeepDirectoryArchiveImportHistory()) : null; if (rddi != null) { diff --git a/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java index 19f2a0f..35128e7 100644 --- a/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java +++ b/modules/legacy/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java @@ -173,7 +173,7 @@ public final class RelayDescriptorDatabaseImporter { private boolean importIntoDatabase; private boolean writeRawImportFiles;

- private File archivesDirectory; + private List<String> archivesDirectories; private File statsDirectory; private boolean keepImportHistory;

@@ -182,14 +182,14 @@ public final class RelayDescriptorDatabaseImporter { * preparing statements. */ public RelayDescriptorDatabaseImporter(String connectionURL, - String rawFilesDirectory, File archivesDirectory, + String rawFilesDirectory, List<String> archivesDirectories, File statsDirectory, boolean keepImportHistory) {

- if (archivesDirectory == null || + if (archivesDirectories == null || statsDirectory == null) { throw new IllegalArgumentException(); } - this.archivesDirectory = archivesDirectory; + this.archivesDirectories = archivesDirectories; this.statsDirectory = statsDirectory; this.keepImportHistory = keepImportHistory;

@@ -805,12 +805,18 @@ public final class RelayDescriptorDatabaseImporter { }

public void importRelayDescriptors() { - if (archivesDirectory.exists()) { - logger.fine("Importing files in directory " + archivesDirectory - + "/..."); + logger.fine("Importing files in directories " + archivesDirectories + + "/..."); + if (!this.archivesDirectories.isEmpty()) { DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(archivesDirectory); + reader.setMaxDescriptorFilesInQueue(10); + for (String archivesPath : this.archivesDirectories) { + File archivesDirectory = new File(archivesPath); + if (archivesDirectory.exists()) { + reader.addDirectory(archivesDirectory); + } + } if (keepImportHistory) { reader.setExcludeFiles(new File(statsDirectory, "database-importer-relay-descriptor-history"));