commit ca5fa45df0cfb14801e3556caa93f2cc6d26d790 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Nov 13 18:10:18 2018 +0100

Remove long unused code from legacy module.

This includes the lock file, the option to write raw output files for importing into the database, and a couple boolean config options that have always been true.

Required changes to existing legacy.config (removals):

ImportDirectoryArchives KeepDirectoryArchiveImportHistory WriteRelayDescriptorDatabase WriteRelayDescriptorsRawFiles RelayDescriptorRawFilesDirectory

Part of #28116. --- .../metrics/stats/servers/Configuration.java | 46 +------- .../torproject/metrics/stats/servers/LockFile.java | 61 ---------- .../org/torproject/metrics/stats/servers/Main.java | 33 +----- .../servers/RelayDescriptorDatabaseImporter.java | 126 +-------------------- src/main/resources/legacy.config.template | 20 ---- 5 files changed, 10 insertions(+), 276 deletions(-)

diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java index 76788df..b6ee397 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java @@ -24,21 +24,11 @@ public class Configuration {

private static Logger log = LoggerFactory.getLogger(Configuration.class);

- private boolean importDirectoryArchives = false; - private List<File> directoryArchivesDirectories = new ArrayList<>();

- private boolean keepDirectoryArchiveImportHistory = false; - - private boolean writeRelayDescriptorDatabase = false; - private String relayDescriptorDatabaseJdbc = "jdbc:postgresql://localhost/tordir?user=metrics&password=password";

- private boolean writeRelayDescriptorsRawFiles = false; - - private String relayDescriptorRawFilesDirectory = "pg-import/"; - /** Initializes this configuration class. */ public Configuration() {

@@ -51,24 +41,10 @@ public class Configuration { String line = null; try (BufferedReader br = new BufferedReader(new FileReader(configFile))) { while ((line = br.readLine()) != null) { - if (line.startsWith("ImportDirectoryArchives")) { - this.importDirectoryArchives = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("DirectoryArchivesDirectory")) { + if (line.startsWith("DirectoryArchivesDirectory")) { this.directoryArchivesDirectories.add(new File(line.split(" ")[1])); - } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { - this.keepDirectoryArchiveImportHistory = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("WriteRelayDescriptorDatabase")) { - this.writeRelayDescriptorDatabase = Integer.parseInt( - line.split(" ")[1]) != 0; } else if (line.startsWith("RelayDescriptorDatabaseJDBC")) { this.relayDescriptorDatabaseJdbc = line.split(" ")[1]; - } else if (line.startsWith("WriteRelayDescriptorsRawFiles")) { - this.writeRelayDescriptorsRawFiles = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("RelayDescriptorRawFilesDirectory")) { - this.relayDescriptorRawFilesDirectory = line.split(" ")[1]; } else if (!line.startsWith("#") && line.length() > 0) { log.error("Configuration file contains unrecognized " + "configuration key in line '{}'! Exiting!", line); @@ -93,10 +69,6 @@ public class Configuration { } }

- public boolean getImportDirectoryArchives() { - return this.importDirectoryArchives; - } - /** Returns directories containing archived descriptors. */ public List<File> getDirectoryArchivesDirectories() { if (this.directoryArchivesDirectories.isEmpty()) { @@ -108,24 +80,8 @@ public class Configuration { } }

- public boolean getKeepDirectoryArchiveImportHistory() { - return this.keepDirectoryArchiveImportHistory; - } - - public boolean getWriteRelayDescriptorDatabase() { - return this.writeRelayDescriptorDatabase; - } - public String getRelayDescriptorDatabaseJdbc() { return this.relayDescriptorDatabaseJdbc; } - - public boolean getWriteRelayDescriptorsRawFiles() { - return this.writeRelayDescriptorsRawFiles; - } - - public String getRelayDescriptorRawFilesDirectory() { - return this.relayDescriptorRawFilesDirectory; - } }

diff --git a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java b/src/main/java/org/torproject/metrics/stats/servers/LockFile.java deleted file mode 100644 index c6063d1..0000000 --- a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright 2011--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.servers; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; - -public class LockFile { - - private File lockFile; - - private static Logger log = LoggerFactory.getLogger(LockFile.class); - - public LockFile() { - this.lockFile = new File("lock"); - } - - /** Acquires the lock by checking whether a lock file already exists, - * and if not, by creating one with the current system time as - * content. */ - public boolean acquireLock() { - log.debug("Trying to acquire lock..."); - try { - if (this.lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader("lock")); - long runStarted = Long.parseLong(br.readLine()); - br.close(); - if (System.currentTimeMillis() - runStarted - < 23L * 60L * 60L * 1000L) { - return false; - } - } - BufferedWriter bw = new BufferedWriter(new FileWriter("lock")); - bw.append("").append(String.valueOf(System.currentTimeMillis())) - .append("\n"); - bw.close(); - log.debug("Acquired lock."); - return true; - } catch (IOException e) { - log.warn("Caught exception while trying to acquire " - + "lock!"); - return false; - } - } - - /** Releases the lock by deleting the lock file, if present. */ - public void releaseLock() { - log.debug("Releasing lock..."); - this.lockFile.delete(); - log.debug("Released lock."); - } -} - diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java index 4d349bc..1454418 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Main.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java @@ -24,38 +24,15 @@ public class Main { // Initialize configuration Configuration config = new Configuration();

- // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(); - if (!lf.acquireLock()) { - log.error("Warning: ERNIE is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } - // Define stats directory for temporary files File statsDirectory = new File("stats");

// Import relay descriptors - if (config.getImportDirectoryArchives()) { - RelayDescriptorDatabaseImporter rddi = - config.getWriteRelayDescriptorDatabase() - || config.getWriteRelayDescriptorsRawFiles() - ? new RelayDescriptorDatabaseImporter( - config.getWriteRelayDescriptorDatabase() - ? config.getRelayDescriptorDatabaseJdbc() : null, - config.getWriteRelayDescriptorsRawFiles() - ? config.getRelayDescriptorRawFilesDirectory() : null, - config.getDirectoryArchivesDirectories(), - statsDirectory, - config.getKeepDirectoryArchiveImportHistory()) : null; - if (null != rddi) { - rddi.importRelayDescriptors(); - rddi.closeConnection(); - } - } - - // Remove lock file - lf.releaseLock(); + RelayDescriptorDatabaseImporter rddi = new RelayDescriptorDatabaseImporter( + config.getRelayDescriptorDatabaseJdbc(), + config.getDirectoryArchivesDirectories(), statsDirectory); + rddi.importRelayDescriptors(); + rddi.closeConnection();

log.info("Terminating ERNIE."); } diff --git a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java index 2d1ae47..d1ae43c 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java @@ -10,15 +10,10 @@ import org.torproject.descriptor.ExtraInfoDescriptor; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus;

-import org.postgresql.util.PGbytea; - import org.slf4j.Logger; import org.slf4j.LoggerFactory;

-import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DriverManager; @@ -95,21 +90,6 @@ public final class RelayDescriptorDatabaseImporter { = LoggerFactory.getLogger(RelayDescriptorDatabaseImporter.class);

/** - * Directory for writing raw import files. - */ - private String rawFilesDirectory; - - /** - * Raw import file containing status entries. - */ - private BufferedWriter statusentryOut; - - /** - * Raw import file containing bandwidth histories. - */ - private BufferedWriter bwhistOut; - - /** * Date format to parse timestamps. */ private SimpleDateFormat dateTimeFormat; @@ -126,30 +106,24 @@ public final class RelayDescriptorDatabaseImporter { */ private Set<String> insertedStatusEntries = new HashSet<>();

- private boolean importIntoDatabase; - - private boolean writeRawImportFiles; + private boolean importIntoDatabase = true;

private List<File> archivesDirectories;

private File statsDirectory;

- private boolean keepImportHistory; - /** * Initialize database importer by connecting to the database and * preparing statements. */ public RelayDescriptorDatabaseImporter(String connectionUrl, - String rawFilesDirectory, List<File> archivesDirectories, - File statsDirectory, boolean keepImportHistory) { + List<File> archivesDirectories, File statsDirectory) {

if (archivesDirectories == null || statsDirectory == null) { throw new IllegalArgumentException(); } this.archivesDirectories = archivesDirectories; this.statsDirectory = statsDirectory; - this.keepImportHistory = keepImportHistory;

if (connectionUrl != null) { try { @@ -175,18 +149,11 @@ public final class RelayDescriptorDatabaseImporter { this.psU = conn.prepareStatement("INSERT INTO scheduled_updates " + "(date) VALUES (?)"); this.scheduledUpdates = new HashSet<>(); - this.importIntoDatabase = true; } catch (SQLException e) { log.warn("Could not connect to database or prepare statements.", e); } }

- /* Remember where we want to write raw import files. */ - if (rawFilesDirectory != null) { - this.rawFilesDirectory = rawFilesDirectory; - this.writeRawImportFiles = true; - } - /* Initialize date format, so that we can format timestamps. */ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -278,53 +245,6 @@ public final class RelayDescriptorDatabaseImporter { this.importIntoDatabase = false; } } - if (this.writeRawImportFiles) { - try { - if (this.statusentryOut == null) { - new File(rawFilesDirectory).mkdirs(); - this.statusentryOut = new BufferedWriter(new FileWriter( - rawFilesDirectory + "/statusentry.sql")); - this.statusentryOut.write(" COPY statusentry (validafter, " - + "nickname, fingerprint, descriptor, published, address, " - + "orport, dirport, isauthority, isbadExit, " - + "isbaddirectory, isexit, isfast, isguard, ishsdir, " - + "isnamed, isstable, isrunning, isunnamed, isvalid, " - + "isv2dir, isv3dir, version, bandwidth, ports, rawdesc) " - + "FROM stdin;\n"); - } - this.statusentryOut.write( - this.dateTimeFormat.format(validAfter) + "\t" + nickname - + "\t" + fingerprint.toLowerCase() + "\t" - + descriptor.toLowerCase() + "\t" - + this.dateTimeFormat.format(published) + "\t" + address - + "\t" + orPort + "\t" + dirPort + "\t" - + (flags.contains("Authority") ? "t" : "f") + "\t" - + (flags.contains("BadExit") ? "t" : "f") + "\t" - + (flags.contains("BadDirectory") ? "t" : "f") + "\t" - + (flags.contains("Exit") ? "t" : "f") + "\t" - + (flags.contains("Fast") ? "t" : "f") + "\t" - + (flags.contains("Guard") ? "t" : "f") + "\t" - + (flags.contains("HSDir") ? "t" : "f") + "\t" - + (flags.contains("Named") ? "t" : "f") + "\t" - + (flags.contains("Stable") ? "t" : "f") + "\t" - + (flags.contains("Running") ? "t" : "f") + "\t" - + (flags.contains("Unnamed") ? "t" : "f") + "\t" - + (flags.contains("Valid") ? "t" : "f") + "\t" - + (flags.contains("V2Dir") ? "t" : "f") + "\t" - + (flags.contains("V3Dir") ? "t" : "f") + "\t" - + (version != null ? version : "\N") + "\t" - + (bandwidth >= 0 ? bandwidth : "\N") + "\t" - + (ports != null ? ports : "\N") + "\t"); - this.statusentryOut.write(PGbytea.toPGString(rawDescriptor) - .replaceAll("\\", "\\\\") + "\n"); - } catch (IOException e) { - log.warn("Could not write network status " - + "consensus entry to raw database import file. We won't " - + "make any further attempts to write raw import files in " - + "this execution.", e); - this.writeRawImportFiles = false; - } - } }

/** @@ -552,26 +472,6 @@ public final class RelayDescriptorDatabaseImporter { this.importIntoDatabase = false; } } - if (this.writeRawImportFiles) { - try { - if (this.bwhistOut == null) { - new File(rawFilesDirectory).mkdirs(); - this.bwhistOut = new BufferedWriter(new FileWriter( - rawFilesDirectory + "/bwhist.sql")); - } - this.bwhistOut.write("SELECT insert_bwhist('" + fingerprint - + "','" + lastDate + "','" + readIntArray.toString() - + "','" + writtenIntArray.toString() + "','" - + dirreadIntArray.toString() + "','" - + dirwrittenIntArray.toString() + "');\n"); - } catch (IOException e) { - log.warn("Could not write bandwidth " - + "history to raw database import file. We won't make " - + "any further attempts to write raw import files in " - + "this execution.", e); - this.writeRawImportFiles = false; - } - } readArray = writtenArray = dirreadArray = dirwrittenArray = null; } if (historyLine.equals("EOL")) { @@ -628,9 +528,7 @@ public final class RelayDescriptorDatabaseImporter { reader.setMaxDescriptorsInQueue(10); File historyFile = new File(statsDirectory, "database-importer-relay-descriptor-history"); - if (keepImportHistory) { - reader.setHistoryFile(historyFile); - } + reader.setHistoryFile(historyFile); for (Descriptor descriptor : reader.readDescriptors( this.archivesDirectories.toArray( new File[this.archivesDirectories.size()]))) { @@ -641,9 +539,7 @@ public final class RelayDescriptorDatabaseImporter { this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); } } - if (keepImportHistory) { - reader.saveHistoryFile(historyFile); - } + reader.saveHistoryFile(historyFile); }

log.info("Finished importing relay descriptors."); @@ -728,20 +624,6 @@ public final class RelayDescriptorDatabaseImporter { log.warn("Could not close database connection.", e); } } - - /* Close raw import files. */ - try { - if (this.statusentryOut != null) { - this.statusentryOut.write("\.\n"); - this.statusentryOut.close(); - } - if (this.bwhistOut != null) { - this.bwhistOut.write("\.\n"); - this.bwhistOut.close(); - } - } catch (IOException e) { - log.warn("Could not close one or more raw database import files.", e); - } } }

diff --git a/src/main/resources/legacy.config.template b/src/main/resources/legacy.config.template index 5475c1e..e2e0dac 100644 --- a/src/main/resources/legacy.config.template +++ b/src/main/resources/legacy.config.template @@ -1,28 +1,8 @@ -## Import directory archives from disk, if available -#ImportDirectoryArchives 0 -# ## Relative paths to directories to import directory archives from #DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/ #DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/ #DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/ # -## Keep a history of imported directory archive files to know which files -## have been imported before. This history can be useful when importing -## from a changing source to avoid importing descriptors over and over -## again, but it can be confusing to users who don't know about it. -#KeepDirectoryArchiveImportHistory 0 -# -## Write relay descriptors to the database -#WriteRelayDescriptorDatabase 0 -# ## JDBC string for relay descriptor database #RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=metrics&password=password # -## Write relay descriptors to raw text files for importing them into the -## database using PostgreSQL's \copy command -#WriteRelayDescriptorsRawFiles 0 -# -## Relative path to directory to write raw text files; note that existing -## files will be overwritten! -#RelayDescriptorRawFilesDirectory pg-import/ -#