[tor-commits] [metrics-web/master] Remove long unused code from legacy module.

22 Nov 2018

commit ca5fa45df0cfb14801e3556caa93f2cc6d26d790
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Tue Nov 13 18:10:18 2018 +0100
Remove long unused code from legacy module.
This includes the lock file, the option to write raw output files for
    importing into the database, and a couple boolean config options that
    have always been true.
Required changes to existing legacy.config (removals):
ImportDirectoryArchives
    KeepDirectoryArchiveImportHistory
    WriteRelayDescriptorDatabase
    WriteRelayDescriptorsRawFiles
    RelayDescriptorRawFilesDirectory
Part of #28116.
---
 .../metrics/stats/servers/Configuration.java       |  46 +-------
 .../torproject/metrics/stats/servers/LockFile.java |  61 ----------
 .../org/torproject/metrics/stats/servers/Main.java |  33 +-----
 .../servers/RelayDescriptorDatabaseImporter.java   | 126 +--------------------
 src/main/resources/legacy.config.template          |  20 ----
 5 files changed, 10 insertions(+), 276 deletions(-)

diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
index 76788df..b6ee397 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java
@@ -24,21 +24,11 @@ public class Configuration {
private static Logger log = LoggerFactory.getLogger(Configuration.class);
-  private boolean importDirectoryArchives = false;
-
   private List<File> directoryArchivesDirectories = new ArrayList<>();
-  private boolean keepDirectoryArchiveImportHistory = false;
-
-  private boolean writeRelayDescriptorDatabase = false;
-
   private String relayDescriptorDatabaseJdbc =
       "jdbc:postgresql://localhost/tordir?user=metrics&password=password";
-  private boolean writeRelayDescriptorsRawFiles = false;
-
-  private String relayDescriptorRawFilesDirectory = "pg-import/";
-
   /** Initializes this configuration class. */
   public Configuration() {
@@ -51,24 +41,10 @@ public class Configuration {
     String line = null;
     try (BufferedReader br = new BufferedReader(new FileReader(configFile))) {
       while ((line = br.readLine()) != null) {
-        if (line.startsWith("ImportDirectoryArchives")) {
-          this.importDirectoryArchives = Integer.parseInt(
-              line.split(" ")[1]) != 0;
-        } else if (line.startsWith("DirectoryArchivesDirectory")) {
+        if (line.startsWith("DirectoryArchivesDirectory")) {
           this.directoryArchivesDirectories.add(new File(line.split(" ")[1]));
-        } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
-          this.keepDirectoryArchiveImportHistory = Integer.parseInt(
-              line.split(" ")[1]) != 0;
-        } else if (line.startsWith("WriteRelayDescriptorDatabase")) {
-          this.writeRelayDescriptorDatabase = Integer.parseInt(
-              line.split(" ")[1]) != 0;
         } else if (line.startsWith("RelayDescriptorDatabaseJDBC")) {
           this.relayDescriptorDatabaseJdbc = line.split(" ")[1];
-        } else if (line.startsWith("WriteRelayDescriptorsRawFiles")) {
-          this.writeRelayDescriptorsRawFiles = Integer.parseInt(
-              line.split(" ")[1]) != 0;
-        } else if (line.startsWith("RelayDescriptorRawFilesDirectory")) {
-          this.relayDescriptorRawFilesDirectory = line.split(" ")[1];
         } else if (!line.startsWith("#") && line.length() > 0) {
           log.error("Configuration file contains unrecognized "
               + "configuration key in line '{}'! Exiting!", line);
@@ -93,10 +69,6 @@ public class Configuration {
     }
   }
-  public boolean getImportDirectoryArchives() {
-    return this.importDirectoryArchives;
-  }
-
   /** Returns directories containing archived descriptors. */
   public List<File> getDirectoryArchivesDirectories() {
     if (this.directoryArchivesDirectories.isEmpty()) {
@@ -108,24 +80,8 @@ public class Configuration {
     }
   }
-  public boolean getKeepDirectoryArchiveImportHistory() {
-    return this.keepDirectoryArchiveImportHistory;
-  }
-
-  public boolean getWriteRelayDescriptorDatabase() {
-    return this.writeRelayDescriptorDatabase;
-  }
-
   public String getRelayDescriptorDatabaseJdbc() {
     return this.relayDescriptorDatabaseJdbc;
   }
-
-  public boolean getWriteRelayDescriptorsRawFiles() {
-    return this.writeRelayDescriptorsRawFiles;
-  }
-
-  public String getRelayDescriptorRawFilesDirectory() {
-    return this.relayDescriptorRawFilesDirectory;
-  }
 }
diff --git a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java b/src/main/java/org/torproject/metrics/stats/servers/LockFile.java
deleted file mode 100644
index c6063d1..0000000
--- a/src/main/java/org/torproject/metrics/stats/servers/LockFile.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/* Copyright 2011--2018 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.stats.servers;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-
-public class LockFile {
-
-  private File lockFile;
-
-  private static Logger log = LoggerFactory.getLogger(LockFile.class);
-
-  public LockFile() {
-    this.lockFile = new File("lock");
-  }
-
-  /** Acquires the lock by checking whether a lock file already exists,
-   * and if not, by creating one with the current system time as
-   * content. */
-  public boolean acquireLock() {
-    log.debug("Trying to acquire lock...");
-    try {
-      if (this.lockFile.exists()) {
-        BufferedReader br = new BufferedReader(new FileReader("lock"));
-        long runStarted = Long.parseLong(br.readLine());
-        br.close();
-        if (System.currentTimeMillis() - runStarted
-            < 23L * 60L * 60L * 1000L) {
-          return false;
-        }
-      }
-      BufferedWriter bw = new BufferedWriter(new FileWriter("lock"));
-      bw.append("").append(String.valueOf(System.currentTimeMillis()))
-          .append("\n");
-      bw.close();
-      log.debug("Acquired lock.");
-      return true;
-    } catch (IOException e) {
-      log.warn("Caught exception while trying to acquire "
-          + "lock!");
-      return false;
-    }
-  }
-
-  /** Releases the lock by deleting the lock file, if present. */
-  public void releaseLock() {
-    log.debug("Releasing lock...");
-    this.lockFile.delete();
-    log.debug("Released lock.");
-  }
-}
-
diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java
index 4d349bc..1454418 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/Main.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java
@@ -24,38 +24,15 @@ public class Main {
     // Initialize configuration
     Configuration config = new Configuration();
-    // Use lock file to avoid overlapping runs
-    LockFile lf = new LockFile();
-    if (!lf.acquireLock()) {
-      log.error("Warning: ERNIE is already running or has not exited "
-          + "cleanly! Exiting!");
-      System.exit(1);
-    }
-
     // Define stats directory for temporary files
     File statsDirectory = new File("stats");
// Import relay descriptors
-    if (config.getImportDirectoryArchives()) {
-      RelayDescriptorDatabaseImporter rddi =
-          config.getWriteRelayDescriptorDatabase()
-          || config.getWriteRelayDescriptorsRawFiles()
-          ? new RelayDescriptorDatabaseImporter(
-          config.getWriteRelayDescriptorDatabase()
-          ? config.getRelayDescriptorDatabaseJdbc() : null,
-          config.getWriteRelayDescriptorsRawFiles()
-          ? config.getRelayDescriptorRawFilesDirectory() : null,
-          config.getDirectoryArchivesDirectories(),
-          statsDirectory,
-          config.getKeepDirectoryArchiveImportHistory()) : null;
-      if (null != rddi) {
-        rddi.importRelayDescriptors();
-        rddi.closeConnection();
-      }
-    }
-
-    // Remove lock file
-    lf.releaseLock();
+    RelayDescriptorDatabaseImporter rddi = new RelayDescriptorDatabaseImporter(
+        config.getRelayDescriptorDatabaseJdbc(),
+        config.getDirectoryArchivesDirectories(), statsDirectory);
+    rddi.importRelayDescriptors();
+    rddi.closeConnection();
log.info("Terminating ERNIE.");
   }
diff --git a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java
index 2d1ae47..d1ae43c 100644
--- a/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java
+++ b/src/main/java/org/torproject/metrics/stats/servers/RelayDescriptorDatabaseImporter.java
@@ -10,15 +10,10 @@ import org.torproject.descriptor.ExtraInfoDescriptor;
 import org.torproject.descriptor.NetworkStatusEntry;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.postgresql.util.PGbytea;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
 import java.sql.CallableStatement;
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -95,21 +90,6 @@ public final class RelayDescriptorDatabaseImporter {
       = LoggerFactory.getLogger(RelayDescriptorDatabaseImporter.class);
/**
-   * Directory for writing raw import files.
-   */
-  private String rawFilesDirectory;
-
-  /**
-   * Raw import file containing status entries.
-   */
-  private BufferedWriter statusentryOut;
-
-  /**
-   * Raw import file containing bandwidth histories.
-   */
-  private BufferedWriter bwhistOut;
-
-  /**
    * Date format to parse timestamps.
    */
   private SimpleDateFormat dateTimeFormat;
@@ -126,30 +106,24 @@ public final class RelayDescriptorDatabaseImporter {
    */
   private Set<String> insertedStatusEntries = new HashSet<>();
-  private boolean importIntoDatabase;
-
-  private boolean writeRawImportFiles;
+  private boolean importIntoDatabase = true;
private List<File> archivesDirectories;
private File statsDirectory;
-  private boolean keepImportHistory;
-
   /**
    * Initialize database importer by connecting to the database and
    * preparing statements.
    */
   public RelayDescriptorDatabaseImporter(String connectionUrl,
-      String rawFilesDirectory, List<File> archivesDirectories,
-      File statsDirectory, boolean keepImportHistory) {
+      List<File> archivesDirectories, File statsDirectory) {
if (archivesDirectories == null || statsDirectory == null) {
       throw new IllegalArgumentException();
     }
     this.archivesDirectories = archivesDirectories;
     this.statsDirectory = statsDirectory;
-    this.keepImportHistory = keepImportHistory;
if (connectionUrl != null) {
       try {
@@ -175,18 +149,11 @@ public final class RelayDescriptorDatabaseImporter {
         this.psU = conn.prepareStatement("INSERT INTO scheduled_updates "
             + "(date) VALUES (?)");
         this.scheduledUpdates = new HashSet<>();
-        this.importIntoDatabase = true;
       } catch (SQLException e) {
         log.warn("Could not connect to database or prepare statements.", e);
       }
     }
-    /* Remember where we want to write raw import files. */
-    if (rawFilesDirectory != null) {
-      this.rawFilesDirectory = rawFilesDirectory;
-      this.writeRawImportFiles = true;
-    }
-
     /* Initialize date format, so that we can format timestamps. */
     this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -278,53 +245,6 @@ public final class RelayDescriptorDatabaseImporter {
         this.importIntoDatabase = false;
       }
     }
-    if (this.writeRawImportFiles) {
-      try {
-        if (this.statusentryOut == null) {
-          new File(rawFilesDirectory).mkdirs();
-          this.statusentryOut = new BufferedWriter(new FileWriter(
-              rawFilesDirectory + "/statusentry.sql"));
-          this.statusentryOut.write(" COPY statusentry (validafter, "
-              + "nickname, fingerprint, descriptor, published, address, "
-              + "orport, dirport, isauthority, isbadExit, "
-              + "isbaddirectory, isexit, isfast, isguard, ishsdir, "
-              + "isnamed, isstable, isrunning, isunnamed, isvalid, "
-              + "isv2dir, isv3dir, version, bandwidth, ports, rawdesc) "
-              + "FROM stdin;\n");
-        }
-        this.statusentryOut.write(
-            this.dateTimeFormat.format(validAfter) + "\t" + nickname
-            + "\t" + fingerprint.toLowerCase() + "\t"
-            + descriptor.toLowerCase() + "\t"
-            + this.dateTimeFormat.format(published) + "\t" + address
-            + "\t" + orPort + "\t" + dirPort + "\t"
-            + (flags.contains("Authority") ? "t" : "f") + "\t"
-            + (flags.contains("BadExit") ? "t" : "f") + "\t"
-            + (flags.contains("BadDirectory") ? "t" : "f") + "\t"
-            + (flags.contains("Exit") ? "t" : "f") + "\t"
-            + (flags.contains("Fast") ? "t" : "f") + "\t"
-            + (flags.contains("Guard") ? "t" : "f") + "\t"
-            + (flags.contains("HSDir") ? "t" : "f") + "\t"
-            + (flags.contains("Named") ? "t" : "f") + "\t"
-            + (flags.contains("Stable") ? "t" : "f") + "\t"
-            + (flags.contains("Running") ? "t" : "f") + "\t"
-            + (flags.contains("Unnamed") ? "t" : "f") + "\t"
-            + (flags.contains("Valid") ? "t" : "f") + "\t"
-            + (flags.contains("V2Dir") ? "t" : "f") + "\t"
-            + (flags.contains("V3Dir") ? "t" : "f") + "\t"
-            + (version != null ? version : "\N") + "\t"
-            + (bandwidth >= 0 ? bandwidth : "\N") + "\t"
-            + (ports != null ? ports : "\N") + "\t");
-        this.statusentryOut.write(PGbytea.toPGString(rawDescriptor)
-            .replaceAll("\\", "\\\\") + "\n");
-      } catch (IOException e) {
-        log.warn("Could not write network status "
-            + "consensus entry to raw database import file.  We won't "
-            + "make any further attempts to write raw import files in "
-            + "this execution.", e);
-        this.writeRawImportFiles = false;
-      }
-    }
   }
/**
@@ -552,26 +472,6 @@ public final class RelayDescriptorDatabaseImporter {
             this.importIntoDatabase = false;
           }
         }
-        if (this.writeRawImportFiles) {
-          try {
-            if (this.bwhistOut == null) {
-              new File(rawFilesDirectory).mkdirs();
-              this.bwhistOut = new BufferedWriter(new FileWriter(
-                  rawFilesDirectory + "/bwhist.sql"));
-            }
-            this.bwhistOut.write("SELECT insert_bwhist('" + fingerprint
-                + "','" + lastDate + "','" + readIntArray.toString()
-                + "','" + writtenIntArray.toString() + "','"
-                + dirreadIntArray.toString() + "','"
-                + dirwrittenIntArray.toString() + "');\n");
-          } catch (IOException e) {
-            log.warn("Could not write bandwidth "
-                + "history to raw database import file.  We won't make "
-                + "any further attempts to write raw import files in "
-                + "this execution.", e);
-            this.writeRawImportFiles = false;
-          }
-        }
         readArray = writtenArray = dirreadArray = dirwrittenArray = null;
       }
       if (historyLine.equals("EOL")) {
@@ -628,9 +528,7 @@ public final class RelayDescriptorDatabaseImporter {
       reader.setMaxDescriptorsInQueue(10);
       File historyFile = new File(statsDirectory,
           "database-importer-relay-descriptor-history");
-      if (keepImportHistory) {
-        reader.setHistoryFile(historyFile);
-      }
+      reader.setHistoryFile(historyFile);
       for (Descriptor descriptor : reader.readDescriptors(
           this.archivesDirectories.toArray(
           new File[this.archivesDirectories.size()]))) {
@@ -641,9 +539,7 @@ public final class RelayDescriptorDatabaseImporter {
           this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
         }
       }
-      if (keepImportHistory) {
-        reader.saveHistoryFile(historyFile);
-      }
+      reader.saveHistoryFile(historyFile);
     }
log.info("Finished importing relay descriptors.");
@@ -728,20 +624,6 @@ public final class RelayDescriptorDatabaseImporter {
         log.warn("Could not close database connection.", e);
       }
     }
-
-    /* Close raw import files. */
-    try {
-      if (this.statusentryOut != null) {
-        this.statusentryOut.write("\.\n");
-        this.statusentryOut.close();
-      }
-      if (this.bwhistOut != null) {
-        this.bwhistOut.write("\.\n");
-        this.bwhistOut.close();
-      }
-    } catch (IOException e) {
-      log.warn("Could not close one or more raw database import files.", e);
-    }
   }
 }
diff --git a/src/main/resources/legacy.config.template b/src/main/resources/legacy.config.template
index 5475c1e..e2e0dac 100644
--- a/src/main/resources/legacy.config.template
+++ b/src/main/resources/legacy.config.template
@@ -1,28 +1,8 @@
-## Import directory archives from disk, if available
-#ImportDirectoryArchives 0
-#
 ## Relative paths to directories to import directory archives from
 #DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/consensuses/
 #DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/server-descriptors/
 #DirectoryArchivesDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/relay-descriptors/extra-infos/
 #
-## Keep a history of imported directory archive files to know which files
-## have been imported before. This history can be useful when importing
-## from a changing source to avoid importing descriptors over and over
-## again, but it can be confusing to users who don't know about it.
-#KeepDirectoryArchiveImportHistory 0
-#
-## Write relay descriptors to the database
-#WriteRelayDescriptorDatabase 0
-#
 ## JDBC string for relay descriptor database
 #RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=metrics&password=password
 #
-## Write relay descriptors to raw text files for importing them into the
-## database using PostgreSQL's \copy command
-#WriteRelayDescriptorsRawFiles 0
-#
-## Relative path to directory to write raw text files; note that existing
-## files will be overwritten!
-#RelayDescriptorRawFilesDirectory pg-import/
-#

    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

[tor-commits] [metrics-web/master] Remove long unused code from legacy module.