[tor-commits] [collector/master] Refactor CachedRelayDescriptorReader.

karsten at torproject.org karsten at torproject.org
Mon Aug 27 12:34:23 UTC 2018


commit 698d15d94f52fe1192fd2afb82439abace27066c
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Mon Aug 20 21:17:03 2018 +0200

    Refactor CachedRelayDescriptorReader.
---
 .../collector/relaydescs/ArchiveWriter.java        |  3 +-
 .../relaydescs/CachedRelayDescriptorReader.java    | 96 ++++++++++++++--------
 2 files changed, 62 insertions(+), 37 deletions(-)

diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 8679439..3429a0a 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -161,7 +161,8 @@ public class ArchiveWriter extends CollecTorMain {
     }
     if (sources.contains(SourceType.Cache)) {
       new CachedRelayDescriptorReader(rdp,
-          config.getStringArray(Key.RelayCacheOrigins), statsDirectory);
+          config.getStringArray(Key.RelayCacheOrigins), statsDirectory)
+          .readDescriptors();
       this.intermediateStats("importing relay descriptors from local "
           + "Tor data directories");
     }
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
index f64d019..4f0d012 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -37,32 +37,52 @@ public class CachedRelayDescriptorReader {
   private static final Logger logger = LoggerFactory.getLogger(
       CachedRelayDescriptorReader.class);
 
-  /** Reads cached-descriptor files from one or more directories and
-   * passes them to the given descriptor parser. */
-  public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
-      String[] inputDirectories, File statsDirectory) {
+  private RelayDescriptorParser rdp;
+
+  private String[] inputDirectories;
+
+  private File importHistoryFile;
 
+  private StringBuilder dumpStats;
+
+  private Set<String> lastImportHistory = new HashSet<>();
+
+  private Set<String> currentImportHistory = new HashSet<>();
+
+  /** Initializes this reader but without starting to read yet. */
+  CachedRelayDescriptorReader(RelayDescriptorParser rdp,
+      String[] inputDirectories, File statsDirectory) {
     if (rdp == null || inputDirectories == null
         || inputDirectories.length == 0 || statsDirectory == null) {
       throw new IllegalArgumentException();
     }
+    this.rdp = rdp;
+    this.inputDirectories = inputDirectories;
+    this.importHistoryFile = new File(statsDirectory,
+        "cacheddesc-import-history");
 
-    StringBuilder dumpStats = new StringBuilder("Finished importing "
+    this.dumpStats = new StringBuilder("Finished importing "
         + "relay descriptors from local Tor data directories:");
+  }
 
-    /* Read import history containing SHA-1 digests of previously parsed
-     * statuses and descriptors, so that we can skip them in this run. */
-    Set<String> lastImportHistory = new HashSet<>();
-    Set<String> currentImportHistory = new HashSet<>();
-    File importHistoryFile = new File(statsDirectory,
-        "cacheddesc-import-history");
+  /** Reads cached-descriptor files from one or more directories and
+   * passes them to the given descriptor parser. */
+  public void readDescriptors() {
+    this.readHistoryFile();
+    this.readDescriptorFiles();
+    this.writeHistoryFile();
+  }
+
+  /** Read import history containing SHA-1 digests of previously parsed
+   * statuses and descriptors, so that we can skip them in this run. */
+  private void readHistoryFile() {
     if (importHistoryFile.exists()) {
       try {
         BufferedReader br = new BufferedReader(new FileReader(
             importHistoryFile));
         String line;
         while ((line = br.readLine()) != null) {
-          lastImportHistory.add(line);
+          this.lastImportHistory.add(line);
         }
         br.close();
       } catch (IOException e) {
@@ -70,9 +90,11 @@ public class CachedRelayDescriptorReader {
             + importHistoryFile.getAbsolutePath() + ".", e);
       }
     }
+  }
 
-    /* Read cached descriptors directories. */
-    for (String inputDirectory : inputDirectories) {
+  /** Read cached descriptors directories. */
+  private void readDescriptorFiles() {
+    for (String inputDirectory : this.inputDirectories) {
       File cachedDescDir = new File(inputDirectory);
       if (!cachedDescDir.exists()) {
         logger.warn("Directory " + cachedDescDir.getAbsolutePath()
@@ -113,7 +135,7 @@ public class CachedRelayDescriptorReader {
             String line;
             while ((line = br.readLine()) != null) {
               if (line.startsWith("valid-after ")) {
-                dumpStats.append("\n").append(f.getName()).append(": ")
+                this.dumpStats.append("\n").append(f.getName()).append(": ")
                     .append(line.substring("valid-after ".length()));
                 SimpleDateFormat dateTimeFormat =
                     new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@@ -125,7 +147,7 @@ public class CachedRelayDescriptorReader {
                       + cachedDescDir.getAbsolutePath() + " are stale. "
                       + "The valid-after line in cached-consensus is '"
                       + line + "'.");
-                  dumpStats.append(" (stale!)");
+                  this.dumpStats.append(" (stale!)");
                 }
                 break;
               }
@@ -136,13 +158,13 @@ public class CachedRelayDescriptorReader {
              * (but regardless of whether it's stale or not). */
             String digest = Hex.encodeHexString(DigestUtils.sha1(
                 allData));
-            if (!lastImportHistory.contains(digest)
-                && !currentImportHistory.contains(digest)) {
-              rdp.parse(allData);
+            if (!this.lastImportHistory.contains(digest)
+                && !this.currentImportHistory.contains(digest)) {
+              this.rdp.parse(allData);
             } else {
-              dumpStats.append(" (skipped)");
+              this.dumpStats.append(" (skipped)");
             }
-            currentImportHistory.add(digest);
+            this.currentImportHistory.add(digest);
           } else if (f.getName().equals("v3-status-votes")) {
             int parsedNum = 0;
             int skippedNum = 0;
@@ -161,18 +183,18 @@ public class CachedRelayDescriptorReader {
                     next - start);
                 String digest = Hex.encodeHexString(DigestUtils.sha1(
                     rawNetworkStatusBytes));
-                if (!lastImportHistory.contains(digest)
-                    && !currentImportHistory.contains(digest)) {
-                  rdp.parse(rawNetworkStatusBytes);
+                if (!this.lastImportHistory.contains(digest)
+                    && !this.currentImportHistory.contains(digest)) {
+                  this.rdp.parse(rawNetworkStatusBytes);
                   parsedNum++;
                 } else {
                   skippedNum++;
                 }
-                currentImportHistory.add(digest);
+                this.currentImportHistory.add(digest);
               }
               start = next;
             }
-            dumpStats.append("\n").append(f.getName()).append(": parsed ")
+            this.dumpStats.append("\n").append(f.getName()).append(": parsed ")
                 .append(parsedNum).append(", skipped ").append(skippedNum)
                 .append(" votes");
           } else if (f.getName().startsWith("cached-descriptors")
@@ -183,7 +205,7 @@ public class CachedRelayDescriptorReader {
             int end = -1;
             String startToken =
                 f.getName().startsWith("cached-descriptors")
-                ? "router " : "extra-info ";
+                    ? "router " : "extra-info ";
             String sigToken = "\nrouter-signature\n";
             String endToken = "\n-----END SIGNATURE-----\n";
             int parsedNum = 0;
@@ -207,16 +229,16 @@ public class CachedRelayDescriptorReader {
               System.arraycopy(allData, start, descBytes, 0, end - start);
               String digest = Hex.encodeHexString(DigestUtils.sha1(
                   descBytes));
-              if (!lastImportHistory.contains(digest)
-                  && !currentImportHistory.contains(digest)) {
-                rdp.parse(descBytes);
+              if (!this.lastImportHistory.contains(digest)
+                  && !this.currentImportHistory.contains(digest)) {
+                this.rdp.parse(descBytes);
                 parsedNum++;
               } else {
                 skippedNum++;
               }
-              currentImportHistory.add(digest);
+              this.currentImportHistory.add(digest);
             }
-            dumpStats.append("\n").append(f.getName()).append(": parsed ")
+            this.dumpStats.append("\n").append(f.getName()).append(": parsed ")
                 .append(parsedNum).append(", skipped ").append(skippedNum)
                 .append(" ").append(f.getName().startsWith("cached-descriptors")
                 ? "server" : "extra-info").append(" descriptors");
@@ -229,19 +251,21 @@ public class CachedRelayDescriptorReader {
       logger.debug("Finished reading "
           + cachedDescDir.getAbsolutePath() + " directory.");
     }
+  }
 
-    /* Write import history containing SHA-1 digests to disk. */
+  /** Write import history containing SHA-1 digests to disk. */
+  private void writeHistoryFile() {
     try {
-      importHistoryFile.getParentFile().mkdirs();
+      this.importHistoryFile.getParentFile().mkdirs();
       BufferedWriter bw = new BufferedWriter(new FileWriter(
-          importHistoryFile));
+          this.importHistoryFile));
       for (String digest : currentImportHistory) {
         bw.write(digest + "\n");
       }
       bw.close();
     } catch (IOException e) {
       logger.warn("Could not write import history to "
-           + importHistoryFile.getAbsolutePath() + ".", e);
+           + this.importHistoryFile.getAbsolutePath() + ".", e);
     }
 
     logger.info(dumpStats.toString());





More information about the tor-commits mailing list