[tor-commits] [metrics-lib/master] Add log message reporting progress reading tarballs.

[karsten at torproject.org]

commit 9f2db9a194d5e290b456765f83cc6eeaf745e619
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Sun Dec 3 21:11:07 2017 +0100

    Add log message reporting progress reading tarballs.
    
    Implements #22990.
---
 CHANGELOG.md                                       |  3 ++
 .../descriptor/impl/DescriptorReaderImpl.java      | 37 +++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5939e38..91f00b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@
    - Add new descriptor type WebServerAccessLog to parse web server
      access logs.
 
+ * Minor changes
+   - Add log message reporting progress reading tarballs.
+
 
 # Changes in version 2.1.1 - 2017-10-09
 
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
index 4da474e..a8a7790 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
@@ -23,6 +23,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -34,6 +35,9 @@ public class DescriptorReaderImpl implements DescriptorReader {
 
   private static Logger log = LoggerFactory.getLogger(
       DescriptorReaderImpl.class);
+
+  private static Logger statisticsLog = LoggerFactory.getLogger("statistics");
+
   private boolean hasStartedReading = false;
 
   private File manualSaveHistoryFile;
@@ -125,6 +129,8 @@ public class DescriptorReaderImpl implements DescriptorReader {
 
     private File manualSaveHistoryFile;
 
+    private List<File> tarballs = new ArrayList<>();
+
     private SortedMap<String, Long> excludedFilesBefore = new TreeMap<>();
 
     private SortedMap<String, Long> excludedFilesAfter = new TreeMap<>();
@@ -153,6 +159,7 @@ public class DescriptorReaderImpl implements DescriptorReader {
         this.readOldHistory(this.autoSaveHistoryFile);
         this.readOldHistory(this.manualSaveHistoryFile);
         this.readDescriptorFiles();
+        this.readTarballs();
         this.hasFinishedReading = true;
       } catch (Throwable t) {
         log.error("Bug: uncaught exception or error while "
@@ -242,7 +249,8 @@ public class DescriptorReaderImpl implements DescriptorReader {
             } else if (file.getName().endsWith(".tar")
                 || file.getName().endsWith(".tar.bz2")
                 || file.getName().endsWith(".tar.xz")) {
-              this.readTarball(file);
+              tarballs.add(file);
+              continue;
             } else {
               this.readDescriptorFile(file);
             }
@@ -254,6 +262,33 @@ public class DescriptorReaderImpl implements DescriptorReader {
       }
     }
 
+    private void readTarballs() {
+      if (this.tarballs.isEmpty()) {
+        return;
+      }
+      long total = 0L;
+      for (File tarball : this.tarballs) {
+        total += tarball.length();
+      }
+      long progress = 0L;
+      for (File tarball : this.tarballs) {
+        try {
+          this.readTarball(tarball);
+          this.parsedFilesAfter.put(tarball.getAbsolutePath(),
+              tarball.lastModified());
+        } catch (IOException e) {
+          log.warn("Unable to read tarball {}.", tarball, e);
+        }
+        long previousPercentDone = 100L * progress / total;
+        progress += tarball.length();
+        long percentDone = 100L * progress / total;
+        if (percentDone > previousPercentDone) {
+          statisticsLog.info("Finished reading {}% of tarball bytes.",
+              percentDone);
+        }
+      }
+    }
+
     private void readTarball(File file) throws IOException {
       FileInputStream in = new FileInputStream(file);
       if (in.available() <= 0) {