commit 9f2db9a194d5e290b456765f83cc6eeaf745e619 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sun Dec 3 21:11:07 2017 +0100

Add log message reporting progress reading tarballs.

Implements #22990. --- CHANGELOG.md | 3 ++ .../descriptor/impl/DescriptorReaderImpl.java | 37 +++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index 5939e38..91f00b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ - Add new descriptor type WebServerAccessLog to parse web server access logs.

+ * Minor changes + - Add log message reporting progress reading tarballs. +

# Changes in version 2.1.1 - 2017-10-09

diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java index 4da474e..a8a7790 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -23,6 +23,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -34,6 +35,9 @@ public class DescriptorReaderImpl implements DescriptorReader {

private static Logger log = LoggerFactory.getLogger( DescriptorReaderImpl.class); + + private static Logger statisticsLog = LoggerFactory.getLogger("statistics"); + private boolean hasStartedReading = false;

private File manualSaveHistoryFile; @@ -125,6 +129,8 @@ public class DescriptorReaderImpl implements DescriptorReader {

private File manualSaveHistoryFile;

+ private List<File> tarballs = new ArrayList<>(); + private SortedMap<String, Long> excludedFilesBefore = new TreeMap<>();

private SortedMap<String, Long> excludedFilesAfter = new TreeMap<>(); @@ -153,6 +159,7 @@ public class DescriptorReaderImpl implements DescriptorReader { this.readOldHistory(this.autoSaveHistoryFile); this.readOldHistory(this.manualSaveHistoryFile); this.readDescriptorFiles(); + this.readTarballs(); this.hasFinishedReading = true; } catch (Throwable t) { log.error("Bug: uncaught exception or error while " @@ -242,7 +249,8 @@ public class DescriptorReaderImpl implements DescriptorReader { } else if (file.getName().endsWith(".tar") || file.getName().endsWith(".tar.bz2") || file.getName().endsWith(".tar.xz")) { - this.readTarball(file); + tarballs.add(file); + continue; } else { this.readDescriptorFile(file); } @@ -254,6 +262,33 @@ public class DescriptorReaderImpl implements DescriptorReader { } }

+ private void readTarballs() { + if (this.tarballs.isEmpty()) { + return; + } + long total = 0L; + for (File tarball : this.tarballs) { + total += tarball.length(); + } + long progress = 0L; + for (File tarball : this.tarballs) { + try { + this.readTarball(tarball); + this.parsedFilesAfter.put(tarball.getAbsolutePath(), + tarball.lastModified()); + } catch (IOException e) { + log.warn("Unable to read tarball {}.", tarball, e); + } + long previousPercentDone = 100L * progress / total; + progress += tarball.length(); + long percentDone = 100L * progress / total; + if (percentDone > previousPercentDone) { + statisticsLog.info("Finished reading {}% of tarball bytes.", + percentDone); + } + } + } + private void readTarball(File file) throws IOException { FileInputStream in = new FileInputStream(file); if (in.available() <= 0) {