[tor-commits] [collector/master] Properly clean up web server logs in recent/.

[karsten at torproject.org]

commit 1f438871787eef7554e2e518d968611c3940e9ab
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Thu Aug 30 10:51:45 2018 +0200

    Properly clean up web server logs in recent/.
    
    Fixes #27390.
---
 CHANGELOG.md                                                |  4 ++++
 .../metrics/collector/persist/PersistenceUtils.java         | 13 ++++++++++++-
 .../metrics/collector/webstats/SanitizeWeblogs.java         |  5 ++++-
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d98520c..fc24633 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changes in version 1.?.? - 2018-0?-??
 
+ * Medium changes
+   - Properly clean up sanitized web server logs in the recent/
+     directory when they turn older than three days.
+
  * Minor changes
    - Once more, fix the bug in the tarball-creation script where
      tarballs are not compressed in a run following an aborted run.
diff --git a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
index 7d30cb1..78eac72 100644
--- a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
+++ b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
@@ -82,12 +82,23 @@ public class PersistenceUtils {
 
   /** Move temporary files to their final location. */
   public static void cleanDirectory(Path pathToClean) throws IOException {
+    PersistenceUtils.cleanDirectory(pathToClean, -1L);
+  }
+
+  /** Clean up the given directory by deleting files that are older than the
+   * given cut-off timestamp, and by moving temporary files to their final
+   * location. */
+  public static void cleanDirectory(Path pathToClean, long cutOffMillis)
+      throws IOException {
     SimpleFileVisitor<Path> sfv = new SimpleFileVisitor<Path>() {
       @Override
       public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
           throws IOException {
         String tempName = file.toString();
-        if (tempName.endsWith(TEMPFIX)) {
+        if (cutOffMillis >= 0L
+            && attrs.lastModifiedTime().toMillis() < cutOffMillis) {
+          file.toFile().delete();
+        } else if (tempName.endsWith(TEMPFIX)) {
           Path outputPath = Paths
               .get(tempName.substring(0, tempName.length() - TEMPFIX.length()));
           Files.deleteIfExists(outputPath);
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
index 0e83598..027cfde 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
@@ -97,7 +97,10 @@ public class SanitizeWeblogs extends CollecTorMain {
       if (sources.contains(SourceType.Local)) {
         log.info("Processing logs using batch value {}.", BATCH);
         findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins));
-        PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath));
+        long cutOffMillis = System.currentTimeMillis()
+            - 3L * 24L * 60L * 60L * 1000L;
+        PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath),
+            cutOffMillis);
       }
     } catch (Exception e) {
       log.error("Cannot sanitize web-logs: {}", e.getMessage(), e);