commit 1f438871787eef7554e2e518d968611c3940e9ab Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Aug 30 10:51:45 2018 +0200
Properly clean up web server logs in recent/.
Fixes #27390. --- CHANGELOG.md | 4 ++++ .../metrics/collector/persist/PersistenceUtils.java | 13 ++++++++++++- .../metrics/collector/webstats/SanitizeWeblogs.java | 5 ++++- 3 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md index d98520c..fc24633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes in version 1.?.? - 2018-0?-??
+ * Medium changes + - Properly clean up sanitized web server logs in the recent/ + directory when they turn older than three days. + * Minor changes - Once more, fix the bug in the tarball-creation script where tarballs are not compressed in a run following an aborted run. diff --git a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java index 7d30cb1..78eac72 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java +++ b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java @@ -82,12 +82,23 @@ public class PersistenceUtils {
/** Move temporary files to their final location. */ public static void cleanDirectory(Path pathToClean) throws IOException { + PersistenceUtils.cleanDirectory(pathToClean, -1L); + } + + /** Clean up the given directory by deleting files that are older than the + * given cut-off timestamp, and by moving temporary files to their final + * location. */ + public static void cleanDirectory(Path pathToClean, long cutOffMillis) + throws IOException { SimpleFileVisitor<Path> sfv = new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String tempName = file.toString(); - if (tempName.endsWith(TEMPFIX)) { + if (cutOffMillis >= 0L + && attrs.lastModifiedTime().toMillis() < cutOffMillis) { + file.toFile().delete(); + } else if (tempName.endsWith(TEMPFIX)) { Path outputPath = Paths .get(tempName.substring(0, tempName.length() - TEMPFIX.length())); Files.deleteIfExists(outputPath); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 0e83598..027cfde 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -97,7 +97,10 @@ public class SanitizeWeblogs extends CollecTorMain { if (sources.contains(SourceType.Local)) { log.info("Processing logs using batch value {}.", BATCH); findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins)); - PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath)); + long cutOffMillis = System.currentTimeMillis() + - 3L * 24L * 60L * 60L * 1000L; + PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath), + cutOffMillis); } } catch (Exception e) { log.error("Cannot sanitize web-logs: {}", e.getMessage(), e);