commit f2789a8671cbbb97c85be0ecfec4108ae66e93b8 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Jan 18 15:22:25 2012 +0100

Move files to out/ when they're at least 4 days old.

We're not moving files to the out/ directory for further processing if they're less than 4 days old. The idea is that these files can still change when new logs arrive, and once they're in out/, they shouldn't change anymore. Instead, we're moving them to state/full/ until they're at least 4 days old.

However, we forgot that last step. Implement it now. --- src/org/torproject/webstats/Main.java | 50 ++++++++++++++++++++++++++++++-- 1 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/org/torproject/webstats/Main.java b/src/org/torproject/webstats/Main.java index 1812567..7f1afe1 100644 --- a/src/org/torproject/webstats/Main.java +++ b/src/org/torproject/webstats/Main.java @@ -73,10 +73,13 @@ import org.apache.commons.compress.compressors.gzip.*; * days old, move the file to state/full/. * c. If b. does not apply, append a line to out-history.new and move * the file to out/. - * 7. Rename state/out-history to state/out-history.old and rename + * 7. For each file in state/full/, check whether the sanitized log is at + * least four (4) days old and not contained in state/out-history. If + * so, append a line to out-history.new and move the file to out/. + * 8. Rename state/out-history to state/out-history.old and rename * state/out-history.new to state/out-history. Delete * state/out-history.old. - * 8. Delete state/lock and exit. + * 9. Delete state/lock and exit. * * If the program is interrupted and leaves a lock file in state/lock, it * requires an operator to fix the state/ directory and make it work @@ -130,8 +133,9 @@ public class Main { for (String outputFileName : updatedOutputFiles) { /* Step 6 */ moveOutputFile(outputFileName); } - overwriteOutHistoryFile(); /* Step 7 */ - deleteLockFile(); /* Step 8 */ + moveFullFilesToOut(); /* Step 7 */ + overwriteOutHistoryFile(); /* Step 8 */ + deleteLockFile(); /* Step 9 */ }

/* Define file and directory names. */ @@ -150,6 +154,7 @@ public class Main { private static File stateOutHistoryOldFile = new File("state/out-history.old"); private static File stateDiffDirectory = new File("state/diff"); + private static String stateFullDirectoryString = "state/full"; private static File stateFullDirectory = new File("state/full"); private static File stateTempDirectory = new File("state/temp");

@@ -489,6 +494,43 @@ public class Main { stateTempFile.renameTo(outFile); } } + private static void moveFullFilesToOut() { + Stack<String> fileNames = new Stack<String>(); + fileNames.add(stateFullDirectoryString); + while (!fileNames.isEmpty()) { + String fileName = fileNames.pop(); + File fileOrDirectory = new File(fileName); + if (fileOrDirectory.isDirectory()) { + for (File file : fileOrDirectory.listFiles()) { + fileNames.add(fileName + "/" + file.getName()); + } + } else { + String outputFileName = fileName.substring( + (stateFullDirectoryString + "/").length()); + File outFile = new File(outDirectory, outputFileName); + File stateFullFile = new File(stateFullDirectory, outputFileName); + long ageInDays = -1L; + try { + ageInDays = (now + - outputFileFormat.parse(outputFileName.substring(0, + outputFileName.lastIndexOf("/") + 1)).getTime()) + / (24L * 60L * 60L * 1000L); + } catch (ParseException e) { + e.printStackTrace(); + System.err.println("Could not parse timestamp from '" + + outputFileName + "'. Exiting."); + System.exit(1); + } + if (!outHistoryFiles.contains(outFile.getAbsolutePath()) && + ageInDays >= 4L) { + outFile.getParentFile().mkdirs(); + String line = outFile.getAbsolutePath(); + appendToHistoryFile(stateOutHistoryNewFile, line); + stateFullFile.renameTo(outFile); + } + } + } + } private static void overwriteOutHistoryFile() { stateOutHistoryFile.renameTo(stateOutHistoryOldFile); stateOutHistoryNewFile.renameTo(stateOutHistoryFile);