[tor-commits] [webstats/master] Move files to out/ when they're at least 4 days old.

[runa at torproject.org]

commit f2789a8671cbbb97c85be0ecfec4108ae66e93b8
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Wed Jan 18 15:22:25 2012 +0100

    Move files to out/ when they're at least 4 days old.
    
    We're not moving files to the out/ directory for further processing if
    they're less than 4 days old.  The idea is that these files can still
    change when new logs arrive, and once they're in out/, they shouldn't
    change anymore.  Instead, we're moving them to state/full/ until they're
    at least 4 days old.
    
    However, we forgot that last step.  Implement it now.
---
 src/org/torproject/webstats/Main.java |   50 ++++++++++++++++++++++++++++++--
 1 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/org/torproject/webstats/Main.java b/src/org/torproject/webstats/Main.java
index 1812567..7f1afe1 100644
--- a/src/org/torproject/webstats/Main.java
+++ b/src/org/torproject/webstats/Main.java
@@ -73,10 +73,13 @@ import org.apache.commons.compress.compressors.gzip.*;
  *        days old, move the file to state/full/.
  *     c. If b. does not apply, append a line to out-history.new and move
  *        the file to out/.
- *  7. Rename state/out-history to state/out-history.old and rename
+ *  7. For each file in state/full/, check whether the sanitized log is at
+ *     least four (4) days old and not contained in state/out-history.  If
+ *     so, append a line to out-history.new and move the file to out/.
+ *  8. Rename state/out-history to state/out-history.old and rename
  *     state/out-history.new to state/out-history.  Delete
  *     state/out-history.old.
- *  8. Delete state/lock and exit.
+ *  9. Delete state/lock and exit.
  *
  * If the program is interrupted and leaves a lock file in state/lock, it
  * requires an operator to fix the state/ directory and make it work
@@ -130,8 +133,9 @@ public class Main {
     for (String outputFileName : updatedOutputFiles) { /* Step 6 */
       moveOutputFile(outputFileName);
     }
-    overwriteOutHistoryFile(); /* Step 7 */
-    deleteLockFile(); /* Step 8 */
+    moveFullFilesToOut(); /* Step 7 */
+    overwriteOutHistoryFile(); /* Step 8 */
+    deleteLockFile(); /* Step 9 */
   }
 
   /* Define file and directory names. */
@@ -150,6 +154,7 @@ public class Main {
   private static File stateOutHistoryOldFile =
       new File("state/out-history.old");
   private static File stateDiffDirectory = new File("state/diff");
+  private static String stateFullDirectoryString = "state/full";
   private static File stateFullDirectory = new File("state/full");
   private static File stateTempDirectory = new File("state/temp");
 
@@ -489,6 +494,43 @@ public class Main {
       stateTempFile.renameTo(outFile);
     }
   }
+  private static void moveFullFilesToOut() {
+    Stack<String> fileNames = new Stack<String>();
+    fileNames.add(stateFullDirectoryString);
+    while (!fileNames.isEmpty()) {
+      String fileName = fileNames.pop();
+      File fileOrDirectory = new File(fileName);
+      if (fileOrDirectory.isDirectory()) {
+        for (File file : fileOrDirectory.listFiles()) {
+          fileNames.add(fileName + "/" + file.getName());
+        }
+      } else {
+        String outputFileName = fileName.substring(
+            (stateFullDirectoryString + "/").length());
+        File outFile = new File(outDirectory, outputFileName);
+        File stateFullFile = new File(stateFullDirectory, outputFileName);
+        long ageInDays = -1L;
+        try {
+          ageInDays = (now
+              - outputFileFormat.parse(outputFileName.substring(0,
+              outputFileName.lastIndexOf("/") + 1)).getTime())
+              / (24L * 60L * 60L * 1000L);
+        } catch (ParseException e) {
+          e.printStackTrace();
+          System.err.println("Could not parse timestamp from '"
+              + outputFileName + "'.  Exiting.");
+          System.exit(1);
+        }
+        if (!outHistoryFiles.contains(outFile.getAbsolutePath()) &&
+            ageInDays >= 4L) {
+          outFile.getParentFile().mkdirs();
+          String line = outFile.getAbsolutePath();
+          appendToHistoryFile(stateOutHistoryNewFile, line);
+          stateFullFile.renameTo(outFile);
+        }
+      }
+    }
+  }
   private static void overwriteOutHistoryFile() {
     stateOutHistoryFile.renameTo(stateOutHistoryOldFile);
     stateOutHistoryNewFile.renameTo(stateOutHistoryFile);