commit 287216d2b141828f6cf108c91f94733a5d2fc7f2 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Nov 7 19:39:12 2012 -0500

Rearrange steps to make it easier to recover from failures. --- src/org/torproject/webstats/Main.java | 55 ++++++++++++++++++++------------ 1 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/src/org/torproject/webstats/Main.java b/src/org/torproject/webstats/Main.java index 4bb035f..378093e 100644 --- a/src/org/torproject/webstats/Main.java +++ b/src/org/torproject/webstats/Main.java @@ -50,11 +50,13 @@ import org.apache.commons.compress.compressors.gzip.*; * - state/diff/ contains new parts for files in the out/ directory which * have been deleted. * - * The steps taken by this program are as follows: + * The phases and steps taken by this program are as follows: + * + * Phase I: Read files from in/, sanitize them, and write them to state/. * 1. Check that state/lock does not exists, or exit immediately. Add a * new state/lock file. - * 2. Read the contents from state/in-history and state/out-history and - * the directory listings of in/ to memory. + * 2. Read the contents from state/in-history and the directory listing + * of in/ to memory. * 3. For each file in in/: * a. Append the file name to state/in-history.new if it was not * contained in state/in-history. If it was contained, skip the @@ -73,7 +75,10 @@ import org.apache.commons.compress.compressors.gzip.*; * 4. Rename state/in-history to state/in-history.old and rename * state/in-history.new to state/in-history. Delete * state/in-history.old. - * 5. For each file in state/temp/: + * + * Phase II: Move files that won't change anymore from state/ to out/. + * 5. Read the contents from state/out-history to memory. + * 6. For each file in state/temp/: * a. Check if there's a corresponding line in state/out-history. If * so, check whether there is a file in state/full/ or out/. If * so, move the file to state/full/. Otherwise move the file to @@ -83,13 +88,13 @@ import org.apache.commons.compress.compressors.gzip.*; * days old, move the file to state/full/. * c. If b. does not apply, append a line to out-history.new and move * the file to out/. - * 6. For each file in state/full/, check whether the sanitized log is at + * 7. For each file in state/full/, check whether the sanitized log is at * least four (4) days old and not contained in state/out-history. If * so, append a line to out-history.new and move the file to out/. - * 7. Rename state/out-history to state/out-history.old and rename + * 8. Rename state/out-history to state/out-history.old and rename * state/out-history.new to state/out-history. Delete * state/out-history.old. - * 8. Delete state/lock and exit. + * 9. Delete state/lock and exit. * * If the program is interrupted and leaves a lock file in state/lock, it * requires an operator to fix the state/ directory and make it work @@ -98,16 +103,17 @@ import org.apache.commons.compress.compressors.gzip.*; * happen. It may make sense to try a solution in a test environment * first: * A. The file state/in-history.new does not exist and there are no files - * in state/temp/. The process died before step 3. Delete state/lock - * and re-run the program. + * in state/temp/. The process died before step 3, that is before + * actually doing anything of phase I. Delete state/lock and re-run + * the program. * B. The file state/in-history.new exists and there are files in - * state/temp/. The process died during steps 3 or 4. Delete all - * files in state/temp/. If state/in-history does not exist but - * state/in-history.old does exist, rename the latter to the former. - * Delete state/lock and re-run the program. + * state/temp/. The process died during steps 3 or 4, that is, during + * phase I. Delete all files in state/temp/. If state/in-history + * does not exist but state/in-history.old does exist, rename the + * latter to the former. Delete state/lock and re-run the program. * C. The file state/in-history.new does not exist, but there are files - * in state/temp/. The process died after step 4. Run the steps 5 to - * 8 manually. Then re-run the program. + * in state/temp/. The process died after step 4, that is during + * phase II. Run the steps 5 to 9 manually. Then re-run the program. * * Whenever logs are parsed that are 4 days old or older, there may * already be output files in out/ that cannot be modified anymore. The @@ -128,8 +134,10 @@ public class Main {

/* Run the steps described above. */ public static void main(String[] args) { + + /* Phase I */ checkAndCreateLockFile(); /* Step 1 */ - readHistoryFiles(); /* Step 2 */ + readInHistoryFile(); /* Step 2 */ readInDirectoryListing(); for (File inFile : inFiles) { /* Step 3 */ appendToInHistoryIfNotContained(inFile); @@ -139,12 +147,15 @@ public class Main { sanitizeInFile(inFile); } overwriteInHistoryFile(); /* Step 4 */ - for (String outputFileName : updatedOutputFiles) { /* Step 5 */ + + /* Phase II */ + readOutHistoryFile(); /* Step 5 */ + for (String outputFileName : updatedOutputFiles) { /* Step 6 */ moveOutputFile(outputFileName); } - moveFullFilesToOut(); /* Step 6 */ - overwriteOutHistoryFile(); /* Step 7 */ - deleteLockFile(); /* Step 8 */ + moveFullFilesToOut(); /* Step 7 */ + overwriteOutHistoryFile(); /* Step 8 */ + deleteLockFile(); /* Step 9 */ }

/* Define file and directory names. */ @@ -209,10 +220,12 @@ public class Main { System.exit(1); } } - private static void readHistoryFiles() { + private static void readInHistoryFile() { inHistoryFiles = readAndCopyHistoryFile(stateInHistoryFile, stateInHistoryNewFile); inHistoryNewFiles = new HashSet<String>(inHistoryFiles); + } + private static void readOutHistoryFile() { outHistoryFiles = readAndCopyHistoryFile(stateOutHistoryFile, stateOutHistoryNewFile); }