commit edddf0983d668ecd8c27700a83cb7e4d64255c07
Author: Sebastian Hahn <sebastian(a)torproject.org>
Date: Fri Nov 13 06:41:57 2015 +0100
Start script to extract logs
---
bin/receive-log.sh | 47 ---
bin/send-logs.sh | 73 -----
build.xml | 36 --
src/org/torproject/webstats/Main.java | 595 ----------------------------------
src/sanitize.py | 82 +++++
src/treat_new_logs.sh | 45 +++
6 files changed, 127 insertions(+), 751 deletions(-)
diff --git a/bin/receive-log.sh b/bin/receive-log.sh
deleted file mode 100755
index 6f98eaa..0000000
--- a/bin/receive-log.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-
-# Copyright 2011 The Tor Project
-# See LICENSE for licensing information
-#
-# Read a binary file from stdin and store it to disk.
-
-set -e
-set -u
-
-# Read remote host name from local command-line argument.
-rhost="$1"
-if [ -z "$rhost" ]; then
- echo "Missing or illegal host name in authorized_keys."
- exit 1
-fi
-
-# Read remote log file name from SSH command-line arguments.
-rfile="${SSH_ORIGINAL_COMMAND:-}"
-if ! [[ "$rfile" =~ ^[a-zA-Z0-9.-]+$ ]]; then
- echo "Missing or illegal file name in SSH command."
- exit 1
-fi
-
-# Create directories for this remote host if it doesn't exist.
-ldir="in/$rhost"
-if ! [ -d "in" ]; then
- mkdir "in"
-fi
-if ! [ -d "$ldir" ]; then
- mkdir "$ldir"
-fi
-
-# Make sure the file doesn't exist yet.
-lfile="$ldir/$rfile"
-if [[ -f "$lfile" ]]; then
- echo "File already exists."
- exit 1
-fi
-
-# Write file from stdin to temporary file.
-tfile="$ldir/.$rfile"
-cat > "$tfile"
-
-# Move file from temp/ to in/ directory.
-mv "$tfile" "$lfile"
-
diff --git a/bin/send-logs.sh b/bin/send-logs.sh
deleted file mode 100755
index 77e7442..0000000
--- a/bin/send-logs.sh
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/bin/bash
-
-# Copyright 2011 The Tor Project
-# See LICENSE for licensing information
-#
-# Send new gzip'ed Apache web log files via ssh to a remote server.
-
-set -e
-set -u
-
-usage() {
- echo "Usage: $0 logs-dir ssh-string state-dir"
- exit 1
-}
-
-# Check command-line arguments.
-if [ "$#" != 3 ]; then
- usage
-fi
-ldir="$1"
-sshstring="$2"
-tdir="$3"
-if ! [ -d "$ldir" ]; then
- echo "Directory '$1' does not exist."
- usage
-fi
-
-# Create the local state directory that has empty files for every log file
-# that we sent to the remote server.
-if ! [ -d "$tdir" ]; then
- mkdir "$tdir"
-fi
-
-# Iterate over files with file names containing access.log and ending in
-# .gz and send the ones we haven't sent before to the remote server.
-for i in $(find "$ldir" -maxdepth 1 -type f -name '*access.log*.gz'); do
-
- # Check that the file exists. This works around issues like filenames
- # containing spaces.
- if ! [ -e $i ]; then
- echo "File '$i' does not exist."
- continue
- fi
-
- # Extract the file name part to pass it as SSH parameter.
- fname="$(basename $i)"
-
- # Warn if file names contain illegal characters that the receiver would
- # not accept.
- if ! [[ $fname =~ ^[a-zA-Z0-9.-]+$ ]]; then
- echo "Illegal file name '$fname'."
- continue
- fi
-
- # Copy the file content to the remote server if we haven't sent it
- # before.
- tfile="$tdir/$fname"
- if ! [ -f "$tfile" ]; then
- ssh -o PreferredAuthentications=publickey $sshstring "$fname" < "$i"
- fi
-
- # Add a state file to note that we don't attempt to send this file in
- # the next execution.
- touch "$tfile"
-
-done
-
-# Delete all state files for which there are no log files anymore.
-for i in $(find "$tdir" -type f); do
- fname="$(basename $i)"
- [ -e "$ldir/$fname" ] || rm -f "$tdir/$fname"
-done
-
diff --git a/build.xml b/build.xml
deleted file mode 100644
index 7f0b430..0000000
--- a/build.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<project default="run" name="webstats" basedir=".">
- <property name="sources" value="src"/>
- <property name="classes" value="classes"/>
- <property name="libs" value="lib"/>
- <path id="classpath">
- <pathelement path="${classes}"/>
- <pathelement location="${libs}/commons-compress-1.0.jar"/>
- </path>
- <target name="init">
- <mkdir dir="${classes}"/>
- </target>
- <target name="compile"
- depends="init">
- <javac destdir="${classes}"
- srcdir="${sources}"
- source="1.5"
- target="1.5"
- debug="true"
- deprecation="true"
- optimize="false"
- failonerror="true"
- includeantruntime="false">
- <classpath>
- <fileset dir="${libs}"/>
- </classpath>
- </javac>
- </target>
- <target name="run" depends="compile">
- <java fork="true"
- maxmemory="512m"
- classname="org.torproject.webstats.Main">
- <classpath refid="classpath"/>
- </java>
- </target>
-</project>
-
diff --git a/src/org/torproject/webstats/Main.java b/src/org/torproject/webstats/Main.java
deleted file mode 100644
index 378093e..0000000
--- a/src/org/torproject/webstats/Main.java
+++ /dev/null
@@ -1,595 +0,0 @@
-package org.torproject.webstats;
-
-import java.io.*;
-import java.text.*;
-import java.util.*;
-import java.util.regex.*;
-
-import org.apache.commons.compress.compressors.gzip.*;
-
-/*
- * Sanitize Apache web logs by removing all potentially sensitive parts.
- *
- * The following sanitizing steps are performed on input web logs:
- * 1. Discard all lines that are not in the Combined Log Format.
- * 2. Discard all lines with other hosts than '0.0.0.0' or '0.0.0.1'.
- * 3. Discard all lines with other methods than GET.
- * 4. Discard all lines with other protocols than HTTP.
- * 5. Discard all lines with status code 404.
- * 6. Override client with '-'.
- * 7. Override user with '-'.
- * 8. Override time with '00:00:00 +0000'.
- * 9. Override referer (sic!) with '"-"'.
- * 10. Override user agent with '"-"'.
- * 11. Truncate resource at the first '?' character.
- *
- * The main operation is to parse Apache web log files from the in/
- * directory and write sanitized web log files to the out/ directory.
- * Files in the in/ directory are assumed to never change and may be
- * deleted after processing by this program. Files in the out/ directory
- * are guaranteed to never change and may be deleted by a subsequently
- * running program.
- *
- * This program uses a couple of state files to make sure that files in
- * in/ are not parsed more than once and that files in out/ do not need to
- * be changed:
- * - state/lock prevents concurrent executions of this program.
- * - state/in-history contains file names of previously read and possibly
- * deleted files in the in/ directory.
- * - state/in-history.new is the file written in the current execution
- * that will replace state/in-history during the execution.
- * - state/temp/ contains new or updated output files parsed in the
- * current execution that are moved to out/, state/full/, or state/diff/
- * during the execution.
- * - state/out-history contains file names of previously written and
- * possibly deleted files in the out/ directory.
- * - state/out-history.new is the file written in the current execution
- * that will replace state/out-history at the end of the execution.
- * - state/full/ contains complete output files that may or may not be
- * newer than files in the out/ directory.
- * - state/diff/ contains new parts for files in the out/ directory which
- * have been deleted.
- *
- * The phases and steps taken by this program are as follows:
- *
- * Phase I: Read files from in/, sanitize them, and write them to state/.
- * 1. Check that state/lock does not exists, or exit immediately. Add a
- * new state/lock file.
- * 2. Read the contents from state/in-history and the directory listing
- * of in/ to memory.
- * 3. For each file in in/:
- * a. Append the file name to state/in-history.new if it was not
- * contained in state/in-history. If it was contained, skip the
- * file.
- * b. Parse and sanitize the file in chunks of 250,000 lines to reduce
- * writes.
- * c. When writing sanitized chunks to output files, for each output
- * file, check in the following order if there is already such a
- * file in
- * i. state/temp/,
- * ii. state/full/,
- * iii. out/, or
- * iv. state/diff/.
- * If there's such a file, merge the newly sanitized lines into
- * that file and write the sorted result to state/temp/.
- * 4. Rename state/in-history to state/in-history.old and rename
- * state/in-history.new to state/in-history. Delete
- * state/in-history.old.
- *
- * Phase II: Move files that won't change anymore from state/ to out/.
- * 5. Read the contents from state/out-history to memory.
- * 6. For each file in state/temp/:
- * a. Check if there's a corresponding line in state/out-history. If
- * so, check whether there is a file in state/full/ or out/. If
- * so, move the file to state/full/. Otherwise move the file to
- * state/diff/. Print out a warning that there is a more recent
- * file available.
- * b. If a. does not apply and the sanitized log is less than four (4)
- * days old, move the file to state/full/.
- * c. If b. does not apply, append a line to out-history.new and move
- * the file to out/.
- * 7. For each file in state/full/, check whether the sanitized log is at
- * least four (4) days old and not contained in state/out-history. If
- * so, append a line to out-history.new and move the file to out/.
- * 8. Rename state/out-history to state/out-history.old and rename
- * state/out-history.new to state/out-history. Delete
- * state/out-history.old.
- * 9. Delete state/lock and exit.
- *
- * If the program is interrupted and leaves a lock file in state/lock, it
- * requires an operator to fix the state/ directory and make it work
- * again. IMPORTANT: DO NOT CHANGE ANYTHING IN THE state/ DIRECTORY
- * UNLESS YOU'RE CERTAIN WHAT YOU'RE DOING! The following situations can
- * happen. It may make sense to try a solution in a test environment
- * first:
- * A. The file state/in-history.new does not exist and there are no files
- * in state/temp/. The process died before step 3, that is before
- * actually doing anything of phase I. Delete state/lock and re-run
- * the program.
- * B. The file state/in-history.new exists and there are files in
- * state/temp/. The process died during steps 3 or 4, that is, during
- * phase I. Delete all files in state/temp/. If state/in-history
- * does not exist but state/in-history.old does exist, rename the
- * latter to the former. Delete state/lock and re-run the program.
- * C. The file state/in-history.new does not exist, but there are files
- * in state/temp/. The process died after step 4, that is during
- * phase II. Run the steps 5 to 9 manually. Then re-run the program.
- *
- * Whenever logs are parsed that are 4 days old or older, there may
- * already be output files in out/ that cannot be modified anymore. The
- * operator may decide to manually overwrite files in out/ with the files
- * in state/full/ or state/diff/. IMPORTANT: ONLY OVERWRITE FILES IN out/
- * IF YOU'RE CERTAIN HOW TO FIX THE PROGRAM THAT PARSES THESE FILES.
- * There are two possible situations:
- * A. There is a file in state/full/. This file is newer than the file
- * with the same name in out/ and contains everything from that file,
- * too. It's okay to overwrite the file in out/ with the file in
- * state/full/ and delete the file in state/full/.
- * B. There is a file in state/diff/. The file in out/ didn't exist
- * anymore when parsing more log lines for it. The file that was in
- * out/ should be located and merged with the file in state/diff/.
- * Afterwards, the file in state/diff/ must be deleted.
- */
-public class Main {
-
- /* Run the steps described above. */
- public static void main(String[] args) {
-
- /* Phase I */
- checkAndCreateLockFile(); /* Step 1 */
- readInHistoryFile(); /* Step 2 */
- readInDirectoryListing();
- for (File inFile : inFiles) { /* Step 3 */
- appendToInHistoryIfNotContained(inFile);
- if (!checkFileName(inFile) || checkParsedBefore(inFile)) {
- continue;
- }
- sanitizeInFile(inFile);
- }
- overwriteInHistoryFile(); /* Step 4 */
-
- /* Phase II */
- readOutHistoryFile(); /* Step 5 */
- for (String outputFileName : updatedOutputFiles) { /* Step 6 */
- moveOutputFile(outputFileName);
- }
- moveFullFilesToOut(); /* Step 7 */
- overwriteOutHistoryFile(); /* Step 8 */
- deleteLockFile(); /* Step 9 */
- }
-
- /* Define file and directory names. */
- private static File inDirectory = new File("in");
- private static File outDirectory = new File("out");
- private static File stateLockFile = new File("state/lock");
- private static File stateInHistoryFile = new File("state/in-history");
- private static File stateInHistoryNewFile =
- new File("state/in-history.new");
- private static File stateInHistoryOldFile =
- new File("state/in-history.old");
- private static File stateOutHistoryFile = new File("state/out-history");
- private static File stateOutHistoryNewFile =
- new File("state/out-history.new");
- private static File stateOutHistoryOldFile =
- new File("state/out-history.old");
- private static File stateDiffDirectory = new File("state/diff");
- private static String stateFullDirectoryString = "state/full";
- private static File stateFullDirectory = new File("state/full");
- private static File stateTempDirectory = new File("state/temp");
-
- /* Define data structures and helper classes. */
- private static Set<String> inHistoryFiles;
- private static Set<String> inHistoryNewFiles;
- private static Set<String> outHistoryFiles;
- private static Set<File> inFiles;
- private static Map<String, List<String>> cachedLinesPerOutputFile =
- new HashMap<String, List<String>>();
- private static int cachedLines = 0;
- private static Set<String> updatedOutputFiles = new HashSet<String>();
- private static SimpleDateFormat outputFileFormat =
- new SimpleDateFormat("yyyy/MM/dd/");
- private static SimpleDateFormat logDateFormat =
- new SimpleDateFormat("dd/MMM/yyyy");
- static {
- outputFileFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- logDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- }
- private static Pattern logLinePattern = Pattern.compile(
- "(0.0.0.[01]) \\S+ \\S+ \\[([\\w/]{11})[\\d:]+\\s[+\\-]\\d{4}\\] "
- + "\"GET (\\S+?) (HTTP/[\\d\\.]+)\" (\\d{3}) ([\\d-]+) \"[^\"]+\" "
- + "\"[^\"]+\"");
- private static long now = System.currentTimeMillis();
-
- /* Implement the substeps. */
- private static void checkAndCreateLockFile() {
- if (stateLockFile.exists()) {
- System.err.println("Lock file '" + stateLockFile.getAbsolutePath()
- + "' exists. This means that a previous run did not exit "
- + "cleanly. Exiting.");
- System.exit(1);
- }
- try {
- stateLockFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- stateLockFile));
- bw.close();
- } catch (IOException e) {
- e.printStackTrace();
- System.err.println("Could not create lock file '"
- + stateLockFile.getAbsolutePath() + "'. Exiting.");
- System.exit(1);
- }
- }
- private static void readInHistoryFile() {
- inHistoryFiles = readAndCopyHistoryFile(stateInHistoryFile,
- stateInHistoryNewFile);
- inHistoryNewFiles = new HashSet<String>(inHistoryFiles);
- }
- private static void readOutHistoryFile() {
- outHistoryFiles = readAndCopyHistoryFile(stateOutHistoryFile,
- stateOutHistoryNewFile);
- }
- private static Set<String> readAndCopyHistoryFile(File historyFile,
- File historyNewFile) {
- Set<String> result = new HashSet<String>();
- try {
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- historyNewFile));
- if (historyFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader(
- historyFile));
- String line;
- while ((line = br.readLine()) != null) {
- result.add(line);
- bw.write(line + "\n");
- }
- br.close();
- }
- bw.close();
- } catch (IOException e) {
- e.printStackTrace();
- System.err.println("Could not read parse history file '"
- + historyFile.getAbsolutePath() + "'. Exiting.");
- System.exit(1);
- }
- return result;
- }
- private static void readInDirectoryListing() {
- inFiles = readDirectoryListing(inDirectory);
- }
- private static Set<File> readDirectoryListing(File directory) {
- Set<File> result = new HashSet<File>();
- if (directory.exists()) {
- Stack<File> files = new Stack<File>();
- files.add(directory);
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else {
- result.add(file);
- }
- }
- }
- return result;
- }
- private static void appendToInHistoryIfNotContained(File inFile) {
- if (!inHistoryNewFiles.contains(inFile.getAbsolutePath())) {
- inHistoryNewFiles.add(inFile.getAbsolutePath());
- String line = inFile.getAbsolutePath();
- appendToHistoryFile(stateInHistoryNewFile, line);
- }
- }
- private static void appendToHistoryFile(File historyFile, String line) {
- try {
- BufferedWriter bw = new BufferedWriter(new FileWriter(historyFile,
- true));
- bw.write(line + "\n");
- bw.close();
- } catch (IOException e) {
- e.printStackTrace();
- System.err.println("Could not append line '" + line + "' to parse "
- + "history file '" + historyFile.getAbsolutePath() + "'. "
- + "Exiting.");
- System.exit(1);
- }
- }
- private static boolean checkFileName(File inFile) {
- return inFile.getName().contains("access.log") &&
- inFile.getName().endsWith(".gz");
- }
- private static boolean checkParsedBefore(File inFile) {
- return inHistoryFiles.contains(inFile.getAbsolutePath());
- }
- private static void sanitizeInFile(File inFile) {
- try {
- BufferedReader br = new BufferedReader(new InputStreamReader(
- new GzipCompressorInputStream(new FileInputStream(inFile))));
- String line = null;
- String outputFilenamePart = inFile.getName().substring(0,
- inFile.getName().indexOf("access.log")) + "access.log";
- while (true) {
- line = br.readLine();
- if (line == null || cachedLines > 250000) {
- writeCachedLines();
- }
- if (line == null) {
- break;
- }
- if (!parseLine(line, outputFilenamePart)) {
- break;
- }
- }
- br.close();
- } catch (IOException e) {
- e.printStackTrace();
- System.out.println("Error while parsing log file '"
- + inFile.getAbsolutePath() + "'. Exiting.");
- System.exit(1);
- }
- }
- private static void writeCachedLines() {
- for (Map.Entry<String, List<String>> e :
- cachedLinesPerOutputFile.entrySet()) {
- String outputFilename = e.getKey();
- List<String> cachedLinesList = e.getValue();
- Collections.sort(cachedLinesList);
- storeOutputFile(outputFilename, cachedLinesList);
- }
- cachedLinesPerOutputFile.clear();
- cachedLines = 0;
- }
- private static void storeOutputFile(String outputFileName,
- List<String> cachedLinesList) {
- String outputGzFileName = outputFileName + ".gz";
- File stateTempFile = new File(stateTempDirectory, outputFileName);
- File stateTempGzFile = new File(stateTempDirectory, outputGzFileName);
- File stateFullFile = new File(stateFullDirectory, outputFileName);
- File stateFullGzFile = new File(stateFullDirectory, outputGzFileName);
- File outFile = new File(outDirectory, outputFileName);
- File outGzFile = new File(outDirectory, outputGzFileName);
- File stateDiffFile = new File(stateDiffDirectory, outputFileName);
- File stateDiffGzFile = new File(stateDiffDirectory, outputGzFileName);
- if (stateTempFile.exists()) {
- updatedOutputFiles.add(outputFileName);
- File stateTempOldFile = new File(stateTempDirectory,
- outputFileName + ".old");
- stateTempFile.renameTo(stateTempOldFile);
- mergeOutputFile(stateTempOldFile, cachedLinesList, stateTempFile);
- stateTempOldFile.delete();
- } else if (stateTempGzFile.exists()) {
- updatedOutputFiles.add(outputGzFileName);
- File stateTempGzOldFile = new File(stateTempDirectory,
- outputGzFileName + ".old");
- stateTempGzFile.renameTo(stateTempGzOldFile);
- mergeOutputFile(stateTempGzOldFile, cachedLinesList,
- stateTempGzFile);
- stateTempGzOldFile.delete();
- } else if (stateFullFile.exists()) {
- updatedOutputFiles.add(outputFileName);
- mergeOutputFile(stateFullFile, cachedLinesList, stateTempFile);
- } else if (stateFullGzFile.exists()) {
- updatedOutputFiles.add(outputGzFileName);
- mergeOutputFile(stateFullGzFile, cachedLinesList, stateTempGzFile);
- } else if (outFile.exists()) {
- updatedOutputFiles.add(outputFileName);
- mergeOutputFile(outFile, cachedLinesList, stateTempFile);
- } else if (outGzFile.exists()) {
- updatedOutputFiles.add(outputGzFileName);
- mergeOutputFile(outGzFile, cachedLinesList, stateTempGzFile);
- } else if (stateDiffFile.exists()) {
- updatedOutputFiles.add(outputFileName);
- mergeOutputFile(stateDiffFile, cachedLinesList, stateTempFile);
- } else if (stateDiffGzFile.exists()) {
- updatedOutputFiles.add(outputGzFileName);
- mergeOutputFile(stateDiffGzFile, cachedLinesList, stateTempGzFile);
- } else {
- updatedOutputFiles.add(outputGzFileName);
- writeNewOutputFile(cachedLinesList, stateTempGzFile);
- }
- }
- private static void mergeOutputFile(File oldOutputFile,
- List<String> cachedLinesList, File newOutputFile) {
- try {
- BufferedReader br;
- if (oldOutputFile.getName().endsWith(".gz")) {
- br = new BufferedReader(new InputStreamReader(
- new GzipCompressorInputStream(new FileInputStream(
- oldOutputFile))));
- } else {
- br = new BufferedReader(new FileReader(oldOutputFile));
- }
- String line;
- newOutputFile.getParentFile().mkdirs();
- BufferedWriter bw;
- if (newOutputFile.getName().endsWith(".gz")) {
- bw = new BufferedWriter(new OutputStreamWriter(
- new GzipCompressorOutputStream(new FileOutputStream(
- newOutputFile))));
- } else {
- bw = new BufferedWriter(new FileWriter(newOutputFile));
- }
- int cachedLinesListPosition = 0,
- totalCachedLines = cachedLinesList.size();
- while ((line = br.readLine()) != null) {
- while (cachedLinesListPosition < totalCachedLines &&
- cachedLinesList.get(cachedLinesListPosition).
- compareTo(line) <= 0) {
- bw.write(cachedLinesList.get(
- cachedLinesListPosition) + "\n");
- cachedLinesListPosition++;
- }
- bw.write(line + "\n");
- }
- while (cachedLinesListPosition < totalCachedLines) {
- bw.write(cachedLinesList.get(cachedLinesListPosition++) + "\n");
- }
- br.close();
- bw.close();
- } catch (IOException e) {
- e.printStackTrace();
- System.err.println("Could not merge old output file '"
- + oldOutputFile.getAbsolutePath() + "' with new log lines and "
- + "write it to '" + newOutputFile.getAbsolutePath()
- + "'. Exiting.");
- System.exit(1);
- }
- }
- private static void writeNewOutputFile(List<String> cachedLinesList,
- File outputFile) {
- try {
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
- new GzipCompressorOutputStream(new FileOutputStream(
- outputFile))));
- for (String cachedLine : cachedLinesList) {
- bw.write(cachedLine + "\n");
- }
- bw.close();
- } catch (IOException e) {
- e.printStackTrace();
- System.err.println("Could not write output file '"
- + outputFile.getAbsolutePath() + "'. Exiting.");
- System.exit(1);
- }
- }
- private static boolean parseLine(String line,
- String outputFilenamePart) {
- Matcher matcher = logLinePattern.matcher(line);
- if (!matcher.matches()) {
- return true;
- }
- String statusCode = matcher.group(5);
- if (statusCode.equals("404")) {
- return true;
- }
- String ipAddress = matcher.group(1);
- String date = matcher.group(2);
- String url = matcher.group(3);
- if (url.contains("?")) {
- url = url.substring(0, url.indexOf("?"));
- }
- String httpVersion = matcher.group(4);
- String returnedBytes = matcher.group(6);
- String sanitizedLine = ipAddress + " - - [" + date
- + ":00:00:00 +0000] \"GET " + url + " " + httpVersion
- + "\" " + statusCode + " " + returnedBytes + " \"-\" \"-\"";
- String outputFilename = null;
- try {
- outputFilename = outputFileFormat.format(logDateFormat.parse(
- date).getTime()) + outputFilenamePart;
- } catch (ParseException e) {
- System.out.println("Error parsing date. Aborting to parse "
- + "this file.");
- return false;
- }
- if (!cachedLinesPerOutputFile.containsKey(outputFilename)) {
- cachedLinesPerOutputFile.put(outputFilename,
- new ArrayList<String>());
- }
- cachedLinesPerOutputFile.get(outputFilename).add(
- sanitizedLine);
- cachedLines++;
- return true;
- }
- private static void overwriteInHistoryFile() {
- stateInHistoryFile.renameTo(stateInHistoryOldFile);
- stateInHistoryNewFile.renameTo(stateInHistoryFile);
- stateInHistoryOldFile.delete();
- }
- private static void moveOutputFile(String outputFileName) {
- File outFile = new File(outDirectory, outputFileName);
- File stateTempFile = new File(stateTempDirectory, outputFileName);
- File stateFullFile = new File(stateFullDirectory, outputFileName);
- File stateDiffFile = new File(stateDiffDirectory, outputFileName);
- long ageInDays = -1L;
- try {
- ageInDays = (now
- - outputFileFormat.parse(outputFileName.substring(0,
- outputFileName.lastIndexOf("/") + 1)).getTime())
- / (24L * 60L * 60L * 1000L);
- } catch (ParseException e) {
- e.printStackTrace();
- System.err.println("Could not parse timestamp from '"
- + outputFileName + "'. Exiting.");
- System.exit(1);
- }
- if (outHistoryFiles.contains(outFile.getAbsolutePath())) {
- if (outFile.exists() || stateFullFile.exists()) {
- System.out.println("Could not write to output file '"
- + outFile.getAbsolutePath() + "', because that file was "
- + "written " + (outFile.exists() ? "" : "and deleted ")
- + "before and we're not supposed to change it anymore. The "
- + "updated file that could replace the output file is '"
- + stateFullFile.getAbsolutePath() + "'.");
- stateFullFile.getParentFile().mkdirs();
- stateTempFile.renameTo(stateFullFile);
- } else {
- System.out.println("Could not write to output file '"
- + outFile.getAbsolutePath() + "', because that file was "
- + "written and deleted before and we're not supposed to "
- + "change it anymore (even if we could). The file "
- + "containing the new lines only is '"
- + stateDiffFile.getAbsolutePath() + "'.");
- stateDiffFile.getParentFile().mkdirs();
- stateTempFile.renameTo(stateDiffFile);
- }
- } else if (ageInDays < 4L) {
- stateFullFile.getParentFile().mkdirs();
- stateTempFile.renameTo(stateFullFile);
- } else {
- outFile.getParentFile().mkdirs();
- String line = outFile.getAbsolutePath();
- appendToHistoryFile(stateOutHistoryNewFile, line);
- stateTempFile.renameTo(outFile);
- }
- }
- private static void moveFullFilesToOut() {
- Stack<String> fileNames = new Stack<String>();
- fileNames.add(stateFullDirectoryString);
- while (!fileNames.isEmpty()) {
- String fileName = fileNames.pop();
- File fileOrDirectory = new File(fileName);
- if (!fileOrDirectory.exists()) {
- continue;
- } else if (fileOrDirectory.isDirectory()) {
- for (File file : fileOrDirectory.listFiles()) {
- fileNames.add(fileName + "/" + file.getName());
- }
- } else {
- String outputFileName = fileName.substring(
- (stateFullDirectoryString + "/").length());
- File outFile = new File(outDirectory, outputFileName);
- File stateFullFile = new File(stateFullDirectory, outputFileName);
- long ageInDays = -1L;
- try {
- ageInDays = (now
- - outputFileFormat.parse(outputFileName.substring(0,
- outputFileName.lastIndexOf("/") + 1)).getTime())
- / (24L * 60L * 60L * 1000L);
- } catch (ParseException e) {
- e.printStackTrace();
- System.err.println("Could not parse timestamp from '"
- + outputFileName + "'. Exiting.");
- System.exit(1);
- }
- if (!outHistoryFiles.contains(outFile.getAbsolutePath()) &&
- ageInDays >= 4L) {
- outFile.getParentFile().mkdirs();
- String line = outFile.getAbsolutePath();
- appendToHistoryFile(stateOutHistoryNewFile, line);
- stateFullFile.renameTo(outFile);
- }
- }
- }
- }
- private static void overwriteOutHistoryFile() {
- stateOutHistoryFile.renameTo(stateOutHistoryOldFile);
- stateOutHistoryNewFile.renameTo(stateOutHistoryFile);
- stateOutHistoryOldFile.delete();
- }
- private static void deleteLockFile() {
- stateLockFile.delete();
- }
-}
-
diff --git a/src/sanitize.py b/src/sanitize.py
new file mode 100755
index 0000000..0c5de19
--- /dev/null
+++ b/src/sanitize.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+""" Sanitize Apache web logs by removing all potentially sensitive parts.
+
+The following sanitizing steps are performed on data read from stdin:
+ 1. Die if a line is not in the Apache2 Combined Log Format.
+ 2. Die if other hosts than '0.0.0.0' or '0.0.0.1' are specified.
+ 3. Discard all lines with other methods than GET.
+ 4. Die if a protocol other than HTTP is used.
+ 5. Discard all lines with status code 404.
+ 6. Override client with '-'.
+ 7. Override user with '-'.
+ 8. Override time with '00:00:00 +0000'.
+ 9. Override referer (sic!) with '"-"'.
+ 10. Override user agent with '"-"'.
+ 11. Truncate resource at the first '?' character.
+ 12. Die if a valid date wasn't passed as the sole cmdline parameter.
+ 13. Die if a line has a date not equal or one day prior.
+
+USAGE: sanitize.py INPUT_FILE OUTPUT_DIR
+
+The main operation is to parse Apache web log files from INPUT_FILE and
+output the sanitized version in files according to the date of the log
+entry in OUTPUT_DIR. All discarded lines are output on stderr. A nonzero
+exit code indicates an error during processing, error messages go to
+stderr.
+
+The input filename is expected to be in the following format:
+ <hostname>.torproject.org-access.log-YYYYMMDD
+"""
+
+from __future__ import print_function
+
+import re
+import fileinput
+import sys
+import dateutil.parser
+import datetime
+
+assert(len(sys.argv) == 3)
+
+# Extract date from filename
+date = re.compile(r'[^0-9]*([0-9]{8})')
+matched = date.match(sys.argv[1])
+if matched is None:
+ print("Could not extract date from", sys.argv[1], file=sys.stderr)
+ sys.exit(1)
+today = dateutil.parser.parse(matched.group(1))
+
+is_valid_regex = re.compile(r'^0\.0\.0\.([01]) - - \[(\d{2}/(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)/\d{4}):00:00:00 \+0000\] "([^ ]*) ([^ ?]*[?]?|)[^ ]* HTTP([^"]*)" (-|\d*) (-|\d*) "([^\"]|\\|\")*" "([^"]|\")*" .*[^ ]$')
+sanitized_regex = r'0.0.0.\1 - - [\2:00:00:00 +0000] "\4 \5 HTTP\6" \7 \8 "-" "-" -\n'
+day_before = today - datetime.timedelta(days=1)
+
+today_fname = sys.argv[2] + "/" + sys.argv[1] + "_sanitized"
+yesterday_fname = today_fname.replace(matched.group(1), day_before.strftime("%Y%m%d"))
+
+with open(yesterday_fname, 'a') as file_old:
+ with open(today_fname, 'a') as file_new:
+ for line in fileinput.input(sys.argv[1]):
+ matched = is_valid_regex.match(line)
+ if matched is None:
+ print(line, "Last line does not match critera", file=sys.stderr)
+ sys.exit(1)
+ date = dateutil.parser.parse(matched.group(2))
+ if today != date and day_before != date:
+ print(line, "Last line does not match date constraints. today:", today,
+ " day before:", day_before, " date:", date, file=sys.stderr)
+ sys.exit(1)
+ requesttype = matched.group(4)
+ if requesttype != "GET" and requesttype != "HEAD":
+ #print(matched.expand(sanitized_regex), file=sys.stderr, end="")
+ continue
+
+ if matched.group(7) == "404":
+ #print(matched.expand(sanitized_regex), file=sys.stderr, end="")
+ continue
+
+ if today == date:
+ file_new.write(matched.expand(sanitized_regex))
+ else:
+ file_old.write(matched.expand(sanitized_regex))
+
+print(yesterday_fname)
diff --git a/src/treat_new_logs.sh b/src/treat_new_logs.sh
new file mode 100755
index 0000000..147ebec
--- /dev/null
+++ b/src/treat_new_logs.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -u
+set -e
+
+BASEDIR=/srv/webstats.torproject.org/
+SCRIPTDIR="${BASEDIR}/bin/"
+
+BASEINCOMINGDIR="${BASEDIR}/incoming/"
+
+cd "${BASEINCOMINGDIR}"
+for host in *; do
+ INCOMINGDIR="${BASEINCOMINGDIR}/${host}/"
+ WORKDIR="${BASEDIR}/work/${host}/"
+ WORKDIR_AWSTATS="${BASEDIR}/work_awstats/${host}/"
+ OUTDIR="${BASEDIR}/out/${host}/"
+ STAMPDIR="${BASEDIR}/stamp/${host}/"
+
+ cd "${INCOMINGDIR}"
+ mkdir -p "${WORKDIR}"
+ mkdir -p "${WORKDIR_AWSTATS}"
+ mkdir -p "${OUTDIR}"
+ mkdir -p "${STAMPDIR}"
+
+ for file in *; do
+ basefile=${file%.gz}
+ if [ -e "${STAMPDIR}/${file}_treated" ]; then
+ continue
+ fi
+ cp "${INCOMINGDIR}/${file}" "${WORKDIR}/${file}"
+ cd "${WORKDIR}"
+ gunzip ${file}
+ COMPLETED=$(${SCRIPTDIR}/sanitize.py "${basefile}" "${WORKDIR}")
+ COMPLETED_BASE=$(basename $COMPLETED)
+ COMPLETED_BASE=${COMPLETED_BASE%_sanitized}
+ sort "${COMPLETED}" > "${COMPLETED}_sorted"
+ xz -ck9e "${COMPLETED}_sorted" > "${OUTDIR}/${COMPLETED_BASE}.xz"
+ mv "${COMPLETED}_sorted" "${WORKDIR_AWSTATS}"
+
+ rm "${WORKDIR}/${basefile}"
+ rm "${WORKDIR}/${COMPLETED_BASE}_sanitized"
+ touch "${STAMPDIR}/${file}_treated"
+ done
+done
+
