[tor-commits] [metrics-tasks/master] Add code to turn BridgeDB logs into assignment.log files.

[karsten at torproject.org]

commit e1ece10fe3629d9a7268780f77ff4b0653ff72a6
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Sun Mar 13 18:43:24 2011 +0100

    Add code to turn BridgeDB logs into assignment.log files.
---
 task-2537/.gitignore                |    4 +
 task-2537/BridgeDBLogConverter.java |  136 +++++++++++++++++++++++++++++++++++
 task-2537/README                    |   15 ++++
 3 files changed, 155 insertions(+), 0 deletions(-)

diff --git a/task-2537/.gitignore b/task-2537/.gitignore
new file mode 100644
index 0000000..d169b51
--- /dev/null
+++ b/task-2537/.gitignore
@@ -0,0 +1,4 @@
+*.log
+*.class
+*.swp
+
diff --git a/task-2537/BridgeDBLogConverter.java b/task-2537/BridgeDBLogConverter.java
new file mode 100644
index 0000000..5a8d69c
--- /dev/null
+++ b/task-2537/BridgeDBLogConverter.java
@@ -0,0 +1,136 @@
+import java.io.*;
+import java.text.*;
+import java.util.*;
+
+public class BridgeDBLogConverter {
+  public static void main(String[] args) throws Exception {
+    if (args.length != 3) {
+      System.out.println("Usage: java BridgeDBLogConverter logfile year "
+          + "outfile");
+      System.exit(1);
+    }
+    File logfile = new File(args[0]), outfile = new File(args[2]);
+    String year = args[1];
+    SimpleDateFormat logFormat = new SimpleDateFormat(
+        "yyyy MMM dd HH:mm:ss");
+    SimpleDateFormat isoFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    SimpleDateFormat fileFormat = new SimpleDateFormat(
+        "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+    logFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    isoFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    fileFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    BufferedReader br = new BufferedReader(new FileReader(logfile));
+    BufferedWriter bw = new BufferedWriter(new FileWriter(outfile));
+    String line;
+    SortedMap<String, String> entries = new TreeMap<String, String>();
+    long lastTimestamp = -1L;
+    String fingerprint = null, lastFingerprint = null, type = null,
+        port = "", flag = "";
+    while ((line = br.readLine()) != null) {
+      long timestamp = logFormat.parse(year + " "
+          + line.substring(0, 15)).getTime();
+      if (timestamp > lastTimestamp + 10L * 60L * 1000L) {
+        if (lastTimestamp > 0L) {
+          bw.write("bridge-pool-assignment "
+              + isoFormat.format(lastTimestamp) + "\n");
+          for (String entry : entries.values()) {
+            bw.write(entry + "\n");
+          }
+          entries.clear();
+        }
+      }
+      lastTimestamp = timestamp;
+      String[] parts = line.split(" ");
+      fingerprint = parts[7];
+      String assignment = line.substring(line.indexOf(parts[7]) + 41);
+      if (!fingerprint.equals(lastFingerprint)) {
+        if (lastFingerprint != null) {
+          entries.put(lastFingerprint, lastFingerprint + " " + type
+          + port + flag);
+        }
+        type = null;
+        port = "";
+        flag = "";
+      }
+      if (assignment.startsWith("to IP ")) {
+        int ring = -1;
+        if (assignment.startsWith("to IP category ring")) {
+          ring = 4; // TODO This is fragile!
+        } else {
+          ring = Integer.parseInt(assignment.split(" ")[3]) - 1;
+        }
+        String newType = "https ring=" + ring;
+        if (type != null && !type.equals(newType)) {
+          System.out.println("type inconsistency in line '" + line + "'");
+          System.exit(1);
+        }
+        type = newType;
+        if (assignment.endsWith(" (port-443 subring)")) {
+          String newPort = " port=443";
+          if (port.length() > 0 && !port.equals(newPort)) {
+            System.out.println("port inconsistency in line '" + line
+                + "'");
+            System.exit(1);
+          }
+          port = newPort;
+        } else if (assignment.endsWith(" (stable subring)")) {
+          String newFlag = " flag=stable";
+          if (flag.length() > 0 && !flag.equals(newFlag)) {
+            System.out.println("flag inconsistency in line '" + line
+                + "'");
+            System.exit(1);
+          }
+          flag = newFlag;
+        }
+      } else if (assignment.equals("to email ring")) {
+        String newType = "email";
+        if (type != null && !type.equals(newType)) {
+          System.out.println("type inconsistency in line '" + line + "'");
+          System.exit(1);
+        }
+        type = newType;
+      } else if (assignment.startsWith("to Ring ")) {
+        String newType = "email";
+        if (type != null && !type.equals(newType)) {
+          System.out.println("type inconsistency in line '" + line + "'");
+          System.exit(1);
+        }
+        type = newType;
+        if (assignment.equals("to Ring (port-443 subring)")) {
+          String newPort = " port=443";
+          if (port.length() > 0 && !port.equals(newPort)) {
+            System.out.println("port inconsistency in line '" + line
+                + "'");
+            System.exit(1);
+          }
+          port = newPort;
+        } else if (assignment.equals("to Ring (stable subring)")) {
+          String newFlag = " flag=stable";
+          if (flag.length() > 0 && !flag.equals(newFlag)) {
+            System.out.println("flag inconsistency in line '" + line
+                + "'");
+            System.exit(1);
+          }
+          flag = newFlag;
+        } else {
+          System.out.println("type inconsistency in line '" + line
+              + "'");
+          System.exit(1);
+        }
+      } else {
+        type = "unallocated";
+      }
+      lastFingerprint = fingerprint;
+    }
+    if (lastTimestamp > 0L) {
+      bw.write("bridge-pool-assignment "
+          + isoFormat.format(lastTimestamp) + "\n");
+      for (String entry : entries.values()) {
+        bw.write(entry + "\n");
+      }
+    }
+    bw.close();
+  }
+}
+
diff --git a/task-2537/README b/task-2537/README
new file mode 100644
index 0000000..1d5375f
--- /dev/null
+++ b/task-2537/README
@@ -0,0 +1,15 @@
+BridgeDBLogConverter.java
+
+The BridgeDBLogConverter takes BridgeDB's logs as input and writes
+(non-sanitized) bridge pool assignments to disk as output.
+
+ - Compile the Java class, e.g.,
+   $ javac BridgeDBLogConverter.java
+
+ - If your bridgedb.log file contains log lines from more than one
+   calendar year, split it into as many log files as necessary to ensure
+   that each file only contains log lines from a single calendar year!
+
+ - Run the Java class, e.g.,
+   $ java BridgeDBLogConverter bridgedb.log 2011 assignments.log
+