[or-cvs] [ernie/master] Keep import history for sanitized bridge descriptors.

[karsten at torproject.org]

Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue, 25 May 2010 15:49:25 +0200
Subject: Keep import history for sanitized bridge descriptors.
Commit: 43431ba884c2d0dd760abedaab0fd88839d67767

---
 config                          |    6 +++++
 src/Configuration.java          |    7 ++++++
 src/Main.java                   |    3 +-
 src/SanitizedBridgesReader.java |   42 ++++++++++++++++++++++++++++++++++++++-
 4 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/config b/config
index 967403e..0bac2b5 100644
--- a/config
+++ b/config
@@ -32,6 +32,12 @@
 ## Relative path to directory to import sanitized bridges from
 #SanitizedBridgesDirectory bridges/
 #
+## Keep a history of imported sanitized bridge descriptors. This history
+## can be useful when importing from a changing data source to avoid
+## importing descriptors more than once, but it can be confusing to users
+## who don't know about it.
+#KeepSanitizedBridgesImportHistory 0
+#
 ## Import bridge snapshots from disk, if available
 #ImportBridgeSnapshots 0
 #
diff --git a/src/Configuration.java b/src/Configuration.java
index 1db6f28..4fddbbb 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -38,6 +38,7 @@ public class Configuration {
   private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
   private boolean importSanitizedBridges = false;
   private String sanitizedBridgesDirectory = "bridges/";
+  private boolean keepSanitizedBridgesImportHistory = false;
   private boolean importBridgeSnapshots = false;
   private String bridgeSnapshotsDirectory = "bridge-directories/";
   private boolean importWriteTorperfStats = false;
@@ -145,6 +146,9 @@ public class Configuration {
               line.split(" ")[1]) != 0;
         } else if (line.startsWith("SanitizedBridgesDirectory")) {
           this.sanitizedBridgesDirectory = line.split(" ")[1];
+        } else if (line.startsWith("KeepSanitizedBridgesImportHistory")) {
+          this.keepSanitizedBridgesImportHistory = Integer.parseInt(
+              line.split(" ")[1]) != 0;
         } else if (line.startsWith("ImportBridgeSnapshots")) {
           this.importBridgeSnapshots = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -341,6 +345,9 @@ public class Configuration {
   public String getSanitizedBridgesDirectory() {
     return this.sanitizedBridgesDirectory;
   }
+  public boolean getKeepSanitizedBridgesImportHistory() {
+    return this.keepSanitizedBridgesImportHistory;
+  }
   public boolean getImportBridgeSnapshots() {
     return this.importBridgeSnapshots;
   }
diff --git a/src/Main.java b/src/Main.java
index 0d01f03..13f705b 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -156,7 +156,8 @@ public class Main {
     // Import bridge descriptors
     if (bdp != null && config.getImportSanitizedBridges()) {
       new SanitizedBridgesReader(bdp,
-          config.getSanitizedBridgesDirectory(), countries);
+          config.getSanitizedBridgesDirectory(), countries,
+          config.getKeepSanitizedBridgesImportHistory());
     }
     if (bdp != null && config.getImportBridgeSnapshots()) {
       new BridgeSnapshotReader(bdp, config.getBridgeSnapshotsDirectory(),
diff --git a/src/SanitizedBridgesReader.java b/src/SanitizedBridgesReader.java
index 3056158..a08acdf 100644
--- a/src/SanitizedBridgesReader.java
+++ b/src/SanitizedBridgesReader.java
@@ -4,9 +4,27 @@ import java.util.logging.*;
 
 public class SanitizedBridgesReader {
   public SanitizedBridgesReader(BridgeDescriptorParser bdp,
-      String bridgesDir, SortedSet<String> countries) {
+      String bridgesDir, SortedSet<String> countries,
+      boolean keepImportHistory) {
     Logger logger =
         Logger.getLogger(SanitizedBridgesReader.class.getName());
+    SortedSet<String> bridgesImportHistory = new TreeSet<String>();
+    File bridgesImportHistoryFile =
+        new File("stats/bridges-import-history");
+    if (keepImportHistory && bridgesImportHistoryFile.exists()) {
+      try {
+        BufferedReader br = new BufferedReader(new FileReader(
+            bridgesImportHistoryFile));
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          bridgesImportHistory.add(line);
+        }
+        br.close();
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Could not read in bridge descriptor "
+            + "import history file. Skipping.");
+      }
+    }
     if (new File(bridgesDir).exists()) {
       logger.fine("Importing files in directory " + bridgesDir + "/...");
       Stack<File> filesInInputDir = new Stack<File>();
@@ -19,6 +37,9 @@ public class SanitizedBridgesReader {
             filesInInputDir.add(f);
           }
           continue;
+        } else if (keepImportHistory && bridgesImportHistory.contains(
+            pop.getName())) {
+          continue;
         } else {
           try {
             BufferedInputStream bis = new BufferedInputStream(
@@ -32,10 +53,15 @@ public class SanitizedBridgesReader {
             bis.close();
             byte[] allData = baos.toByteArray();
             String fn = pop.getName();
+            // TODO dateTime extraction doesn't work for sanitized network
+            // statuses!
             String dateTime = fn.substring(0, 4) + "-" + fn.substring(4, 6)
                 + "-" + fn.substring(6, 8) + " " + fn.substring(9, 11)
                 + ":" + fn.substring(11, 13) + ":" + fn.substring(13, 15);
             bdp.parse(allData, dateTime, true);
+            if (keepImportHistory) {
+              bridgesImportHistory.add(pop.getName());
+            }
           } catch (IOException e) {
             problems.add(pop);
             if (problems.size() > 3) {
@@ -60,6 +86,20 @@ public class SanitizedBridgesReader {
         }
         logger.warning(sb.toString());
       }
+      if (keepImportHistory) {
+        try {
+          bridgesImportHistoryFile.getParentFile().mkdirs();
+          BufferedWriter bw = new BufferedWriter(new FileWriter(
+              bridgesImportHistoryFile));
+          for (String line : bridgesImportHistory) {
+            bw.write(line + "\n");
+          }
+          bw.close();
+        } catch (IOException e) {
+          logger.log(Level.WARNING, "Could not write bridge descriptor "
+              + "import history file.");
+        }
+      }
     }
   }
 }
-- 
1.6.5