[or-cvs] [metrics-db/master] Allow limiting internal bridge descriptor mapping.

karsten at torproject.org karsten at torproject.org
Mon Feb 7 06:36:28 UTC 2011


commit 62bd96d53037fa2712d75bbf9f23855a45f4689a
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Fri Feb 4 18:48:51 2011 +0100

    Allow limiting internal bridge descriptor mapping.
---
 config.template                                    |    4 +
 src/org/torproject/ernie/db/Configuration.java     |    7 ++
 src/org/torproject/ernie/db/Main.java              |    3 +-
 .../ernie/db/SanitizedBridgesWriter.java           |   96 +++++++++++++++++++-
 .../ernie/test/SanitizedBridgesWriterTest.java     |    4 +-
 5 files changed, 108 insertions(+), 6 deletions(-)

diff --git a/config.template b/config.template
index a1d35d4..1ef915c 100644
--- a/config.template
+++ b/config.template
@@ -98,6 +98,10 @@
 ## can learn about IP address changes.
 #ReplaceIPAddressesWithHashes 0
 #
+## Limit internal bridge descriptor mapping state to the following number
+## of days, or -1 for unlimited.
+#LimitBridgeDescriptorMappings -1
+#
 ## Relative path to directory to write sanitized bridges to
 #SanitizedBridgesWriteDirectory sanitized-bridges/
 #
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 48e5abe..37676ec 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -31,6 +31,7 @@ public class Configuration {
   private String relayDescriptorRawFilesDirectory = "pg-import/";
   private boolean writeSanitizedBridges = false;
   private boolean replaceIPAddressesWithHashes = false;
+  private long limitBridgeDescriptorMappings = -1L;
   private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
   private boolean importSanitizedBridges = false;
   private String sanitizedBridgesDirectory = "bridges/";
@@ -117,6 +118,9 @@ public class Configuration {
         } else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
           this.replaceIPAddressesWithHashes = Integer.parseInt(
               line.split(" ")[1]) != 0;
+        } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
+          this.limitBridgeDescriptorMappings = Long.parseLong(
+              line.split(" ")[1]);
         } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
           this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
         } else if (line.startsWith("ImportSanitizedBridges")) {
@@ -297,6 +301,9 @@ public class Configuration {
   public boolean getReplaceIPAddressesWithHashes() {
     return this.replaceIPAddressesWithHashes;
   }
+  public long getLimitBridgeDescriptorMappings() {
+    return this.limitBridgeDescriptorMappings;
+  }
   public String getSanitizedBridgesWriteDirectory() {
     return this.sanitizedBridgesWriteDirectory;
   }
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index a159642..20107d0 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -138,7 +138,8 @@ public class Main {
     SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
         new SanitizedBridgesWriter(
         new File(config.getSanitizedBridgesWriteDirectory()),
-        statsDirectory, config.getReplaceIPAddressesWithHashes()) : null;
+        statsDirectory, config.getReplaceIPAddressesWithHashes(),
+        config.getLimitBridgeDescriptorMappings()) : null;
 
     // Prepare bridge descriptor parser
     BridgeDescriptorParser bdp = config.getWriteConsensusStats() ||
diff --git a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
index 243b0b6..6edcc62 100644
--- a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
@@ -169,12 +169,17 @@ public class SanitizedBridgesWriter {
 
   private byte[] secretForHashingIPAddresses;
 
+  private String bridgeDescriptorMappingsCutOffTimestamp;
+
+  private boolean haveWarnedAboutLimitedMapping;
+
   /**
    * Initializes this class, including reading in the known descriptor
    * mapping.
    */
   public SanitizedBridgesWriter(File sanitizedBridgesDirectory,
-      File statsDirectory, boolean replaceIPAddressesWithHashes) {
+      File statsDirectory, boolean replaceIPAddressesWithHashes,
+      long limitBridgeDescriptorMappings) {
 
     if (sanitizedBridgesDirectory == null || statsDirectory == null) {
       throw new IllegalArgumentException();
@@ -199,6 +204,20 @@ public class SanitizedBridgesWriter {
     this.secretForHashingIPAddresses =
         "secret for hashing IP addresses".getBytes();
 
+    /* If we're configured to keep descriptor mappings only for a limited
+     * time, define the cut-off day and time. */
+    if (limitBridgeDescriptorMappings >= 0L) {
+      SimpleDateFormat formatter = new SimpleDateFormat(
+          "yyyy-MM-dd HH:mm:ss");
+      formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+      this.bridgeDescriptorMappingsCutOffTimestamp = formatter.format(
+          System.currentTimeMillis() - 24L * 60L * 60L * 1000L
+          * limitBridgeDescriptorMappings);
+    } else {
+      this.bridgeDescriptorMappingsCutOffTimestamp =
+          "1999-12-31 23:59:59";
+    }
+
     /* Read known descriptor mappings from disk. */
     this.bridgeDescriptorMappingsFile = new File(
         "stats/bridge-descriptor-mappings");
@@ -207,9 +226,16 @@ public class SanitizedBridgesWriter {
         BufferedReader br = new BufferedReader(new FileReader(
             this.bridgeDescriptorMappingsFile));
         String line = null;
+        int read = 0, skipped = 0;
         while ((line = br.readLine()) != null) {
           if (line.split(",").length == 4) {
             String[] parts = line.split(",");
+            if (this.bridgeDescriptorMappingsCutOffTimestamp.
+                compareTo(parts[1]) > 0) {
+              skipped++;
+              continue;
+            }
+            read++;
             DescriptorMapping dm = new DescriptorMapping(line);
             dm.hashedBridgeIdentity = parts[0];
             dm.published = parts[1];
@@ -225,6 +251,8 @@ public class SanitizedBridgesWriter {
           }
         }
         br.close();
+        this.logger.fine("Finished reading " + read + " descriptor "
+            + "mappings from disk, skipped " + skipped + ".");
       } catch (IOException e) {
         this.logger.log(Level.WARNING, "Could not read in "
             + this.bridgeDescriptorMappingsFile.getAbsolutePath()
@@ -262,6 +290,15 @@ public class SanitizedBridgesWriter {
   public void sanitizeAndStoreNetworkStatus(byte[] data,
       String publicationTime) {
 
+    if (this.bridgeDescriptorMappingsCutOffTimestamp.
+        compareTo(publicationTime) > 0) {
+      this.logger.log(!this.haveWarnedAboutLimitedMapping ? Level.WARNING
+          : Level.FINE, "Sanitizing and storing network status with "
+          + "publication time outside our descriptor mapping interval. "
+          + "We might not be able to repair references.");
+      this.haveWarnedAboutLimitedMapping = true;
+    }
+
     /* Parse the given network status line by line. */
     StringBuilder scrubbed = new StringBuilder();
     try {
@@ -425,6 +462,15 @@ public class SanitizedBridgesWriter {
          * the sanitizing procedure. */
         } else if (line.startsWith("published ")) {
           published = line.substring("published ".length());
+          if (this.bridgeDescriptorMappingsCutOffTimestamp.
+              compareTo(published) > 0) {
+            this.logger.log(!this.haveWarnedAboutLimitedMapping
+                ? Level.WARNING : Level.FINE, "Sanitizing and storing "
+                + "server descriptor with publication time outside our "
+                + "descriptor mapping interval. We might not be able to "
+                + "repair references.");
+            this.haveWarnedAboutLimitedMapping = true;
+          }
           this.descriptorPublicationTimes.add(published);
           scrubbed.append(line + "\n");
 
@@ -618,6 +664,15 @@ public class SanitizedBridgesWriter {
         } else if (line.startsWith("published ")) {
           scrubbed.append(line + "\n");
           published = line.substring("published ".length());
+          if (this.bridgeDescriptorMappingsCutOffTimestamp.
+              compareTo(published) > 0) {
+            this.logger.log(!this.haveWarnedAboutLimitedMapping
+                ? Level.WARNING : Level.FINE, "Sanitizing and storing "
+                + "extra-info descriptor with publication time outside "
+                + "our descriptor mapping interval. We might not be able "
+                + "to repair references.");
+            this.haveWarnedAboutLimitedMapping = true;
+          }
 
         /* Write the following lines unmodified to the sanitized
          * descriptor. */
@@ -697,6 +752,14 @@ public class SanitizedBridgesWriter {
   }
 
   public void storeSanitizedNetworkStatus(byte[] data, String published) {
+    if (this.bridgeDescriptorMappingsCutOffTimestamp.
+        compareTo(published) > 0) {
+      this.logger.log(!this.haveWarnedAboutLimitedMapping ? Level.WARNING
+          : Level.FINE, "Storing sanitized network status with "
+          + "publication time outside our descriptor mapping interval. "
+          + "We might not be able to repair references.");
+      this.haveWarnedAboutLimitedMapping = true;
+    }
     String scrubbed = null;
     try {
       String ascii = new String(data, "US-ASCII");
@@ -806,6 +869,15 @@ public class SanitizedBridgesWriter {
               + line2.split(" ")[5] + "\n");
         } else if (line2.startsWith("published ")) {
           published = line2.substring("published ".length());
+          if (this.bridgeDescriptorMappingsCutOffTimestamp.
+              compareTo(published) > 0) {
+            this.logger.log(!this.haveWarnedAboutLimitedMapping
+                ? Level.WARNING : Level.FINE, "Storing sanitized "
+                + "server descriptor with publication time outside our "
+                + "descriptor mapping interval. We might not be able to "
+                + "repair references.");
+            this.haveWarnedAboutLimitedMapping = true;
+          }
           sb.append(line2 + "\n");
           this.descriptorPublicationTimes.add(published);
         } else if (line2.startsWith("opt fingerprint ")) {
@@ -874,6 +946,15 @@ public class SanitizedBridgesWriter {
         } else if (line2.startsWith("published ")) {
           sb.append(line2 + "\n");
           published = line2.substring("published ".length());
+          if (this.bridgeDescriptorMappingsCutOffTimestamp.
+              compareTo(published) > 0) {
+            this.logger.log(!this.haveWarnedAboutLimitedMapping
+                ? Level.WARNING : Level.FINE, "Storing sanitized "
+                + "extra-info descriptor with publication time outside "
+                + "our descriptor mapping interval. We might not be able "
+                + "to repair references.");
+            this.haveWarnedAboutLimitedMapping = true;
+          }
           this.descriptorPublicationTimes.add(published);
         } else {
           sb.append(line2 + "\n");
@@ -1038,12 +1119,21 @@ public class SanitizedBridgesWriter {
       this.logger.fine("Writing descriptor mappings to disk.");
       BufferedWriter bw = new BufferedWriter(new FileWriter(
           this.bridgeDescriptorMappingsFile));
+      int wrote = 0, skipped = 0;
       for (DescriptorMapping mapping :
           this.bridgeDescriptorMappings.values()) {
-        bw.write(mapping.toString() + "\n");
+        String mappingString = mapping.toString();
+        if (this.bridgeDescriptorMappingsCutOffTimestamp.
+            compareTo(mappingString.split(",")[1]) > 0) {
+          skipped++;
+        } else {
+          wrote++;
+          bw.write(mapping.toString() + "\n");
+        }
       }
       bw.close();
-      this.logger.fine("Finished writing descriptor mappings to disk.");
+      this.logger.fine("Finished writing " + wrote + " descriptor "
+          + "mappings to disk, skipped " + skipped + ".");
     } catch (IOException e) {
       this.logger.log(Level.WARNING, "Could not write descriptor "
           + "mappings to disk.", e);
diff --git a/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java b/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java
index d256122..f5b3b64 100644
--- a/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java
+++ b/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java
@@ -27,13 +27,13 @@ public class SanitizedBridgesWriterTest {
 
   @Test(expected = IllegalArgumentException.class)
   public void testSanitizedBridgesDirectoryNull() {
-    new SanitizedBridgesWriter(null, this.tempStatsDirectory, false);
+    new SanitizedBridgesWriter(null, this.tempStatsDirectory, false, -1L);
   }
 
   @Test(expected = IllegalArgumentException.class)
   public void testStatsDirectoryNull() {
     new SanitizedBridgesWriter(this.tempSanitizedBridgesDirectory, null,
-        false);
+        false, -1L);
   }
 }
 





More information about the tor-commits mailing list