[tor-commits] [metrics-db/master] Remove GetTor stats database import.

karsten at torproject.org karsten at torproject.org
Thu Mar 3 18:38:07 UTC 2011


commit f67c7db477740b4f13a949b32c1fec0b5a72c464
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Thu Mar 3 18:25:31 2011 +0100

    Remove GetTor stats database import.
---
 config.template                                   |   15 +--
 src/org/torproject/ernie/db/Configuration.java    |   39 ++----
 src/org/torproject/ernie/db/GetTorDownloader.java |  101 +++++++++++++++
 src/org/torproject/ernie/db/GetTorProcessor.java  |  143 ---------------------
 src/org/torproject/ernie/db/Main.java             |    7 +-
 5 files changed, 122 insertions(+), 183 deletions(-)

diff --git a/config.template b/config.template
index 72cc411..0d6743e 100644
--- a/config.template
+++ b/config.template
@@ -32,12 +32,15 @@
 ## Relative path to directory to import bridge descriptor snapshots from
 #BridgeSnapshotsDirectory bridge-directories/
 #
-## Download and process GetTor stats
-#DownloadProcessGetTorStats 0
+## Download GetTor stats
+#DownloadGetTorStats 0
 #
 ## URL to download GetTor stats from
 #GetTorStatsURL http://gettor.torproject.org:8080/~gettor/gettor_stats.txt
 #
+## Relative path to directory to store GetTor stats in
+#GetTorDirectory gettor/
+#
 ## Download exit list and store it to disk
 #DownloadExitList 0
 #
@@ -49,14 +52,6 @@
 ## Relative path to directory to write directory archives to
 #DirectoryArchivesOutputDirectory directory-archive/
 #
-## Write aggregate statistics (bridges and bridge users per day, directory
-## clients per day, torperf results, packages requested from GetTor, etc.)
-## to database for later evaluation
-#WriteAggregateStatsDatabase 0
-#
-## JDBC string for relay descriptor database
-#RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=ernie&password=password
-#
 ## Write sanitized bridges to disk
 #WriteSanitizedBridges 0
 #
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 4d75b59..c1cdbea 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -21,7 +21,6 @@ public class Configuration {
   private boolean importDirectoryArchives = false;
   private String directoryArchivesDirectory = "archives/";
   private boolean keepDirectoryArchiveImportHistory = false;
-  private boolean writeAggregateStatsDatabase = false;
   private String relayDescriptorDatabaseJdbc =
       "jdbc:postgresql://localhost/tordir?user=ernie&password=password";
   private boolean writeSanitizedBridges = false;
@@ -33,9 +32,10 @@ public class Configuration {
   private boolean downloadRelayDescriptors = false;
   private List<String> downloadFromDirectoryAuthorities = Arrays.asList(
       "86.59.21.38,194.109.206.212,80.190.246.100:8180".split(","));
-  private boolean downloadProcessGetTorStats = false;
+  private boolean downloadGetTorStats = false;
   private String getTorStatsUrl = "http://gettor.torproject.org:8080/"
       + "~gettor/gettor_stats.txt";
+  private String getTorDirectory = "gettor/";
   private boolean downloadExitList = false;
   public Configuration() {
 
@@ -82,11 +82,6 @@ public class Configuration {
         } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
           this.keepDirectoryArchiveImportHistory = Integer.parseInt(
               line.split(" ")[1]) != 0;
-        } else if (line.startsWith("WriteAggregateStatsDatabase")) {
-          this.writeAggregateStatsDatabase = Integer.parseInt(
-              line.split(" ")[1]) != 0;
-        } else if (line.startsWith("RelayDescriptorDatabaseJDBC")) {
-          this.relayDescriptorDatabaseJdbc = line.split(" ")[1];
         } else if (line.startsWith("WriteSanitizedBridges")) {
           this.writeSanitizedBridges = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -119,14 +114,16 @@ public class Configuration {
             new URL("http://" + dir + "/");
             this.downloadFromDirectoryAuthorities.add(dir);
           }
-        } else if (line.startsWith("DownloadProcessGetTorStats")) {
-          this.downloadProcessGetTorStats = Integer.parseInt(
+        } else if (line.startsWith("DownloadGetTorStats")) {
+          this.downloadGetTorStats = Integer.parseInt(
               line.split(" ")[1]) != 0;
         } else if (line.startsWith("GetTorStatsURL")) {
           String newUrl = line.split(" ")[1];
           /* Test if URL has correct format. */
           new URL(newUrl);
           this.getTorStatsUrl = newUrl;
+        } else if (line.startsWith("GetTorDirectory")) {
+          this.getTorDirectory = line.split(" ")[1];
         } else if (line.startsWith("DownloadExitList")) {
           this.downloadExitList = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -158,9 +155,8 @@ public class Configuration {
     /** Make some checks if configuration is valid. */
     if (!this.importCachedRelayDescriptors &&
         !this.importDirectoryArchives && !this.downloadRelayDescriptors &&
-        !this.importBridgeSnapshots && !this.downloadProcessGetTorStats &&
+        !this.importBridgeSnapshots && !this.downloadGetTorStats &&
         !this.downloadExitList && !this.writeDirectoryArchives &&
-        !this.writeAggregateStatsDatabase &&
         !this.writeSanitizedBridges) {
       logger.warning("We have not been configured to read data from any "
           + "data source or write data to any data sink. You need to "
@@ -182,8 +178,7 @@ public class Configuration {
           + "least one data sink, but we don't have a single data source "
           + "containing relay descriptors.");
     }
-    if (this.importBridgeSnapshots && !(this.writeSanitizedBridges ||
-        this.writeAggregateStatsDatabase)) {
+    if (this.importBridgeSnapshots && !this.writeSanitizedBridges) {
       logger.warning("We are configured to import/download bridge "
           + "descriptors, but we don't have a single data sink to write "
           + "bridge descriptors to.");
@@ -193,11 +188,6 @@ public class Configuration {
           + "least one data sink, but we don't have a single data source "
           + "containing bridge descriptors.");
     }
-    if (this.downloadProcessGetTorStats &&
-        !this.writeAggregateStatsDatabase) {
-      logger.warning("We are configured to download GetTor statistics, "
-          + "but not to import them into the database.");
-    }
   }
   public boolean getWriteDirectoryArchives() {
     return this.writeDirectoryArchives;
@@ -220,12 +210,6 @@ public class Configuration {
   public boolean getKeepDirectoryArchiveImportHistory() {
     return this.keepDirectoryArchiveImportHistory;
   }
-  public boolean getWriteAggregateStatsDatabase() {
-    return this.writeAggregateStatsDatabase;
-  }
-  public String getRelayDescriptorDatabaseJDBC() {
-    return this.relayDescriptorDatabaseJdbc;
-  }
   public boolean getWriteSanitizedBridges() {
     return this.writeSanitizedBridges;
   }
@@ -250,12 +234,15 @@ public class Configuration {
   public List<String> getDownloadFromDirectoryAuthorities() {
     return this.downloadFromDirectoryAuthorities;
   }
-  public boolean getDownloadProcessGetTorStats() {
-    return this.downloadProcessGetTorStats;
+  public boolean getDownloadGetTorStats() {
+    return this.downloadGetTorStats;
   }
   public String getGetTorStatsUrl() {
     return this.getTorStatsUrl;
   }
+  public String getGetTorDirectory() {
+    return this.getTorDirectory;
+  }
   public boolean getDownloadExitList() {
     return this.downloadExitList;
   }
diff --git a/src/org/torproject/ernie/db/GetTorDownloader.java b/src/org/torproject/ernie/db/GetTorDownloader.java
new file mode 100644
index 0000000..91e9f5f
--- /dev/null
+++ b/src/org/torproject/ernie/db/GetTorDownloader.java
@@ -0,0 +1,101 @@
+/* Copyright 2010 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+public class GetTorDownloader {
+
+  public GetTorDownloader(String gettorStatsUrl, File getTorDirectory) {
+
+    Logger logger = Logger.getLogger(GetTorDownloader.class.getName());
+
+    File getTorFile = new File(getTorDirectory, "gettor_stats.txt");
+    SortedMap<String, String> getTorStats = new TreeMap<String, String>();
+
+    if (getTorFile.exists() && !getTorFile.isDirectory()) {
+      try {
+        logger.fine("Reading local gettor_stats.txt file...");
+        BufferedReader br = new BufferedReader(new FileReader(
+            getTorFile));
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          String date = line.split(" ")[0];
+          getTorStats.put(date, line);
+        }
+        br.close();
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Failed parsing local GetTor stats!",
+            e);
+        return;
+      }
+    }
+
+    String unparsed = null;
+    try {
+      logger.fine("Downloading GetTor stats...");
+      URL u = new URL(gettorStatsUrl);
+      HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+      huc.setRequestMethod("GET");
+      huc.connect();
+      int response = huc.getResponseCode();
+      if (response == 200) {
+        BufferedInputStream in = new BufferedInputStream(
+            huc.getInputStream());
+        StringBuilder sb = new StringBuilder();
+        int len;
+        byte[] data = new byte[1024];
+        while ((len = in.read(data, 0, 1024)) >= 0) {
+          sb.append(new String(data, 0, len));
+        }
+        in.close();
+        unparsed = sb.toString();
+      }
+      logger.fine("Finished downloading GetTor stats.");
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed downloading GetTor stats", e);
+      return;
+    }
+
+    try {
+      logger.fine("Parsing downloaded GetTor stats...");
+      BufferedReader br = new BufferedReader(new StringReader(unparsed));
+      String line = null;
+      while ((line = br.readLine()) != null) {
+        String date = line.split(" ")[0];
+        getTorStats.put(date, line);
+      }
+      br.close();
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed parsing downloaded GetTor stats!",
+          e);
+      return;
+    }
+
+    try {
+      logger.fine("Writing GetTor stats to local gettor_stats.txt "
+          + "file...");
+      if (!getTorDirectory.exists()) {
+        getTorDirectory.mkdirs();
+      }
+      BufferedWriter bw = new BufferedWriter(new FileWriter(getTorFile));
+      for (String line : getTorStats.values()) {
+        bw.write(line + "\n");
+      }
+      bw.close();
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed writing GetTor stats to local "
+          + "gettor_stats.txt file", e);
+      return;
+    }
+
+    logger.info("Finished downloading and processing statistics on Tor "
+        + "packages delivered by GetTor.\nDownloaded " + unparsed.length()
+        + " bytes. Last date in statistics is " + getTorStats.lastKey()
+        + ".");
+  }
+}
+
diff --git a/src/org/torproject/ernie/db/GetTorProcessor.java b/src/org/torproject/ernie/db/GetTorProcessor.java
deleted file mode 100644
index 6d9a935..0000000
--- a/src/org/torproject/ernie/db/GetTorProcessor.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/* Copyright 2010 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.net.*;
-import java.sql.*;
-import java.util.*;
-import java.util.logging.*;
-
-public class GetTorProcessor {
-  public GetTorProcessor(String gettorStatsUrl, String connectionURL) {
-    Logger logger = Logger.getLogger(GetTorProcessor.class.getName());
-    String unparsed = null;
-    try {
-      logger.fine("Downloading gettor stats...");
-      URL u = new URL(gettorStatsUrl);
-      HttpURLConnection huc = (HttpURLConnection) u.openConnection();
-      huc.setRequestMethod("GET");
-      huc.connect();
-      int response = huc.getResponseCode();
-      if (response == 200) {
-        BufferedInputStream in = new BufferedInputStream(
-            huc.getInputStream());
-        StringBuilder sb = new StringBuilder();
-        int len;
-        byte[] data = new byte[1024];
-        while ((len = in.read(data, 0, 1024)) >= 0) {
-          sb.append(new String(data, 0, len));
-        }
-        in.close();
-        unparsed = sb.toString();
-      }
-      logger.fine("Finished downloading gettor stats.");
-    } catch (IOException e) {
-      logger.log(Level.WARNING, "Failed downloading gettor stats", e);
-      return;
-    }
-
-    SortedSet<String> columns = new TreeSet<String>();
-    SortedMap<String, Map<String, Integer>> data =
-        new TreeMap<String, Map<String, Integer>>();
-    try {
-      logger.fine("Parsing downloaded gettor stats...");
-      BufferedReader br = new BufferedReader(new StringReader(unparsed));
-      String line = null;
-      while ((line = br.readLine()) != null) {
-        String[] parts = line.split(" ");
-        String date = parts[0];
-        Map<String, Integer> obs = new HashMap<String, Integer>();
-        data.put(date, obs);
-        for (int i = 2; i < parts.length; i++) {
-          String key = parts[i].split(":")[0].toLowerCase();
-          Integer value = new Integer(parts[i].split(":")[1]);
-          columns.add(key);
-          obs.put(key, value);
-        }
-      }
-      br.close();
-    } catch (IOException e) {
-      logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
-      return;
-    } catch (NumberFormatException e) {
-      logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
-      return;
-    }
-
-    /* Write results to database. */
-    if (connectionURL != null) {
-      try {
-        Map<String, Integer> updateRows = new HashMap<String, Integer>(),
-            insertRows = new HashMap<String, Integer>();
-        for (Map.Entry<String, Map<String, Integer>> e :
-            data.entrySet()) {
-          String date = e.getKey();
-          Map<String, Integer> obs = e.getValue();
-          for (String column : columns) {
-            if (obs.containsKey(column)) {
-              Integer value = obs.get(column);
-              String key = date + "," + column;
-              insertRows.put(key, value);
-            }
-          }
-        }
-        Connection conn = DriverManager.getConnection(connectionURL);
-        PreparedStatement psI = conn.prepareStatement(
-            "INSERT INTO gettor_stats (downloads, date, bundle) "
-            + "VALUES (?, ?, ?)");
-        PreparedStatement psU = conn.prepareStatement(
-            "UPDATE gettor_stats SET downloads = ? "
-            + "WHERE date = ? AND bundle = ?");
-        conn.setAutoCommit(false);
-        Statement statement = conn.createStatement();
-        ResultSet rs = statement.executeQuery(
-            "SELECT date, bundle, downloads FROM gettor_stats");
-        while (rs.next()) {
-          String date = rs.getDate(1).toString();
-          String bundle = rs.getString(2);
-          String key = date + "," + bundle;
-          if (insertRows.containsKey(key)) {
-            int insertRow = insertRows.remove(key);
-            int oldCount = rs.getInt(3);
-            if (insertRow != oldCount) {
-              updateRows.put(key, insertRow);
-            }
-          }
-        }
-        for (Map.Entry<String, Integer> e : updateRows.entrySet()) {
-          String[] keyParts = e.getKey().split(",");
-          java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
-          String bundle = keyParts[1];
-          int downloads = e.getValue();
-          psU.clearParameters();
-          psU.setLong(1, downloads);
-          psU.setDate(2, date);
-          psU.setString(3, bundle);
-          psU.executeUpdate();
-        }
-        for (Map.Entry<String, Integer> e : insertRows.entrySet()) {
-          String[] keyParts = e.getKey().split(",");
-          java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
-          String bundle = keyParts[1];
-          int downloads = e.getValue();
-          psI.clearParameters();
-          psI.setLong(1, downloads);
-          psI.setDate(2, date);
-          psI.setString(3, bundle);
-          psI.executeUpdate();
-        }
-        conn.commit();
-        conn.close();
-      } catch (SQLException e) {
-        logger.log(Level.WARNING, "Failed to add GetTor stats to "
-            + "database.", e);
-      }
-    }
-
-    logger.info("Finished downloading and processing statistics on Tor "
-        + "packages delivered by GetTor.\nDownloaded " + unparsed.length()
-        + " bytes. Last date in statistics is " + data.lastKey() + ".");
-  }
-}
-
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index e10b6c7..657cdfc 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -118,10 +118,9 @@ public class Main {
     }
 
     // Download and process GetTor stats
-    if (config.getDownloadProcessGetTorStats()) {
-      new GetTorProcessor(config.getGetTorStatsUrl(),
-          config.getWriteAggregateStatsDatabase() ?
-          config.getRelayDescriptorDatabaseJDBC() : null);
+    if (config.getDownloadGetTorStats()) {
+      new GetTorDownloader(config.getGetTorStatsUrl(),
+          new File(config.getGetTorDirectory()));
     }
 
     // Download exit list and store it to disk



More information about the tor-commits mailing list