[tor-commits] [metrics-web/master] Import GetTor stats into database.
karsten at torproject.org
karsten at torproject.org
Thu Mar 3 18:38:21 UTC 2011
commit e0934bf15e83c8020351c890b6efc7c78f4e3492
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Mar 3 18:24:19 2011 +0100
Import GetTor stats into database.
---
config.template | 6 +
src/org/torproject/ernie/cron/Configuration.java | 13 ++
src/org/torproject/ernie/cron/GetTorProcessor.java | 124 ++++++++++++++++++++
src/org/torproject/ernie/cron/Main.java | 7 +
4 files changed, 150 insertions(+), 0 deletions(-)
diff --git a/config.template b/config.template
index 4a5588a..fd37f4d 100644
--- a/config.template
+++ b/config.template
@@ -48,4 +48,10 @@
#
## Relative path to directory to import torperf results from
#TorperfDirectory torperf/
+#
+## Process GetTor stats and import them into the database
+#ProcessGetTorStats 0
+#
+## Relative path to directory where to find GetTor stats
+#GetTorDirectory gettor/
diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java
index dd09bae..e2c630b 100644
--- a/src/org/torproject/ernie/cron/Configuration.java
+++ b/src/org/torproject/ernie/cron/Configuration.java
@@ -28,6 +28,8 @@ public class Configuration {
private boolean writeBridgeStats = false;
private boolean importWriteTorperfStats = false;
private String torperfDirectory = "torperf/";
+ private boolean processGetTorStats = false;
+ private String getTorDirectory = "gettor/";
public Configuration() {
/* Initialize logger. */
@@ -82,6 +84,11 @@ public class Configuration {
line.split(" ")[1]) != 0;
} else if (line.startsWith("TorperfDirectory")) {
this.torperfDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ProcessGetTorStats")) {
+ this.processGetTorStats = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("GetTorDirectory")) {
+ this.getTorDirectory = line.split(" ")[1];
} else {
logger.severe("Configuration file contains unrecognized "
+ "configuration key in line '" + line + "'! Exiting!");
@@ -149,5 +156,11 @@ public class Configuration {
public String getTorperfDirectory() {
return this.torperfDirectory;
}
+ public boolean getProcessGetTorStats() {
+ return this.processGetTorStats;
+ }
+ public String getGetTorDirectory() {
+ return this.getTorDirectory;
+ }
}
diff --git a/src/org/torproject/ernie/cron/GetTorProcessor.java b/src/org/torproject/ernie/cron/GetTorProcessor.java
new file mode 100644
index 0000000..a6ddf11
--- /dev/null
+++ b/src/org/torproject/ernie/cron/GetTorProcessor.java
@@ -0,0 +1,124 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.sql.*;
+import java.util.*;
+import java.util.logging.*;
+
+public class GetTorProcessor {
+ public GetTorProcessor(File getTorDirectory, String connectionURL) {
+
+ Logger logger = Logger.getLogger(GetTorProcessor.class.getName());
+
+ /* Parse stats file. */
+ File getTorFile = new File(getTorDirectory, "gettor_stats.txt");
+ if (!getTorFile.exists() || getTorFile.isDirectory()) {
+ logger.warning("Could not read GetTor stats");
+ return;
+ }
+ SortedSet<String> columns = new TreeSet<String>();
+ SortedMap<String, Map<String, Integer>> data =
+ new TreeMap<String, Map<String, Integer>>();
+ try {
+ logger.fine("Parsing GetTor stats...");
+ BufferedReader br = new BufferedReader(new FileReader(getTorFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ String date = parts[0];
+ Map<String, Integer> obs = new HashMap<String, Integer>();
+ data.put(date, obs);
+ for (int i = 2; i < parts.length; i++) {
+ String key = parts[i].split(":")[0].toLowerCase();
+ Integer value = new Integer(parts[i].split(":")[1]);
+ columns.add(key);
+ obs.put(key, value);
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed parsing GetTor stats!", e);
+ return;
+ } catch (NumberFormatException e) {
+ logger.log(Level.WARNING, "Failed parsing GetTor stats!", e);
+ return;
+ }
+
+ /* Write results to database. */
+ if (connectionURL != null) {
+ try {
+ Map<String, Integer> updateRows = new HashMap<String, Integer>(),
+ insertRows = new HashMap<String, Integer>();
+ for (Map.Entry<String, Map<String, Integer>> e :
+ data.entrySet()) {
+ String date = e.getKey();
+ Map<String, Integer> obs = e.getValue();
+ for (String column : columns) {
+ if (obs.containsKey(column)) {
+ Integer value = obs.get(column);
+ String key = date + "," + column;
+ insertRows.put(key, value);
+ }
+ }
+ }
+ Connection conn = DriverManager.getConnection(connectionURL);
+ PreparedStatement psI = conn.prepareStatement(
+ "INSERT INTO gettor_stats (downloads, date, bundle) "
+ + "VALUES (?, ?, ?)");
+ PreparedStatement psU = conn.prepareStatement(
+ "UPDATE gettor_stats SET downloads = ? "
+ + "WHERE date = ? AND bundle = ?");
+ conn.setAutoCommit(false);
+ Statement statement = conn.createStatement();
+ ResultSet rs = statement.executeQuery(
+ "SELECT date, bundle, downloads FROM gettor_stats");
+ while (rs.next()) {
+ String date = rs.getDate(1).toString();
+ String bundle = rs.getString(2);
+ String key = date + "," + bundle;
+ if (insertRows.containsKey(key)) {
+ int insertRow = insertRows.remove(key);
+ int oldCount = rs.getInt(3);
+ if (insertRow != oldCount) {
+ updateRows.put(key, insertRow);
+ }
+ }
+ }
+ for (Map.Entry<String, Integer> e : updateRows.entrySet()) {
+ String[] keyParts = e.getKey().split(",");
+ java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
+ String bundle = keyParts[1];
+ int downloads = e.getValue();
+ psU.clearParameters();
+ psU.setLong(1, downloads);
+ psU.setDate(2, date);
+ psU.setString(3, bundle);
+ psU.executeUpdate();
+ }
+ for (Map.Entry<String, Integer> e : insertRows.entrySet()) {
+ String[] keyParts = e.getKey().split(",");
+ java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
+ String bundle = keyParts[1];
+ int downloads = e.getValue();
+ psI.clearParameters();
+ psI.setLong(1, downloads);
+ psI.setDate(2, date);
+ psI.setString(3, bundle);
+ psI.executeUpdate();
+ }
+ conn.commit();
+ conn.close();
+ } catch (SQLException e) {
+ logger.log(Level.WARNING, "Failed to add GetTor stats to "
+ + "database.", e);
+ }
+ }
+
+ logger.info("Finished processing statistics on Tor packages "
+ + "delivered by GetTor.\nLast date in statistics is "
+ + data.lastKey() + ".");
+ }
+}
+
diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java
index 943d326..ec02130 100644
--- a/src/org/torproject/ernie/cron/Main.java
+++ b/src/org/torproject/ernie/cron/Main.java
@@ -111,6 +111,13 @@ public class Main {
statsDirectory, config.getRelayDescriptorDatabaseJDBC());
}
+ // Download and process GetTor stats
+ if (config.getProcessGetTorStats()) {
+ new GetTorProcessor(
+ new File(config.getGetTorDirectory()),
+ config.getRelayDescriptorDatabaseJDBC());
+ }
+
// Remove lock file
lf.releaseLock();
More information about the tor-commits
mailing list