commit e0934bf15e83c8020351c890b6efc7c78f4e3492 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Mar 3 18:24:19 2011 +0100

Import GetTor stats into database. --- config.template | 6 + src/org/torproject/ernie/cron/Configuration.java | 13 ++ src/org/torproject/ernie/cron/GetTorProcessor.java | 124 ++++++++++++++++++++ src/org/torproject/ernie/cron/Main.java | 7 + 4 files changed, 150 insertions(+), 0 deletions(-)

diff --git a/config.template b/config.template index 4a5588a..fd37f4d 100644 --- a/config.template +++ b/config.template @@ -48,4 +48,10 @@ # ## Relative path to directory to import torperf results from #TorperfDirectory torperf/ +# +## Process GetTor stats and import them into the database +#ProcessGetTorStats 0 +# +## Relative path to directory where to find GetTor stats +#GetTorDirectory gettor/

diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java index dd09bae..e2c630b 100644 --- a/src/org/torproject/ernie/cron/Configuration.java +++ b/src/org/torproject/ernie/cron/Configuration.java @@ -28,6 +28,8 @@ public class Configuration { private boolean writeBridgeStats = false; private boolean importWriteTorperfStats = false; private String torperfDirectory = "torperf/"; + private boolean processGetTorStats = false; + private String getTorDirectory = "gettor/"; public Configuration() {

/* Initialize logger. */ @@ -82,6 +84,11 @@ public class Configuration { line.split(" ")[1]) != 0; } else if (line.startsWith("TorperfDirectory")) { this.torperfDirectory = line.split(" ")[1]; + } else if (line.startsWith("ProcessGetTorStats")) { + this.processGetTorStats = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("GetTorDirectory")) { + this.getTorDirectory = line.split(" ")[1]; } else { logger.severe("Configuration file contains unrecognized " + "configuration key in line '" + line + "'! Exiting!"); @@ -149,5 +156,11 @@ public class Configuration { public String getTorperfDirectory() { return this.torperfDirectory; } + public boolean getProcessGetTorStats() { + return this.processGetTorStats; + } + public String getGetTorDirectory() { + return this.getTorDirectory; + } }

diff --git a/src/org/torproject/ernie/cron/GetTorProcessor.java b/src/org/torproject/ernie/cron/GetTorProcessor.java new file mode 100644 index 0000000..a6ddf11 --- /dev/null +++ b/src/org/torproject/ernie/cron/GetTorProcessor.java @@ -0,0 +1,124 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.sql.*; +import java.util.*; +import java.util.logging.*; + +public class GetTorProcessor { + public GetTorProcessor(File getTorDirectory, String connectionURL) { + + Logger logger = Logger.getLogger(GetTorProcessor.class.getName()); + + /* Parse stats file. */ + File getTorFile = new File(getTorDirectory, "gettor_stats.txt"); + if (!getTorFile.exists() || getTorFile.isDirectory()) { + logger.warning("Could not read GetTor stats"); + return; + } + SortedSet<String> columns = new TreeSet<String>(); + SortedMap<String, Map<String, Integer>> data = + new TreeMap<String, Map<String, Integer>>(); + try { + logger.fine("Parsing GetTor stats..."); + BufferedReader br = new BufferedReader(new FileReader(getTorFile)); + String line = null; + while ((line = br.readLine()) != null) { + String[] parts = line.split(" "); + String date = parts[0]; + Map<String, Integer> obs = new HashMap<String, Integer>(); + data.put(date, obs); + for (int i = 2; i < parts.length; i++) { + String key = parts[i].split(":")[0].toLowerCase(); + Integer value = new Integer(parts[i].split(":")[1]); + columns.add(key); + obs.put(key, value); + } + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed parsing GetTor stats!", e); + return; + } catch (NumberFormatException e) { + logger.log(Level.WARNING, "Failed parsing GetTor stats!", e); + return; + } + + /* Write results to database. */ + if (connectionURL != null) { + try { + Map<String, Integer> updateRows = new HashMap<String, Integer>(), + insertRows = new HashMap<String, Integer>(); + for (Map.Entry<String, Map<String, Integer>> e : + data.entrySet()) { + String date = e.getKey(); + Map<String, Integer> obs = e.getValue(); + for (String column : columns) { + if (obs.containsKey(column)) { + Integer value = obs.get(column); + String key = date + "," + column; + insertRows.put(key, value); + } + } + } + Connection conn = DriverManager.getConnection(connectionURL); + PreparedStatement psI = conn.prepareStatement( + "INSERT INTO gettor_stats (downloads, date, bundle) " + + "VALUES (?, ?, ?)"); + PreparedStatement psU = conn.prepareStatement( + "UPDATE gettor_stats SET downloads = ? " + + "WHERE date = ? AND bundle = ?"); + conn.setAutoCommit(false); + Statement statement = conn.createStatement(); + ResultSet rs = statement.executeQuery( + "SELECT date, bundle, downloads FROM gettor_stats"); + while (rs.next()) { + String date = rs.getDate(1).toString(); + String bundle = rs.getString(2); + String key = date + "," + bundle; + if (insertRows.containsKey(key)) { + int insertRow = insertRows.remove(key); + int oldCount = rs.getInt(3); + if (insertRow != oldCount) { + updateRows.put(key, insertRow); + } + } + } + for (Map.Entry<String, Integer> e : updateRows.entrySet()) { + String[] keyParts = e.getKey().split(","); + java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); + String bundle = keyParts[1]; + int downloads = e.getValue(); + psU.clearParameters(); + psU.setLong(1, downloads); + psU.setDate(2, date); + psU.setString(3, bundle); + psU.executeUpdate(); + } + for (Map.Entry<String, Integer> e : insertRows.entrySet()) { + String[] keyParts = e.getKey().split(","); + java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); + String bundle = keyParts[1]; + int downloads = e.getValue(); + psI.clearParameters(); + psI.setLong(1, downloads); + psI.setDate(2, date); + psI.setString(3, bundle); + psI.executeUpdate(); + } + conn.commit(); + conn.close(); + } catch (SQLException e) { + logger.log(Level.WARNING, "Failed to add GetTor stats to " + + "database.", e); + } + } + + logger.info("Finished processing statistics on Tor packages " + + "delivered by GetTor.\nLast date in statistics is " + + data.lastKey() + "."); + } +} + diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java index 943d326..ec02130 100644 --- a/src/org/torproject/ernie/cron/Main.java +++ b/src/org/torproject/ernie/cron/Main.java @@ -111,6 +111,13 @@ public class Main { statsDirectory, config.getRelayDescriptorDatabaseJDBC()); }

+ // Download and process GetTor stats + if (config.getProcessGetTorStats()) { + new GetTorProcessor( + new File(config.getGetTorDirectory()), + config.getRelayDescriptorDatabaseJDBC()); + } + // Remove lock file lf.releaseLock();