commit f67c7db477740b4f13a949b32c1fec0b5a72c464 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Mar 3 18:25:31 2011 +0100
Remove GetTor stats database import. --- config.template | 15 +-- src/org/torproject/ernie/db/Configuration.java | 39 ++---- src/org/torproject/ernie/db/GetTorDownloader.java | 101 +++++++++++++++ src/org/torproject/ernie/db/GetTorProcessor.java | 143 --------------------- src/org/torproject/ernie/db/Main.java | 7 +- 5 files changed, 122 insertions(+), 183 deletions(-)
diff --git a/config.template b/config.template index 72cc411..0d6743e 100644 --- a/config.template +++ b/config.template @@ -32,12 +32,15 @@ ## Relative path to directory to import bridge descriptor snapshots from #BridgeSnapshotsDirectory bridge-directories/ # -## Download and process GetTor stats -#DownloadProcessGetTorStats 0 +## Download GetTor stats +#DownloadGetTorStats 0 # ## URL to download GetTor stats from #GetTorStatsURL http://gettor.torproject.org:8080/~gettor/gettor_stats.txt # +## Relative path to directory to store GetTor stats in +#GetTorDirectory gettor/ +# ## Download exit list and store it to disk #DownloadExitList 0 # @@ -49,14 +52,6 @@ ## Relative path to directory to write directory archives to #DirectoryArchivesOutputDirectory directory-archive/ # -## Write aggregate statistics (bridges and bridge users per day, directory -## clients per day, torperf results, packages requested from GetTor, etc.) -## to database for later evaluation -#WriteAggregateStatsDatabase 0 -# -## JDBC string for relay descriptor database -#RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=ernie&password=password -# ## Write sanitized bridges to disk #WriteSanitizedBridges 0 # diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index 4d75b59..c1cdbea 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -21,7 +21,6 @@ public class Configuration { private boolean importDirectoryArchives = false; private String directoryArchivesDirectory = "archives/"; private boolean keepDirectoryArchiveImportHistory = false; - private boolean writeAggregateStatsDatabase = false; private String relayDescriptorDatabaseJdbc = "jdbc:postgresql://localhost/tordir?user=ernie&password=password"; private boolean writeSanitizedBridges = false; @@ -33,9 +32,10 @@ public class Configuration { private boolean downloadRelayDescriptors = false; private List<String> downloadFromDirectoryAuthorities = Arrays.asList( "86.59.21.38,194.109.206.212,80.190.246.100:8180".split(",")); - private boolean downloadProcessGetTorStats = false; + private boolean downloadGetTorStats = false; private String getTorStatsUrl = "http://gettor.torproject.org:8080/" + "~gettor/gettor_stats.txt"; + private String getTorDirectory = "gettor/"; private boolean downloadExitList = false; public Configuration() {
@@ -82,11 +82,6 @@ public class Configuration { } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { this.keepDirectoryArchiveImportHistory = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("WriteAggregateStatsDatabase")) { - this.writeAggregateStatsDatabase = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("RelayDescriptorDatabaseJDBC")) { - this.relayDescriptorDatabaseJdbc = line.split(" ")[1]; } else if (line.startsWith("WriteSanitizedBridges")) { this.writeSanitizedBridges = Integer.parseInt( line.split(" ")[1]) != 0; @@ -119,14 +114,16 @@ public class Configuration { new URL("http://" + dir + "/"); this.downloadFromDirectoryAuthorities.add(dir); } - } else if (line.startsWith("DownloadProcessGetTorStats")) { - this.downloadProcessGetTorStats = Integer.parseInt( + } else if (line.startsWith("DownloadGetTorStats")) { + this.downloadGetTorStats = Integer.parseInt( line.split(" ")[1]) != 0; } else if (line.startsWith("GetTorStatsURL")) { String newUrl = line.split(" ")[1]; /* Test if URL has correct format. */ new URL(newUrl); this.getTorStatsUrl = newUrl; + } else if (line.startsWith("GetTorDirectory")) { + this.getTorDirectory = line.split(" ")[1]; } else if (line.startsWith("DownloadExitList")) { this.downloadExitList = Integer.parseInt( line.split(" ")[1]) != 0; @@ -158,9 +155,8 @@ public class Configuration { /** Make some checks if configuration is valid. */ if (!this.importCachedRelayDescriptors && !this.importDirectoryArchives && !this.downloadRelayDescriptors && - !this.importBridgeSnapshots && !this.downloadProcessGetTorStats && + !this.importBridgeSnapshots && !this.downloadGetTorStats && !this.downloadExitList && !this.writeDirectoryArchives && - !this.writeAggregateStatsDatabase && !this.writeSanitizedBridges) { logger.warning("We have not been configured to read data from any " + "data source or write data to any data sink. You need to " @@ -182,8 +178,7 @@ public class Configuration { + "least one data sink, but we don't have a single data source " + "containing relay descriptors."); } - if (this.importBridgeSnapshots && !(this.writeSanitizedBridges || - this.writeAggregateStatsDatabase)) { + if (this.importBridgeSnapshots && !this.writeSanitizedBridges) { logger.warning("We are configured to import/download bridge " + "descriptors, but we don't have a single data sink to write " + "bridge descriptors to."); @@ -193,11 +188,6 @@ public class Configuration { + "least one data sink, but we don't have a single data source " + "containing bridge descriptors."); } - if (this.downloadProcessGetTorStats && - !this.writeAggregateStatsDatabase) { - logger.warning("We are configured to download GetTor statistics, " - + "but not to import them into the database."); - } } public boolean getWriteDirectoryArchives() { return this.writeDirectoryArchives; @@ -220,12 +210,6 @@ public class Configuration { public boolean getKeepDirectoryArchiveImportHistory() { return this.keepDirectoryArchiveImportHistory; } - public boolean getWriteAggregateStatsDatabase() { - return this.writeAggregateStatsDatabase; - } - public String getRelayDescriptorDatabaseJDBC() { - return this.relayDescriptorDatabaseJdbc; - } public boolean getWriteSanitizedBridges() { return this.writeSanitizedBridges; } @@ -250,12 +234,15 @@ public class Configuration { public List<String> getDownloadFromDirectoryAuthorities() { return this.downloadFromDirectoryAuthorities; } - public boolean getDownloadProcessGetTorStats() { - return this.downloadProcessGetTorStats; + public boolean getDownloadGetTorStats() { + return this.downloadGetTorStats; } public String getGetTorStatsUrl() { return this.getTorStatsUrl; } + public String getGetTorDirectory() { + return this.getTorDirectory; + } public boolean getDownloadExitList() { return this.downloadExitList; } diff --git a/src/org/torproject/ernie/db/GetTorDownloader.java b/src/org/torproject/ernie/db/GetTorDownloader.java new file mode 100644 index 0000000..91e9f5f --- /dev/null +++ b/src/org/torproject/ernie/db/GetTorDownloader.java @@ -0,0 +1,101 @@ +/* Copyright 2010 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.db; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.logging.*; + +public class GetTorDownloader { + + public GetTorDownloader(String gettorStatsUrl, File getTorDirectory) { + + Logger logger = Logger.getLogger(GetTorDownloader.class.getName()); + + File getTorFile = new File(getTorDirectory, "gettor_stats.txt"); + SortedMap<String, String> getTorStats = new TreeMap<String, String>(); + + if (getTorFile.exists() && !getTorFile.isDirectory()) { + try { + logger.fine("Reading local gettor_stats.txt file..."); + BufferedReader br = new BufferedReader(new FileReader( + getTorFile)); + String line = null; + while ((line = br.readLine()) != null) { + String date = line.split(" ")[0]; + getTorStats.put(date, line); + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed parsing local GetTor stats!", + e); + return; + } + } + + String unparsed = null; + try { + logger.fine("Downloading GetTor stats..."); + URL u = new URL(gettorStatsUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + int response = huc.getResponseCode(); + if (response == 200) { + BufferedInputStream in = new BufferedInputStream( + huc.getInputStream()); + StringBuilder sb = new StringBuilder(); + int len; + byte[] data = new byte[1024]; + while ((len = in.read(data, 0, 1024)) >= 0) { + sb.append(new String(data, 0, len)); + } + in.close(); + unparsed = sb.toString(); + } + logger.fine("Finished downloading GetTor stats."); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed downloading GetTor stats", e); + return; + } + + try { + logger.fine("Parsing downloaded GetTor stats..."); + BufferedReader br = new BufferedReader(new StringReader(unparsed)); + String line = null; + while ((line = br.readLine()) != null) { + String date = line.split(" ")[0]; + getTorStats.put(date, line); + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed parsing downloaded GetTor stats!", + e); + return; + } + + try { + logger.fine("Writing GetTor stats to local gettor_stats.txt " + + "file..."); + if (!getTorDirectory.exists()) { + getTorDirectory.mkdirs(); + } + BufferedWriter bw = new BufferedWriter(new FileWriter(getTorFile)); + for (String line : getTorStats.values()) { + bw.write(line + "\n"); + } + bw.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed writing GetTor stats to local " + + "gettor_stats.txt file", e); + return; + } + + logger.info("Finished downloading and processing statistics on Tor " + + "packages delivered by GetTor.\nDownloaded " + unparsed.length() + + " bytes. Last date in statistics is " + getTorStats.lastKey() + + "."); + } +} + diff --git a/src/org/torproject/ernie/db/GetTorProcessor.java b/src/org/torproject/ernie/db/GetTorProcessor.java deleted file mode 100644 index 6d9a935..0000000 --- a/src/org/torproject/ernie/db/GetTorProcessor.java +++ /dev/null @@ -1,143 +0,0 @@ -/* Copyright 2010 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.*; -import java.net.*; -import java.sql.*; -import java.util.*; -import java.util.logging.*; - -public class GetTorProcessor { - public GetTorProcessor(String gettorStatsUrl, String connectionURL) { - Logger logger = Logger.getLogger(GetTorProcessor.class.getName()); - String unparsed = null; - try { - logger.fine("Downloading gettor stats..."); - URL u = new URL(gettorStatsUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int response = huc.getResponseCode(); - if (response == 200) { - BufferedInputStream in = new BufferedInputStream( - huc.getInputStream()); - StringBuilder sb = new StringBuilder(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - sb.append(new String(data, 0, len)); - } - in.close(); - unparsed = sb.toString(); - } - logger.fine("Finished downloading gettor stats."); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed downloading gettor stats", e); - return; - } - - SortedSet<String> columns = new TreeSet<String>(); - SortedMap<String, Map<String, Integer>> data = - new TreeMap<String, Map<String, Integer>>(); - try { - logger.fine("Parsing downloaded gettor stats..."); - BufferedReader br = new BufferedReader(new StringReader(unparsed)); - String line = null; - while ((line = br.readLine()) != null) { - String[] parts = line.split(" "); - String date = parts[0]; - Map<String, Integer> obs = new HashMap<String, Integer>(); - data.put(date, obs); - for (int i = 2; i < parts.length; i++) { - String key = parts[i].split(":")[0].toLowerCase(); - Integer value = new Integer(parts[i].split(":")[1]); - columns.add(key); - obs.put(key, value); - } - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed parsing gettor stats!", e); - return; - } catch (NumberFormatException e) { - logger.log(Level.WARNING, "Failed parsing gettor stats!", e); - return; - } - - /* Write results to database. */ - if (connectionURL != null) { - try { - Map<String, Integer> updateRows = new HashMap<String, Integer>(), - insertRows = new HashMap<String, Integer>(); - for (Map.Entry<String, Map<String, Integer>> e : - data.entrySet()) { - String date = e.getKey(); - Map<String, Integer> obs = e.getValue(); - for (String column : columns) { - if (obs.containsKey(column)) { - Integer value = obs.get(column); - String key = date + "," + column; - insertRows.put(key, value); - } - } - } - Connection conn = DriverManager.getConnection(connectionURL); - PreparedStatement psI = conn.prepareStatement( - "INSERT INTO gettor_stats (downloads, date, bundle) " - + "VALUES (?, ?, ?)"); - PreparedStatement psU = conn.prepareStatement( - "UPDATE gettor_stats SET downloads = ? " - + "WHERE date = ? AND bundle = ?"); - conn.setAutoCommit(false); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "SELECT date, bundle, downloads FROM gettor_stats"); - while (rs.next()) { - String date = rs.getDate(1).toString(); - String bundle = rs.getString(2); - String key = date + "," + bundle; - if (insertRows.containsKey(key)) { - int insertRow = insertRows.remove(key); - int oldCount = rs.getInt(3); - if (insertRow != oldCount) { - updateRows.put(key, insertRow); - } - } - } - for (Map.Entry<String, Integer> e : updateRows.entrySet()) { - String[] keyParts = e.getKey().split(","); - java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); - String bundle = keyParts[1]; - int downloads = e.getValue(); - psU.clearParameters(); - psU.setLong(1, downloads); - psU.setDate(2, date); - psU.setString(3, bundle); - psU.executeUpdate(); - } - for (Map.Entry<String, Integer> e : insertRows.entrySet()) { - String[] keyParts = e.getKey().split(","); - java.sql.Date date = java.sql.Date.valueOf(keyParts[0]); - String bundle = keyParts[1]; - int downloads = e.getValue(); - psI.clearParameters(); - psI.setLong(1, downloads); - psI.setDate(2, date); - psI.setString(3, bundle); - psI.executeUpdate(); - } - conn.commit(); - conn.close(); - } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to add GetTor stats to " - + "database.", e); - } - } - - logger.info("Finished downloading and processing statistics on Tor " - + "packages delivered by GetTor.\nDownloaded " + unparsed.length() - + " bytes. Last date in statistics is " + data.lastKey() + "."); - } -} - diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index e10b6c7..657cdfc 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -118,10 +118,9 @@ public class Main { }
// Download and process GetTor stats - if (config.getDownloadProcessGetTorStats()) { - new GetTorProcessor(config.getGetTorStatsUrl(), - config.getWriteAggregateStatsDatabase() ? - config.getRelayDescriptorDatabaseJDBC() : null); + if (config.getDownloadGetTorStats()) { + new GetTorDownloader(config.getGetTorStatsUrl(), + new File(config.getGetTorDirectory())); }
// Download exit list and store it to disk